From d50f9b973b956e367144d878fdf317f738f9e4e7 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 28 Jun 2022 01:09:15 +0200 Subject: [PATCH] feat: allow to save and load metric data metrics can be saved and loaded to/from csv. The files are ok-ish and it's reasonably fast. File format could still change. Also some small fixes and tweaks, like bigger buttons in confirmation prompts and source name in logs. --- Cargo.toml | 4 +- src/app/data/mod.rs | 1 + src/app/data/store.rs | 19 ++- src/app/gui/scaffold.rs | 8 +- src/app/gui/source.rs | 298 +++++++++++++++++++++++++--------------- src/app/mod.rs | 8 +- src/app/util.rs | 43 +++++- src/app/worker.rs | 5 +- 8 files changed, 256 insertions(+), 130 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fc7754d..7cb410e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dashboard" -version = "0.3.3" +version = "0.3.4" edition = "2021" [[bin]] @@ -21,7 +21,9 @@ tracing = "0.1" # egui / eframe use tracing tracing-subscriber = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" +csv = "1.1" rusqlite = "0.27" jql = { version = "4", default-features = false } ureq = { version = "2", features = ["json"] } +rfd = "0.9" eframe = "0.18" diff --git a/src/app/data/mod.rs b/src/app/data/mod.rs index 15a354b..36429e8 100644 --- a/src/app/data/mod.rs +++ b/src/app/data/mod.rs @@ -92,6 +92,7 @@ impl ApplicationState { false, false, false, + true, self.panels.read().expect("Panels RwLock poisoned").len() as i32, // todo can this be made more compact and without acquisition? )?; // TODO make values customizable and useful self.panels diff --git a/src/app/data/store.rs b/src/app/data/store.rs index 99f22d3..5338510 100644 --- a/src/app/data/store.rs +++ b/src/app/data/store.rs @@ -102,13 +102,25 @@ impl SQLiteDataStore { Ok(values) } - pub fn put_value(&self, metric_id: i32, v: Value) -> rusqlite::Result { + pub fn put_value(&self, metric_id: i32, v: &Value) -> rusqlite::Result { self.conn.execute( "INSERT INTO points(metric_id, x, y) VALUES (?, ?, ?)", params![metric_id, v.x, v.y], ) } + pub fn put_values(&mut self, metric_id: i32, values: &Vec) -> rusqlite::Result<()> { + let tx = self.conn.transaction()?; + for v in values { + tx.execute( + "INSERT INTO points(metric_id, x, y) VALUES (?, ?, ?)", + params![metric_id, v.x, v.y], + )?; + } + tx.commit()?; + Ok(()) + } + pub fn delete_values(&self, metric_id: i32) -> rusqlite::Result { self.conn.execute( "DELETE FROM points WHERE metric_id = ?", @@ -327,11 +339,12 @@ impl SQLiteDataStore { limit: bool, reduce: bool, shift: bool, + average: bool, position: i32, ) -> rusqlite::Result { self.conn.execute( - "INSERT INTO panels (name, view_scroll, view_size, timeserie, width, height, limit_view, position, reduce_view, view_chunks, shift_view, view_offset) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - params![name, view_scroll, view_size, timeserie, width, height, limit, position, reduce, view_chunks, shift, view_offset] + "INSERT INTO panels (name, view_scroll, view_size, timeserie, width, height, limit_view, position, reduce_view, view_chunks, shift_view, view_offset, average_view) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + params![name, view_scroll, view_size, timeserie, width, height, limit, position, reduce, view_chunks, shift, view_offset, average] )?; let mut statement = self.conn.prepare("SELECT * FROM panels WHERE name = ?")?; for panel in statement.query_map(params![name], |row| { diff --git a/src/app/gui/scaffold.rs b/src/app/gui/scaffold.rs index 1921638..ee29827 100644 --- a/src/app/gui/scaffold.rs +++ b/src/app/gui/scaffold.rs @@ -13,7 +13,7 @@ pub fn confirmation_popup_delete_metric(app: &mut App, ui: &mut Ui, metric_index ui.label("This will remove all its metrics and delete all points from archive. This action CANNOT BE UNDONE!"); ui.with_layout(Layout::top_down(Align::RIGHT), |ui| { ui.horizontal(|ui| { - if ui.button("yes").clicked() { + if ui.button("\n yes \n").clicked() { let store = app.data.storage.lock().expect("Storage Mutex poisoned"); let mut metrics = app.data.metrics.write().expect("Metrics RwLock poisoned"); store.delete_metric(metrics[metric_index].id).expect("Failed deleting metric"); @@ -21,7 +21,7 @@ pub fn confirmation_popup_delete_metric(app: &mut App, ui: &mut Ui, metric_index metrics.remove(metric_index); app.deleting_metric = None; } - if ui.button(" no ").clicked() { + if ui.button("\n no \n").clicked() { app.deleting_metric = None; } }); @@ -34,7 +34,7 @@ pub fn confirmation_popup_delete_source(app: &mut App, ui: &mut Ui, source_index ui.label("This will remove all its metrics and delete all points from archive. This action CANNOT BE UNDONE!"); ui.with_layout(Layout::top_down(Align::RIGHT), |ui| { ui.horizontal(|ui| { - if ui.button("YEAH").clicked() { + if ui.button("\n yes \n").clicked() { let store = app.data.storage.lock().expect("Storage Mutex poisoned"); let mut sources = app.data.sources.write().expect("sources RwLock poisoned"); let mut metrics = app.data.metrics.write().expect("Metrics RwLock poisoned"); @@ -53,7 +53,7 @@ pub fn confirmation_popup_delete_source(app: &mut App, ui: &mut Ui, source_index sources.remove(source_index); app.deleting_source = None; } - if ui.button(" NO WAY ").clicked() { + if ui.button("\n no \n").clicked() { app.deleting_source = None; } }); diff --git a/src/app/gui/source.rs b/src/app/gui/source.rs index fc03c86..40573ec 100644 --- a/src/app/gui/source.rs +++ b/src/app/gui/source.rs @@ -1,134 +1,197 @@ -use eframe::{egui::{Ui, TextEdit, DragValue, Checkbox, ScrollArea, Layout}, emath::Align}; +use eframe::{ + egui::{Checkbox, DragValue, Layout, ScrollArea, TextEdit, Ui}, + emath::Align, epaint::Color32, +}; +use rfd::FileDialog; use tracing::error; -use crate::app::{data::source::{Source, Metric}, App}; +use crate::app::{ + data::source::{Metric, Source}, + util::{deserialize_values, serialize_values}, + App, +}; -use super::metric::{metric_edit_ui, metric_display_ui}; +use super::metric::{metric_display_ui, metric_edit_ui}; pub fn source_panel(app: &mut App, ui: &mut Ui) { let mut to_swap: Option = None; + let mut to_insert: Vec = Vec::new(); // let mut to_delete: Option = None; let panels = app.data.panels.read().expect("Panels RwLock poisoned"); let panel_width = ui.available_width(); - ScrollArea::both().max_width(panel_width).show(ui, |ui| { - // TODO only vertical! - { - let mut sources = - app.data.sources.write().expect("Sources RwLock poisoned"); - let sources_count = sources.len(); - ui.heading("Sources"); - ui.separator(); - for (i, source) in sources.iter_mut().enumerate() { - ui.horizontal(|ui| { - if app.edit { - ui.vertical(|ui| { - ui.add_space(10.0); - if ui.small_button("+").clicked() { - if i > 0 { - to_swap = Some(i); // TODO kinda jank but is there a better way? - } - } - if ui.small_button("−").clicked() { - if i < sources_count - 1 { - to_swap = Some(i + 1); // TODO kinda jank but is there a better way? - } - } - }); - } - ui.vertical(|ui| { - let remaining_width = ui.available_width(); - if app.edit { - ui.group(|ui| { - ui.horizontal(|ui| { - source_edit_ui( - ui, - source, - remaining_width - 34.0, - ); - if ui.small_button("×").clicked() { - app.deleting_metric = None; - app.deleting_source = Some(i); - } - }); - for (j, metric) in app.data.metrics.write().expect("Metrics RwLock poisoned").iter_mut().enumerate() { - if metric.source_id == source.id { - ui.horizontal(|ui| { - metric_edit_ui(ui, metric, Some(&panels), remaining_width - 31.0); - if ui.small_button("×").clicked() { - app.deleting_source = None; - app.deleting_metric = Some(j); - } - }); + ScrollArea::vertical() + .max_width(panel_width) + .show(ui, |ui| { + // TODO only vertical! + { + let mut sources = app.data.sources.write().expect("Sources RwLock poisoned"); + let sources_count = sources.len(); + ui.heading("Sources"); + ui.separator(); + for (i, source) in sources.iter_mut().enumerate() { + ui.horizontal(|ui| { + if app.edit { // show buttons to move sources up and down + ui.vertical(|ui| { + ui.add_space(10.0); + if ui.small_button("+").clicked() { + if i > 0 { + to_swap = Some(i); // TODO kinda jank but is there a better way? } } - ui.horizontal(|ui| { - metric_edit_ui( - ui, - &mut app.input_metric, - None, - remaining_width - 30.0, - ); - if ui.small_button(" + ").clicked() { // TODO find a better - if let Err(e) = app - .data - .add_metric(&app.input_metric, source) - { - error!(target: "ui", "Error adding metric : {:?}", e); + if ui.small_button("−").clicked() { + if i < sources_count - 1 { + to_swap = Some(i + 1); // TODO kinda jank but is there a better way? + } + } + }); + } + ui.vertical(|ui| { // actual sources list container + let remaining_width = ui.available_width(); + if app.edit { + ui.group(|ui| { + ui.horizontal(|ui| { + source_edit_ui(ui, source, remaining_width - 34.0); + if ui.small_button("×").clicked() { + app.deleting_metric = None; + app.deleting_source = Some(i); + } + }); + let mut metrics = app + .data + .metrics + .write() + .expect("Metrics RwLock poisoned"); + for (j, metric) in metrics.iter_mut().enumerate() { + if metric.source_id == source.id { + ui.horizontal(|ui| { + metric_edit_ui( + ui, + metric, + Some(&panels), + remaining_width - 53.0, + ); + if ui.small_button("s").clicked() { + let path = FileDialog::new() + .add_filter("csv", &["csv"]) + .set_file_name(format!("{}-{}.csv", source.name, metric.name).as_str()) + .save_file(); + if let Some(path) = path { + serialize_values( + &*metric + .data + .read() + .expect("Values RwLock poisoned"), + metric, + path, + ) + .expect("Could not serialize data"); + } + } + if ui.small_button("×").clicked() { + app.deleting_source = None; + app.deleting_metric = Some(j); + } + }); } } - ui.add_space(1.0); // DAMN! - if ui.small_button("×").clicked() { - app.input_metric = Metric::default(); + ui.horizontal(|ui| { + metric_edit_ui( + ui, + &mut app.input_metric, + None, + remaining_width - 53.0, + ); + ui.add_space(2.0); + if ui.small_button(" + ").clicked() { + // TODO find a better + if let Err(e) = + app.data.add_metric(&app.input_metric, source) + { + error!(target: "ui", "Error adding metric : {:?}", e); + } + } + ui.add_space(1.0); // DAMN! + if ui.small_button("o").clicked() { + let path = FileDialog::new() + .add_filter("csv", &["csv"]) + .pick_file(); + if let Some(path) = path { + match deserialize_values(path) { + Ok((name, query_x, query_y, data)) => { + let mut store = app + .data + .storage + .lock() + .expect("Storage Mutex poisoned"); + match store.new_metric( + name.as_str(), + source.id, + query_x.as_str(), + query_y.as_str(), + -1, + Color32::TRANSPARENT, + metrics.len() as i32, + ) { + Ok(verified_metric) => { + store.put_values(verified_metric.id, &data).unwrap(); + *verified_metric.data.write().expect("Values RwLock poisoned") = data; + to_insert.push(verified_metric); + } + Err(e) => { + error!(target: "ui", "could not save metric into archive : {:?}", e); + } + } + } + Err(e) => { + error!(target: "ui", "Could not deserialize metric from file : {:?}", e); + } + } + } + } + if ui.small_button("×").clicked() { + app.input_metric = Metric::default(); + } + }) + }); + } else { + let metrics = + app.data.metrics.read().expect("Metrics RwLock poisoned"); + source_display_ui(ui, source, remaining_width); + for metric in metrics.iter() { + if metric.source_id == source.id { + metric_display_ui(ui, metric, ui.available_width()); } - }) - }); - } else { - let metrics = - app.data.metrics.read().expect("Metrics RwLock poisoned"); - source_display_ui( - ui, - source, - remaining_width, - ); - for metric in metrics.iter() { - if metric.source_id == source.id { - metric_display_ui(ui, metric, ui.available_width()); + } + ui.separator(); + } + }); + }); + } + } + if app.edit { + ui.separator(); + ui.horizontal(|ui| { + ui.heading("new source"); + ui.with_layout(Layout::top_down(Align::RIGHT), |ui| { + ui.horizontal(|ui| { + if ui.button("add").clicked() { + if let Err(e) = app.data.add_source(&app.input_source) { + error!(target: "ui", "Error adding source : {:?}", e); + } else { + app.input_source.id += 1; } } - ui.separator(); - } + ui.toggle_value(&mut app.padding, "#"); + }); }); }); + source_edit_ui(ui, &mut app.input_source, panel_width - 10.0); + ui.add_space(5.0); + if app.padding { + ui.add_space(300.0); + } } - } - if app.edit { - ui.separator(); - ui.horizontal(|ui| { - ui.heading("new source"); - ui.with_layout(Layout::top_down(Align::RIGHT), |ui| { - ui.horizontal(|ui| { - if ui.button("add").clicked() { - if let Err(e) = app.data.add_source(&app.input_source) { - error!(target: "ui", "Error adding source : {:?}", e); - } else { - app.input_source.id += 1; - } - } - ui.toggle_value(&mut app.padding, "#"); - }); - }); - }); - source_edit_ui( - ui, - &mut app.input_source, - panel_width - 10.0, - ); - ui.add_space(5.0); - if app.padding { - ui.add_space(300.0); - } - } - }); + }); //if let Some(i) = to_delete { // // TODO can this be done in background? idk // let mut panels = app.data.panels.write().expect("Panels RwLock poisoned"); @@ -139,12 +202,21 @@ pub fn source_panel(app: &mut App, ui: &mut Ui) { let mut sources = app.data.sources.write().expect("Sources RwLock poisoned"); sources.swap(i - 1, i); } + if to_insert.len() > 0 { + let mut metrics = app.data.metrics.write().expect("Metrics RwLock poisoned"); + for m in to_insert { + metrics.push(m); + } + } } pub fn source_display_ui(ui: &mut Ui, source: &mut Source, _width: f32) { ui.horizontal(|ui| { ui.add_enabled(false, Checkbox::new(&mut source.enabled, "")); - ui.add_enabled(false, DragValue::new(&mut source.interval).clamp_range(1..=120)); + ui.add_enabled( + false, + DragValue::new(&mut source.interval).clamp_range(1..=120), + ); ui.heading(&source.name).on_hover_text(&source.url); }); } diff --git a/src/app/mod.rs b/src/app/mod.rs index 0855155..b2c4854 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -54,18 +54,18 @@ impl eframe::App for App { }); if let Some(index) = self.deleting_metric { - Window::new(format!("Delete Metric #{}", index)) + Window::new(format!("Delete Metric #{}?", index)) .show(ctx, |ui| confirmation_popup_delete_metric(self, ui, index)); } if let Some(index) = self.deleting_source { - Window::new(format!("Delete Source #{}", index)) + Window::new(format!("Delete Source #{}?", index)) .show(ctx, |ui| confirmation_popup_delete_source(self, ui, index)); } if self.sources { SidePanel::left("sources-bar") - .width_range(280.0..=800.0) - .default_width(330.0) + .width_range(if self.edit { 400.0..=1000.0 } else { 280.0..=680.0 }) + .default_width(if self.edit { 450.0 } else { 330.0 }) .show(ctx, |ui| source_panel(self, ui)); } diff --git a/src/app/util.rs b/src/app/util.rs index 5dcdfd2..41bbca3 100644 --- a/src/app/util.rs +++ b/src/app/util.rs @@ -1,13 +1,50 @@ use chrono::{DateTime, Local, NaiveDateTime, Utc}; -use eframe::egui::Color32; -use std::sync::Arc; +use eframe::egui::{Color32, plot::Value}; +use std::{sync::Arc, error::Error, path::PathBuf}; use tracing_subscriber::Layer; -use super::data::ApplicationState; +use super::data::{ApplicationState, source::Metric}; // if you're handling more than terabytes of data, it's the future and you ought to update this code! const PREFIXES: &'static [&'static str] = &["", "k", "M", "G", "T"]; +pub fn serialize_values(values: &Vec, metric: &Metric, path: PathBuf) -> Result<(), Box> { + let mut wtr = csv::Writer::from_writer(std::fs::File::create(path)?); + wtr.write_record(&[metric.name.as_str(), metric.query_x.as_str(), metric.query_y.as_str()])?; + for v in values { + wtr.serialize(("", v.x, v.y))?; + } + wtr.flush()?; + Ok(()) +} + +pub fn deserialize_values(path: PathBuf) -> Result<(String, String, String, Vec), Box> { + let mut values = Vec::new(); + + let mut rdr = csv::Reader::from_reader(std::fs::File::open(path)?); + let mut name = "N/A".to_string(); + let mut query_x = "".to_string(); + let mut query_y = "".to_string(); + if rdr.has_headers() { + let record = rdr.headers()?; + name = record[0].to_string(); + query_x = record[1].to_string(); + query_y = record[2].to_string(); + } + for result in rdr.records() { + if let Ok(record) = result { + values.push(Value { x: record[1].parse::()?, y: record[2].parse::()? }); + } + } + + Ok(( + name, + query_x, + query_y, + values, + )) +} + pub fn human_size(size: u64) -> String { let mut buf: f64 = size as f64; let mut prefix: usize = 0; diff --git a/src/app/worker.rs b/src/app/worker.rs index d3d00a8..45a07a7 100644 --- a/src/app/worker.rs +++ b/src/app/worker.rs @@ -88,6 +88,7 @@ impl BackgroundWorker for NativeBackgroundWorker { .expect("Sources RwLock poisoned"); *last_update = Utc::now(); let state2 = state.clone(); + let source_name = sources[j].name.clone(); let url = sources[j].url.clone(); std::thread::spawn(move || { // TODO this can overspawn if a request takes longer than the refresh interval! @@ -100,12 +101,12 @@ impl BackgroundWorker for NativeBackgroundWorker { match metric.extract(&res) { Ok(v) => { metric.data.write().expect("Data RwLock poisoned").push(v); - if let Err(e) = store.put_value(metric.id, v) { + if let Err(e) = store.put_value(metric.id, &v) { warn!(target:"background-worker", "Could not put sample for source #{} in db: {:?}", s_id, e); } } Err(e) => { - warn!(target:"background-worker", "[{}] Could not extract value from result: {:?}", metric.name, e); // TODO: more info! + warn!(target:"background-worker", "[{}|{}] Could not extract value from result: {:?}", source_name, metric.name, e); // TODO: more info! } } }