From 5ded746434936c22e1380bd479615969fa358943 Mon Sep 17 00:00:00 2001 From: alemi Date: Mon, 6 Jun 2022 04:41:34 +0200 Subject: [PATCH] feat: reworked storage to use sqlite --- src/app/data/mod.rs | 75 +++++++++++++++ src/app/data/store.rs | 207 ++++++++++++++++++++++++++++++++++++++++++ src/app/datasource.rs | 202 ----------------------------------------- src/app/mod.rs | 125 +++++++++---------------- 4 files changed, 326 insertions(+), 283 deletions(-) create mode 100644 src/app/data/mod.rs create mode 100644 src/app/data/store.rs delete mode 100644 src/app/datasource.rs diff --git a/src/app/data/mod.rs b/src/app/data/mod.rs new file mode 100644 index 0000000..54b8923 --- /dev/null +++ b/src/app/data/mod.rs @@ -0,0 +1,75 @@ +pub mod source; +pub mod store; + +use std::sync::{Arc, Mutex}; +use std::num::ParseFloatError; +use chrono::{DateTime, Utc}; +use eframe::egui::plot::{Values, Value}; + +#[derive(Debug)] +pub enum FetchError { + ReqwestError(reqwest::Error), + JqError(jq_rs::Error), + RusqliteError(rusqlite::Error), + ParseFloatError(ParseFloatError), +} + +impl From:: for FetchError { + fn from(e: reqwest::Error) -> Self { FetchError::ReqwestError(e) } +} +impl From:: for FetchError { + fn from(e: jq_rs::Error) -> Self { FetchError::JqError(e) } +} +impl From:: for FetchError { + fn from(e: ParseFloatError) -> Self { FetchError::ParseFloatError(e) } +} +impl From:: for FetchError { + fn from(e: rusqlite::Error) -> Self { FetchError::RusqliteError(e) } +} + + + +pub struct Panel { + pub(crate) id: i32, + pub name: String, + pub view_scroll: bool, + pub view_size: i32, + pub(crate) width: i32, + pub(crate) height: i32, + pub(crate) sources: Mutex>, +} + +impl Panel { +} + +pub struct Source { + pub(crate) id: i32, + pub name: String, + pub url: String, + pub interval: i32, + pub(crate) last_fetch: DateTime, + pub query_x: String, + // pub(crate) compiled_query_x: Arc>, + pub query_y: String, + // pub(crate) compiled_query_y: Arc>, + pub(crate) panel_id: i32, + pub(crate) data: Mutex>, +} + +impl Source { + pub fn valid(&self) -> bool { + return (Utc::now() - self.last_fetch).num_seconds() < self.interval as i64; + } + + pub fn values(&self) -> Values { + Values::from_values(self.data.lock().unwrap().clone()) + } + + pub async fn fetch(&self) -> Result { + let res = reqwest::get(&self.url).await?; + let body = res.text().await?; + let x = jq_rs::compile(&self.query_x)?.run(&body)?.parse::()?; + let y = jq_rs::compile(&self.query_y)?.run(&body)?.parse::()?; + return Ok( Value { x, y } ); + } +} \ No newline at end of file diff --git a/src/app/data/store.rs b/src/app/data/store.rs new file mode 100644 index 0000000..78a6fb7 --- /dev/null +++ b/src/app/data/store.rs @@ -0,0 +1,207 @@ +use std::sync::{Arc, Mutex}; +use chrono::{DateTime, TimeZone, NaiveDateTime, Utc}; +use rusqlite::{Connection, params}; +use eframe::egui::plot::Value; +use crate::app::data::{Panel, Source}; + +use super::FetchError; + +pub trait DataStorage { + fn add_panel(&self, name:&str); +} + +pub struct SQLiteDataStore { + conn: Connection, + pub(crate) panels: Mutex>, +} + +impl SQLiteDataStore { + pub fn new(path: std::path::PathBuf) -> Result { + let conn = Connection::open(path)?; + + conn.execute( + "CREATE TABLE IF NOT EXISTS panels ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + view_scroll BOOL, + view_size INT, + width INT, + height INT + );", + [], + )?; + + conn.execute( + "CREATE TABLE IF NOT EXISTS sources ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + url TEXT, + interval INT, + query_x TEXT, + query_y TEXT, + panel_id INT + );", + [], + )?; + + conn.execute( + "CREATE TABLE IF NOT EXISTS points ( + id INTEGER PRIMARY KEY, + panel_id INT, + source_id INT, + x FLOAT, + y FLOAT + );", + [], + )?; + + let mut store = SQLiteDataStore { + conn, + panels: Mutex::new(Vec::new()), + }; + + store.load_panels()?; + + return Ok(store); + } + + fn load_values(&self, panel_id:i32, source_id:i32) -> rusqlite::Result> { + let mut values : Vec = Vec::new(); + let mut statement = self.conn.prepare("SELECT x, y FROM points WHERE panel_id = ? AND source_id = ?")?; + let values_iter = statement.query_map(params![panel_id, source_id], |row| { + Ok(Value{ x: row.get(0)?, y: row.get(1)? }) + })?; + + for value in values_iter { + if let Ok(v) = value { + values.push(v); + } + } + + Ok(values) + } + + fn put_value(&self, panel_id:i32, source_id:i32, v:Value) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO points(panel_id, source_id, x, y) VALUES (?, ?, ?, ?)", + params![panel_id, source_id, v.x, v.y], + ) + } + + fn load_sources(&self, panel_id:i32) -> rusqlite::Result> { + let mut sources : Vec = Vec::new(); + let mut statement = self.conn.prepare("SELECT * FROM sources WHERE panel_id = ?")?; + let sources_iter = statement.query_map(params![panel_id], |row| { + Ok(Source{ + id: row.get(0)?, + name: row.get(1)?, + url: row.get(2)?, + interval: row.get(3)?, + last_fetch: Utc.ymd(1970, 1, 1).and_hms(0, 0, 0), + query_x: row.get(4)?, + // compiled_query_x: Arc::new(Mutex::new(jq_rs::compile(row.get::(4)?.as_str()).unwrap())), + query_y: row.get(5)?, + // compiled_query_y: Arc::new(Mutex::new(jq_rs::compile(row.get::(5)?.as_str()).unwrap())), + panel_id: row.get(6)?, + data: Mutex::new(Vec::new()), + }) + })?; + + for source in sources_iter { + if let Ok(mut s) = source { + s.data = Mutex::new(self.load_values(panel_id, s.id)?); + sources.push(s); + } + } + + Ok(sources) + } + + fn put_source(&self, panel_id:i32, s:Source) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO sources(id, name, url, interval, query_x, query_y, panel_id) VALUES (?, ?, ?, ?, ?, ?, ?)", + params![s.id, s.name, s.url, s.interval, s.query_x, s.query_y, panel_id], + ) + } + + fn load_panels(&self) -> rusqlite::Result> { + let mut panels : Vec = Vec::new(); + let mut statement = self.conn.prepare("SELECT * FROM panels")?; + let panels_iter = statement.query_map([], |row| { + Ok(Panel{ + id: row.get(0)?, + name: row.get(1)?, + view_scroll: row.get(2)?, + view_size: row.get(3)?, + width: row.get(4)?, + height: row.get(5)?, + sources: Mutex::new(Vec::new()), + }) + })?; + + for panel in panels_iter { + if let Ok(mut p) = panel { + p.sources = Mutex::new(self.load_sources(p.id)?); + panels.push(p); + } + } + + Ok(panels) + } + + fn put_panel(&self, name:&str, view_scroll:bool, view_size:i32, width:i32, height:i32) -> rusqlite::Result { + self.conn.execute( + "INSERT INTO panels (name, view_scroll, view_size, width, height) VALUES (?, ?, ?, ?, ?)", + params![name, view_scroll, view_size, width, height] + ) + } + + // jank! TODO make it not jank! + fn new_panel(&self, name:&str) -> rusqlite::Result { + self.put_panel(name, true, 100, 400, 280)?; + let mut statement = self.conn.prepare("SELECT * FROM panels WHERE name = ?")?; + for panel in statement.query_map(params![name], |row| { + Ok(Panel{ + id: row.get(0)?, + name: row.get(1)?, + view_scroll: row.get(2)?, + view_size: row.get(3)?, + width: row.get(4)?, + height: row.get(5)?, + sources: Mutex::new(Vec::new()), + }) + })? { + if let Ok(p) = panel { + return Ok(p); + } else { + println!("WTF"); + } + } + + Err(rusqlite::Error::QueryReturnedNoRows) + } + + pub async fn fetch_all(&self) -> Result<(), FetchError> { + let panels = &*self.panels.lock().unwrap(); + for i in 0..panels.len() { + let sources = &*panels[i].sources.lock().unwrap(); + for j in 0..sources.len() { + if !sources[j].valid() { + let v = sources[j].fetch().await?; + self.put_value(panels[i].id, sources[j].id, v)?; + sources[j].data.lock().unwrap().push(v); + } + } + } + + Ok(()) + } + +} + +impl DataStorage for SQLiteDataStore { + fn add_panel(&self, name:&str) { + let panel = self.new_panel(name).unwrap(); + self.panels.lock().unwrap().push(panel); + } +} \ No newline at end of file diff --git a/src/app/datasource.rs b/src/app/datasource.rs deleted file mode 100644 index 4b32411..0000000 --- a/src/app/datasource.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::sync::{Arc, Mutex}; -use rand::Rng; -use std::io::{Write, Read}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize, de::{DeserializeOwned}}; -use eframe::egui::{plot::Value, Context}; - -pub fn native_save(name: &str, data:String) -> std::io::Result<()> { - let mut file = std::fs::File::create(name)?; - file.write_all(data.as_bytes())?; - return Ok(()); -} - -pub struct DataSource { - data : Arc>>, -} - -#[derive(Serialize, Deserialize)] -struct SerializableValue { - x : f64, - y : f64, -} - -impl DataSource { - pub fn new() -> Self { - Self{ data: Arc::new(Mutex::new(Vec::new())) } - } - - pub fn view(&self) -> Vec { // TODO handle errors - return self.data.lock().unwrap().clone(); - } - - pub fn serialize(&self) -> String { - let mut out : Vec = Vec::new(); - for value in self.view() { - out.push(SerializableValue { x: value.x, y: value.y }); - } - return serde_json::to_string(&out).unwrap(); - } -} - -pub trait PlotValue { - fn as_value(&self) -> Value; -} - -pub trait Data { - fn load_remote(&mut self, url:&str, ctx:Context); - fn load_local(&mut self, file:&str, ctx:Context); - - fn read(&mut self, file:&str, storage:Arc>>, ctx:Context) -> std::io::Result<()> { - let mut file = std::fs::File::open(file)?; - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - let data : Vec = serde_json::from_str(contents.as_str())?; - for v in data { - storage.lock().unwrap().push(Value { x: v.x, y: v.y }); - } - ctx.request_repaint(); - Ok(()) - } - - fn fetch(&mut self, base:&str, endpoint:&str, storage:Arc>>, ctx:Context) - where T : DeserializeOwned + PlotValue { - let request = ehttp::Request::get(format!("{}/{}", base, endpoint)); - ehttp::fetch(request, move |result: ehttp::Result| { - let data : T = serde_json::from_slice(result.unwrap().bytes.as_slice()).unwrap(); - storage.lock().unwrap().push(data.as_value()); - ctx.request_repaint(); - }); - - } -} - -pub struct TpsData { - pub ds: DataSource, - load_interval : i64, - last_load : DateTime, -} - -#[derive(Serialize, Deserialize)] -struct TpsResponseData { - tps: f64 -} - -impl PlotValue for TpsResponseData { - fn as_value(&self) -> Value { - Value { x: Utc::now().timestamp() as f64, y: self.tps } - } -} - -impl TpsData { - pub fn new(load_interval:i64) -> Self { - Self { ds: DataSource::new() , last_load: Utc::now(), load_interval } - } -} - -impl Data for TpsData{ - fn load_remote(&mut self, url:&str, ctx:Context) { - if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; } - self.last_load = Utc::now(); - self.fetch::(url, "tps", self.ds.data.clone(), ctx); - } - - fn load_local(&mut self, file:&str, ctx:Context) { - self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file)); - } -} - -pub struct ChatData { - pub ds : DataSource, - load_interval : i64, - last_load : DateTime, -} - -#[derive(Serialize, Deserialize)] -struct ChatResponseData { - volume: f64 -} - -impl PlotValue for ChatResponseData { - fn as_value(&self) -> Value { - Value { x:Utc::now().timestamp() as f64, y: self.volume } - } -} - -impl ChatData { - pub fn new(load_interval:i64) -> Self { - Self { ds: DataSource::new() , last_load: Utc::now(), load_interval } - } -} - -impl Data for ChatData{ - fn load_remote(&mut self, url:&str, ctx:Context) { - if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; } - self.last_load = Utc::now(); - self.fetch::(url, "chat_activity", self.ds.data.clone(), ctx); - } - - fn load_local(&mut self, file:&str, ctx:Context) { - self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file)); - } -} - -pub struct PlayerCountData { - pub ds : DataSource, - load_interval : i64, - last_load : DateTime, -} - -#[derive(Serialize, Deserialize)] -struct PlayerCountResponseData { - count: i32 -} - -impl PlotValue for PlayerCountResponseData { - fn as_value(&self) -> Value { - Value { x:Utc::now().timestamp() as f64, y: self.count as f64 } - } -} - -impl PlayerCountData { - pub fn new(load_interval:i64) -> Self { - Self { ds: DataSource::new() , last_load: Utc::now(), load_interval } - } -} - -impl Data for PlayerCountData{ - fn load_remote(&mut self, url:&str, ctx:Context) { - if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; } - self.last_load = Utc::now(); - self.fetch::(url, "player_count", self.ds.data.clone(), ctx); - } - - fn load_local(&mut self, file:&str, ctx:Context) { - self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file)); - } -} - -pub struct RandomData { - pub ds : DataSource, - load_interval : i64, - last_load : DateTime, - rng: rand::rngs::ThreadRng, -} - -impl RandomData { - #[allow(dead_code)] - pub fn new(load_interval:i64) -> Self { - Self { ds: DataSource::new() , last_load: Utc::now(), load_interval, rng : rand::thread_rng() } - } -} - -impl Data for RandomData{ - fn load_remote(&mut self, _url:&str, ctx:Context) { - if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; } - self.last_load = Utc::now(); - self.ds.data.lock().unwrap().push(Value {x:Utc::now().timestamp() as f64, y:self.rng.gen()}); - ctx.request_repaint(); - } - - fn load_local(&mut self, _file:&str, _ctx:Context) {} -} \ No newline at end of file diff --git a/src/app/mod.rs b/src/app/mod.rs index 0330c4c..469e9ea 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1,12 +1,17 @@ -mod datasource; +pub mod data; +use std::sync::{Arc, Mutex}; use chrono::{DateTime, NaiveDateTime, Utc}; -use datasource::{ChatData, PlayerCountData, TpsData, Data, native_save}; +use data::source::{ChatData, PlayerCountData, TpsData, Data, native_save}; use eframe::egui; use eframe::egui::plot::{Line, Plot, Values}; +use crate::app::data::store::DataStorage; + +use self::data::store::SQLiteDataStore; pub struct App { - servers : Vec, + // data : SQLiteDataStore, + data : Arc, } struct ServerOptions { @@ -19,83 +24,20 @@ struct ServerOptions { } impl App { - pub fn new(_cc: &eframe::CreationContext) -> Self { - let mut servers = Vec::new(); - servers.push(ServerOptions::new("9b9t", "https://alemi.dev/mcbots/9b")); - servers.push(ServerOptions::new("const", "https://alemi.dev/mcbots/const")); - servers.push(ServerOptions::new("of", "https://alemi.dev/mcbots/of")); - Self { servers } - } -} - -impl ServerOptions { - fn new(title:&str, url:&str) -> Self { - Self { - title: title.to_string(), - url: url.to_string(), - player_count: PlayerCountData::new(60), - tps: TpsData::new(15), - chat: ChatData::new(30), - sync_time: false, - } - } - - fn display(&mut self, ui:&mut eframe::egui::Ui) { - ui.vertical(|ui| { - ui.horizontal(|ui| { - ui.heading(self.title.as_str()); - ui.checkbox(&mut self.sync_time, "Lock X to now"); - }); - let mut p = Plot::new(format!("plot-{}", self.title)).x_axis_formatter(|x, _range| { - format!( - "{}", - DateTime::::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc) - .format("%Y/%m/%d %H:%M:%S") - ) - }).center_x_axis(false).height(260.0); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do... - - if self.sync_time { - p = p.include_x(Utc::now().timestamp() as f64); - } - - p.show(ui, |plot_ui| { - plot_ui.line( - Line::new(Values::from_values(self.player_count.ds.view())).name("Player Count"), - ); - plot_ui.line(Line::new(Values::from_values(self.tps.ds.view())).name("TPS over 15s")); - plot_ui.line( - Line::new(Values::from_values(self.chat.ds.view())) - .name("Chat messages per minute"), - ); - }); - }); + // pub fn new(_cc: &eframe::CreationContext, data: SQLiteDataStore) -> Self { + pub fn new(_cc: &eframe::CreationContext, data: Arc) -> Self { + Self { data } } } impl eframe::App for App { fn update(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { - for server in &mut self.servers { - server.tps.load_remote(server.url.as_str(), ctx.clone()); - server.player_count.load_remote(server.url.as_str(), ctx.clone()); - server.chat.load_remote(server.url.as_str(), ctx.clone()); - } egui::TopBottomPanel::top("??? wtf").show(ctx, |ui| { ui.horizontal(|ui| { egui::widgets::global_dark_light_mode_switch(ui); - ui.heading("nnbot dashboard"); - if ui.button("save").clicked() { - for server in &self.servers { - native_save(format!("{}-tps.json", server.title).as_str(), server.tps.ds.serialize()).unwrap(); - native_save(format!("{}-chat.json", server.title).as_str(), server.chat.ds.serialize()).unwrap(); - native_save(format!("{}-players.json", server.title).as_str(), server.player_count.ds.serialize()).unwrap(); - } - } - if ui.button("load").clicked() { - for server in &mut self.servers { - server.tps.load_local(format!("{}-tps.json", server.title).as_str(), ctx.clone()); - server.chat.load_local(format!("{}-chat.json", server.title).as_str(), ctx.clone()); - server.player_count.load_local(format!("{}-players.json", server.title).as_str(), ctx.clone()); - } + ui.heading("dashboard"); + if ui.button("test add").clicked() { + self.data.add_panel("test panel"); } ui.with_layout(egui::Layout::top_down(egui::Align::RIGHT), |ui| { if ui.button("x").clicked() { @@ -105,15 +47,36 @@ impl eframe::App for App { }); }); egui::CentralPanel::default().show(ctx, |ui| { - ui.group(|v_ui| { - self.servers[0].display(v_ui); - }); - ui.group(|v_ui| { - self.servers[1].display(v_ui); - }); - ui.group(|v_ui| { - self.servers[2].display(v_ui); - }); + let panels = &*self.data.panels.lock().unwrap(); + for i in 0..panels.len() { + // for panel in self.data.view() { + ui.group(|ui| { + ui.vertical(|ui| { + ui.horizontal(|ui| { + ui.heading(panels[i].name.as_str()); + // ui.checkbox(&mut panel.view_scroll, "autoscroll"); + }); + let mut p = Plot::new(format!("plot-{}", panels[i].name)).x_axis_formatter(|x, _range| { + format!( + "{}", + DateTime::::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc) + .format("%Y/%m/%d %H:%M:%S") + ) + }).center_x_axis(false).height(panels[i].height as f32); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do... + + if panels[i].view_scroll { + p = p.include_x(Utc::now().timestamp() as f64); + } + + p.show(ui, |plot_ui| { + let sources = &*panels[i].sources.lock().unwrap(); + for j in 0..sources.len() { + plot_ui.line(Line::new(sources[j].values()).name(sources[j].name.as_str())); + } + }); + }); + }); + } }); ctx.request_repaint(); // TODO super jank way to sorta keep drawing }