feat: reworked storage to use sqlite

src/app/data/mod.rs

@@ -0,0 +1,75 @@
+pub mod source;
+pub mod store;
+
+use std::sync::{Arc, Mutex};
+use std::num::ParseFloatError;
+use chrono::{DateTime, Utc};
+use eframe::egui::plot::{Values, Value};
+
+#[derive(Debug)]
+pub enum FetchError {
+	ReqwestError(reqwest::Error),
+	JqError(jq_rs::Error),
+	RusqliteError(rusqlite::Error),
+	ParseFloatError(ParseFloatError),
+}
+
+impl From::<reqwest::Error> for FetchError {
+	fn from(e: reqwest::Error) -> Self { FetchError::ReqwestError(e) }
+}
+impl From::<jq_rs::Error> for FetchError {
+	fn from(e: jq_rs::Error) -> Self { FetchError::JqError(e) }
+}
+impl From::<ParseFloatError> for FetchError {
+	fn from(e: ParseFloatError) -> Self { FetchError::ParseFloatError(e) }
+}
+impl From::<rusqlite::Error> for FetchError {
+	fn from(e: rusqlite::Error) -> Self { FetchError::RusqliteError(e) }
+}
+
+
+
+pub struct Panel {
+	pub(crate) id: i32,
+	pub name: String,
+	pub view_scroll: bool,
+	pub view_size: i32,
+	pub(crate) width: i32,
+	pub(crate) height: i32,
+	pub(crate) sources: Mutex<Vec<Source>>,
+}
+
+impl Panel {
+}
+
+pub struct Source {
+	pub(crate) id: i32,
+	pub name: String,
+	pub url: String,
+	pub interval: i32,
+	pub(crate) last_fetch: DateTime<Utc>,
+	pub query_x: String,
+	// pub(crate) compiled_query_x: Arc<Mutex<jq_rs::JqProgram>>,
+	pub query_y: String,
+	// pub(crate) compiled_query_y: Arc<Mutex<jq_rs::JqProgram>>,
+	pub(crate) panel_id: i32,
+	pub(crate) data: Mutex<Vec<Value>>,
+}
+
+impl Source {
+	pub fn valid(&self) -> bool {
+		return (Utc::now() - self.last_fetch).num_seconds() < self.interval as i64;
+	}
+
+	pub fn values(&self) -> Values {
+		Values::from_values(self.data.lock().unwrap().clone())
+	}
+
+	pub async fn fetch(&self) -> Result<Value, FetchError> {
+		let res = reqwest::get(&self.url).await?;
+		let body = res.text().await?;
+		let x = jq_rs::compile(&self.query_x)?.run(&body)?.parse::<f64>()?;
+		let y = jq_rs::compile(&self.query_y)?.run(&body)?.parse::<f64>()?;
+		return Ok( Value { x, y } );
+	}
+}

src/app/data/store.rs

@@ -0,0 +1,207 @@
+use std::sync::{Arc, Mutex};
+use chrono::{DateTime, TimeZone, NaiveDateTime, Utc};
+use rusqlite::{Connection, params};
+use eframe::egui::plot::Value;
+use crate::app::data::{Panel, Source};
+
+use super::FetchError;
+
+pub trait DataStorage {
+	fn add_panel(&self, name:&str);
+}
+
+pub struct SQLiteDataStore {
+	conn: Connection,
+	pub(crate) panels: Mutex<Vec<Panel>>,
+}
+
+impl SQLiteDataStore {
+	pub fn new(path: std::path::PathBuf) -> Result<Self, rusqlite::Error> {
+		let conn = Connection::open(path)?;
+		
+		conn.execute(
+			"CREATE TABLE IF NOT EXISTS panels (
+				id INTEGER PRIMARY KEY,
+				name TEXT UNIQUE,
+				view_scroll BOOL,
+				view_size INT,
+				width INT,
+				height INT
+			);",
+			[],
+		)?;
+
+		conn.execute(
+			"CREATE TABLE IF NOT EXISTS sources (
+				id INTEGER PRIMARY KEY,
+				name TEXT UNIQUE,
+				url TEXT,
+				interval INT,
+				query_x TEXT,
+				query_y TEXT,
+				panel_id INT
+			);",
+			[],
+		)?;
+
+		conn.execute(
+			"CREATE TABLE IF NOT EXISTS points (
+				id INTEGER PRIMARY KEY,
+				panel_id INT,
+				source_id INT,
+				x FLOAT,
+				y FLOAT
+			);",
+			[],
+		)?;
+
+		let mut store = SQLiteDataStore { 
+			conn,
+			panels: Mutex::new(Vec::new()),
+		};
+
+		store.load_panels()?;
+
+		return Ok(store);
+	}
+
+	fn load_values(&self, panel_id:i32, source_id:i32) -> rusqlite::Result<Vec<Value>> {
+		let mut values : Vec<Value> = Vec::new();
+		let mut statement = self.conn.prepare("SELECT x, y FROM points WHERE panel_id = ? AND source_id = ?")?;
+		let values_iter = statement.query_map(params![panel_id, source_id], |row| {
+			Ok(Value{ x: row.get(0)?, y: row.get(1)? })
+		})?;
+
+		for value in values_iter {
+			if let Ok(v) = value {
+				values.push(v);
+			}
+		}
+
+		Ok(values)
+	}
+
+	fn put_value(&self, panel_id:i32, source_id:i32, v:Value) -> rusqlite::Result<usize> {
+		self.conn.execute(
+			"INSERT INTO points(panel_id, source_id, x, y) VALUES (?, ?, ?, ?)",
+			params![panel_id, source_id, v.x, v.y],
+		)
+	}
+
+	fn load_sources(&self, panel_id:i32) -> rusqlite::Result<Vec<Source>> {
+		let mut sources : Vec<Source> = Vec::new();
+		let mut statement = self.conn.prepare("SELECT * FROM sources WHERE panel_id = ?")?;
+		let sources_iter = statement.query_map(params![panel_id], |row| {
+			Ok(Source{
+				id: row.get(0)?,
+				name: row.get(1)?,
+				url: row.get(2)?,
+				interval: row.get(3)?,
+				last_fetch: Utc.ymd(1970, 1, 1).and_hms(0, 0, 0),
+				query_x: row.get(4)?,
+				// compiled_query_x: Arc::new(Mutex::new(jq_rs::compile(row.get::<usize, String>(4)?.as_str()).unwrap())),
+				query_y: row.get(5)?,
+				// compiled_query_y: Arc::new(Mutex::new(jq_rs::compile(row.get::<usize, String>(5)?.as_str()).unwrap())),
+				panel_id: row.get(6)?,
+				data: Mutex::new(Vec::new()),
+			})
+		})?;
+
+		for source in sources_iter {
+			if let Ok(mut s) = source {
+				s.data = Mutex::new(self.load_values(panel_id, s.id)?);
+				sources.push(s);
+			}
+		}
+
+		Ok(sources)
+	}
+
+	fn put_source(&self, panel_id:i32, s:Source) -> rusqlite::Result<usize> {
+		self.conn.execute(
+			"INSERT INTO sources(id, name, url, interval, query_x, query_y, panel_id) VALUES (?, ?, ?, ?, ?, ?, ?)",
+			params![s.id, s.name, s.url, s.interval, s.query_x, s.query_y, panel_id],
+		)
+	}
+
+	fn load_panels(&self) -> rusqlite::Result<Vec<Panel>> {
+		let mut panels : Vec<Panel> = Vec::new();
+		let mut statement = self.conn.prepare("SELECT * FROM panels")?;
+		let panels_iter = statement.query_map([], |row| {
+			Ok(Panel{
+				id: row.get(0)?,
+				name: row.get(1)?,
+				view_scroll: row.get(2)?,
+				view_size: row.get(3)?,
+				width: row.get(4)?,
+				height: row.get(5)?,
+				sources: Mutex::new(Vec::new()),
+			})
+		})?;
+
+		for panel in panels_iter {
+			if let Ok(mut p) = panel {
+				p.sources = Mutex::new(self.load_sources(p.id)?);
+				panels.push(p);
+			}
+		}
+
+		Ok(panels)
+	}
+
+	fn put_panel(&self, name:&str, view_scroll:bool, view_size:i32, width:i32, height:i32) -> rusqlite::Result<usize> {
+		self.conn.execute(
+			"INSERT INTO panels (name, view_scroll, view_size, width, height) VALUES (?, ?, ?, ?, ?)",
+			params![name, view_scroll, view_size, width, height]
+		)
+	}
+
+	// jank! TODO make it not jank!
+	fn new_panel(&self, name:&str) -> rusqlite::Result<Panel> {
+		self.put_panel(name, true, 100, 400, 280)?;
+		let mut statement = self.conn.prepare("SELECT * FROM panels WHERE name = ?")?;
+		for panel in statement.query_map(params![name], |row| {
+			Ok(Panel{
+				id: row.get(0)?,
+				name: row.get(1)?,
+				view_scroll: row.get(2)?,
+				view_size: row.get(3)?,
+				width: row.get(4)?,
+				height: row.get(5)?,
+				sources: Mutex::new(Vec::new()),
+			})
+		})? {
+			if let Ok(p) = panel {
+				return Ok(p);
+			} else {
+				println!("WTF");
+			}
+		}
+
+		Err(rusqlite::Error::QueryReturnedNoRows)
+	}
+
+	pub async fn fetch_all(&self) -> Result<(), FetchError> {
+		let panels = &*self.panels.lock().unwrap();
+		for i in 0..panels.len() {
+			let sources = &*panels[i].sources.lock().unwrap();
+			for j in 0..sources.len() {
+				if !sources[j].valid() {
+					let v = sources[j].fetch().await?;
+					self.put_value(panels[i].id, sources[j].id, v)?;
+					sources[j].data.lock().unwrap().push(v);
+				}
+			}
+		}
+
+		Ok(())
+	}
+
+}
+
+impl DataStorage for SQLiteDataStore {
+	fn add_panel(&self, name:&str) {
+		let panel = self.new_panel(name).unwrap();
+		self.panels.lock().unwrap().push(panel);
+	}
+}

src/app/datasource.rs

@@ -1,202 +0,0 @@
-use std::sync::{Arc, Mutex};
-use rand::Rng;
-use std::io::{Write, Read};
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize, de::{DeserializeOwned}};
-use eframe::egui::{plot::Value, Context};
-
-pub fn native_save(name: &str, data:String) -> std::io::Result<()> {
-	let mut file = std::fs::File::create(name)?;
-	file.write_all(data.as_bytes())?;
-	return Ok(());
-}
-
-pub struct DataSource {
-	data : Arc<Mutex<Vec<Value>>>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct SerializableValue {
-	x : f64,
-	y : f64,
-}
-
-impl DataSource {
-	pub fn new() -> Self {
-		Self{ data: Arc::new(Mutex::new(Vec::new())) }
-	}
-
-	pub fn view(&self) -> Vec<Value> { // TODO handle errors
-		return self.data.lock().unwrap().clone();
-	}
-
-	pub fn serialize(&self) -> String {
-		let mut out : Vec<SerializableValue> = Vec::new();
-		for value in self.view() {
-			out.push(SerializableValue { x: value.x, y: value.y });
-		}
-		return serde_json::to_string(&out).unwrap();
-	}
-}
-
-pub trait PlotValue {
-	fn as_value(&self) -> Value;
-}
-
-pub trait Data {
-	fn load_remote(&mut self, url:&str, ctx:Context);
-	fn load_local(&mut self, file:&str, ctx:Context);
-
-	fn read(&mut self, file:&str, storage:Arc<Mutex<Vec<Value>>>, ctx:Context) -> std::io::Result<()> {
-		let mut file = std::fs::File::open(file)?;
-		let mut contents = String::new();
-		file.read_to_string(&mut contents)?;
-		let data : Vec<SerializableValue> = serde_json::from_str(contents.as_str())?;
-		for v in data {
-			storage.lock().unwrap().push(Value { x: v.x, y: v.y });
-		}
-		ctx.request_repaint();
-		Ok(())
-	}
-
-	fn fetch<T>(&mut self, base:&str, endpoint:&str, storage:Arc<Mutex<Vec<Value>>>, ctx:Context) 
-	where T : DeserializeOwned + PlotValue {
-		let request = ehttp::Request::get(format!("{}/{}", base, endpoint));
-		ehttp::fetch(request, move |result: ehttp::Result<ehttp::Response>| {
-			let data : T = serde_json::from_slice(result.unwrap().bytes.as_slice()).unwrap();
-			storage.lock().unwrap().push(data.as_value()); 
-			ctx.request_repaint();
-		});
-
-	}
-}
-
-pub struct TpsData {
-	pub ds: DataSource,
-	load_interval : i64,
-	last_load : DateTime<Utc>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct TpsResponseData {
-	tps: f64
-}
-
-impl PlotValue for TpsResponseData {
-	fn as_value(&self) -> Value {
-		Value { x: Utc::now().timestamp() as f64, y: self.tps }
-	}
-}
-
-impl TpsData {
-	pub fn new(load_interval:i64) -> Self {
-		Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
-	}
-}
-
-impl Data for TpsData{
-	fn load_remote(&mut self, url:&str, ctx:Context) {
-		if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
-		self.last_load = Utc::now();
-		self.fetch::<TpsResponseData>(url, "tps", self.ds.data.clone(), ctx);
-	}
-
-	fn load_local(&mut self, file:&str, ctx:Context) {
-		self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
-	}
-}
-
-pub struct ChatData {
-	pub ds : DataSource,
-	load_interval : i64,
-	last_load : DateTime<Utc>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct ChatResponseData {
-	volume: f64
-}
-
-impl PlotValue for ChatResponseData {
-	fn as_value(&self) -> Value {
-		Value { x:Utc::now().timestamp() as f64, y: self.volume }
-	}
-}
-
-impl ChatData {
-	pub fn new(load_interval:i64) -> Self {
-		Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
-	}
-}
-
-impl Data for ChatData{
-	fn load_remote(&mut self, url:&str, ctx:Context) {
-		if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
-		self.last_load = Utc::now();
-		self.fetch::<ChatResponseData>(url, "chat_activity", self.ds.data.clone(), ctx);
-	}
-
-	fn load_local(&mut self, file:&str, ctx:Context) {
-		self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
-	}
-}
-
-pub struct PlayerCountData {
-	pub ds : DataSource,
-	load_interval : i64,
-	last_load : DateTime<Utc>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct PlayerCountResponseData {
-	count: i32
-}
-
-impl PlotValue for PlayerCountResponseData {
-	fn as_value(&self) -> Value {
-		Value { x:Utc::now().timestamp() as f64, y: self.count as f64 }
-	}
-}
-
-impl PlayerCountData {
-	pub fn new(load_interval:i64) -> Self {
-		Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
-	}
-}
-
-impl Data for PlayerCountData{
-	fn load_remote(&mut self, url:&str, ctx:Context) {
-		if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
-		self.last_load = Utc::now();
-		self.fetch::<PlayerCountResponseData>(url, "player_count", self.ds.data.clone(), ctx);
-	}
-
-	fn load_local(&mut self, file:&str, ctx:Context) {
-		self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
-	}
-}
-
-pub struct RandomData {
-	pub ds : DataSource,
-	load_interval : i64,
-	last_load : DateTime<Utc>,
-	rng: rand::rngs::ThreadRng,
-}
-
-impl RandomData {
-	#[allow(dead_code)]
-	pub fn new(load_interval:i64) -> Self {
-		Self { ds: DataSource::new() , last_load: Utc::now(), load_interval, rng : rand::thread_rng() }
-	}
-}
-
-impl Data for RandomData{
-	fn load_remote(&mut self, _url:&str, ctx:Context) {
-		if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
-		self.last_load = Utc::now();
-		self.ds.data.lock().unwrap().push(Value {x:Utc::now().timestamp() as f64, y:self.rng.gen()}); 
-		ctx.request_repaint();
-	}
-
-	fn load_local(&mut self, _file:&str, _ctx:Context) {}
-}

src/app/mod.rs

@@ -1,12 +1,17 @@
-mod datasource;
+pub mod data;
 
+use std::sync::{Arc, Mutex};
 use chrono::{DateTime, NaiveDateTime, Utc};
-use datasource::{ChatData, PlayerCountData, TpsData, Data,  native_save};
+use data::source::{ChatData, PlayerCountData, TpsData, Data, native_save};
 use eframe::egui;
 use eframe::egui::plot::{Line, Plot, Values};
+use crate::app::data::store::DataStorage;
+
+use self::data::store::SQLiteDataStore;
 
 pub struct App {
-	servers : Vec<ServerOptions>,
+	// data : SQLiteDataStore,
+	data : Arc<SQLiteDataStore>,
 }
 
 struct ServerOptions {
@@ -19,83 +24,20 @@ struct ServerOptions {
 }
 
 impl App {
-	pub fn new(_cc: &eframe::CreationContext) -> Self {
-		let mut servers = Vec::new();
-		servers.push(ServerOptions::new("9b9t", "https://alemi.dev/mcbots/9b"));
-		servers.push(ServerOptions::new("const", "https://alemi.dev/mcbots/const"));
-		servers.push(ServerOptions::new("of", "https://alemi.dev/mcbots/of"));
-		Self { servers }
-	}
-}
-
-impl ServerOptions {
-	fn new(title:&str, url:&str) -> Self {
-		Self {
-			title: title.to_string(),
-			url: url.to_string(),
-			player_count: PlayerCountData::new(60),
-			tps: TpsData::new(15),
-			chat: ChatData::new(30),
-			sync_time: false,
-		}
-	}
-
-	fn display(&mut self, ui:&mut eframe::egui::Ui) {
-		ui.vertical(|ui| {
-			ui.horizontal(|ui| {
-				ui.heading(self.title.as_str());
-				ui.checkbox(&mut self.sync_time, "Lock X to now");
-			});
-			let mut p = Plot::new(format!("plot-{}", self.title)).x_axis_formatter(|x, _range| {
-				format!(
-					"{}",
-					DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc)
-						.format("%Y/%m/%d %H:%M:%S")
-				)
-			}).center_x_axis(false).height(260.0); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do...
-
-			if self.sync_time {
-				p = p.include_x(Utc::now().timestamp() as f64);
-			}
-
-			p.show(ui, |plot_ui| {
-				plot_ui.line(
-					Line::new(Values::from_values(self.player_count.ds.view())).name("Player Count"),
-				);
-				plot_ui.line(Line::new(Values::from_values(self.tps.ds.view())).name("TPS over 15s"));
-				plot_ui.line(
-					Line::new(Values::from_values(self.chat.ds.view()))
-						.name("Chat messages per minute"),
-				);
-			});
-		});
+	// pub fn new(_cc: &eframe::CreationContext, data: SQLiteDataStore) -> Self {
+	pub fn new(_cc: &eframe::CreationContext, data: Arc<SQLiteDataStore>) -> Self {
+		Self { data }
 	}
 }
 
 impl eframe::App for App {
 	fn update(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) {
-		for server in &mut self.servers {
-			server.tps.load_remote(server.url.as_str(), ctx.clone());
-			server.player_count.load_remote(server.url.as_str(), ctx.clone());
-			server.chat.load_remote(server.url.as_str(), ctx.clone());
-		}
 		egui::TopBottomPanel::top("??? wtf").show(ctx, |ui| {
 			ui.horizontal(|ui| {
 				egui::widgets::global_dark_light_mode_switch(ui);
-				ui.heading("nnbot dashboard");
-				if ui.button("save").clicked() {
-					for server in &self.servers {
-						native_save(format!("{}-tps.json", server.title).as_str(), server.tps.ds.serialize()).unwrap();
-						native_save(format!("{}-chat.json", server.title).as_str(), server.chat.ds.serialize()).unwrap();
-						native_save(format!("{}-players.json", server.title).as_str(), server.player_count.ds.serialize()).unwrap();
-					}
-				}
-				if ui.button("load").clicked() {
-					for server in &mut self.servers {
-						server.tps.load_local(format!("{}-tps.json", server.title).as_str(), ctx.clone());
-						server.chat.load_local(format!("{}-chat.json", server.title).as_str(), ctx.clone());
-						server.player_count.load_local(format!("{}-players.json", server.title).as_str(), ctx.clone());
-					}
+				ui.heading("dashboard");
+				if ui.button("test add").clicked() {
+					self.data.add_panel("test panel");
 				}
 				ui.with_layout(egui::Layout::top_down(egui::Align::RIGHT), |ui| {
 					if ui.button("x").clicked() {
@@ -105,16 +47,37 @@ impl eframe::App for App {
 			});
 		});
 		egui::CentralPanel::default().show(ctx, |ui| {
-			ui.group(|v_ui| {
-				self.servers[0].display(v_ui);
+			let panels = &*self.data.panels.lock().unwrap();
+			for i in 0..panels.len() {
+			// for panel in self.data.view() {
+				ui.group(|ui| {
+					ui.vertical(|ui| {
+						ui.horizontal(|ui| {
+							ui.heading(panels[i].name.as_str());
+							// ui.checkbox(&mut panel.view_scroll, "autoscroll");
 						});
-			ui.group(|v_ui| {
-				self.servers[1].display(v_ui);
+						let mut p = Plot::new(format!("plot-{}", panels[i].name)).x_axis_formatter(|x, _range| {
+							format!(
+								"{}",
+								DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc)
+									.format("%Y/%m/%d %H:%M:%S")
+							)
+						}).center_x_axis(false).height(panels[i].height as f32); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do...
+
+						if panels[i].view_scroll {
+							p = p.include_x(Utc::now().timestamp() as f64);
+						}
+
+						p.show(ui, |plot_ui| {
+							let sources = &*panels[i].sources.lock().unwrap();
+							for j in 0..sources.len() {
+								plot_ui.line(Line::new(sources[j].values()).name(sources[j].name.as_str()));
+							}
 						});
-			ui.group(|v_ui| {
-				self.servers[2].display(v_ui);
 					});
 				});
+			}
+		});
 		ctx.request_repaint(); // TODO super jank way to sorta keep drawing
 	}
 }