feat: reworked storage to use sqlite

This commit is contained in:
əlemi 2022-06-06 04:41:34 +02:00
parent 4465c672ab
commit 5ded746434
Signed by: alemi
GPG key ID: A4895B84D311642C
4 changed files with 326 additions and 283 deletions

75
src/app/data/mod.rs Normal file
View file

@ -0,0 +1,75 @@
pub mod source;
pub mod store;
use std::sync::{Arc, Mutex};
use std::num::ParseFloatError;
use chrono::{DateTime, Utc};
use eframe::egui::plot::{Values, Value};
#[derive(Debug)]
pub enum FetchError {
ReqwestError(reqwest::Error),
JqError(jq_rs::Error),
RusqliteError(rusqlite::Error),
ParseFloatError(ParseFloatError),
}
impl From::<reqwest::Error> for FetchError {
fn from(e: reqwest::Error) -> Self { FetchError::ReqwestError(e) }
}
impl From::<jq_rs::Error> for FetchError {
fn from(e: jq_rs::Error) -> Self { FetchError::JqError(e) }
}
impl From::<ParseFloatError> for FetchError {
fn from(e: ParseFloatError) -> Self { FetchError::ParseFloatError(e) }
}
impl From::<rusqlite::Error> for FetchError {
fn from(e: rusqlite::Error) -> Self { FetchError::RusqliteError(e) }
}
pub struct Panel {
pub(crate) id: i32,
pub name: String,
pub view_scroll: bool,
pub view_size: i32,
pub(crate) width: i32,
pub(crate) height: i32,
pub(crate) sources: Mutex<Vec<Source>>,
}
impl Panel {
}
pub struct Source {
pub(crate) id: i32,
pub name: String,
pub url: String,
pub interval: i32,
pub(crate) last_fetch: DateTime<Utc>,
pub query_x: String,
// pub(crate) compiled_query_x: Arc<Mutex<jq_rs::JqProgram>>,
pub query_y: String,
// pub(crate) compiled_query_y: Arc<Mutex<jq_rs::JqProgram>>,
pub(crate) panel_id: i32,
pub(crate) data: Mutex<Vec<Value>>,
}
impl Source {
pub fn valid(&self) -> bool {
return (Utc::now() - self.last_fetch).num_seconds() < self.interval as i64;
}
pub fn values(&self) -> Values {
Values::from_values(self.data.lock().unwrap().clone())
}
pub async fn fetch(&self) -> Result<Value, FetchError> {
let res = reqwest::get(&self.url).await?;
let body = res.text().await?;
let x = jq_rs::compile(&self.query_x)?.run(&body)?.parse::<f64>()?;
let y = jq_rs::compile(&self.query_y)?.run(&body)?.parse::<f64>()?;
return Ok( Value { x, y } );
}
}

207
src/app/data/store.rs Normal file
View file

@ -0,0 +1,207 @@
use std::sync::{Arc, Mutex};
use chrono::{DateTime, TimeZone, NaiveDateTime, Utc};
use rusqlite::{Connection, params};
use eframe::egui::plot::Value;
use crate::app::data::{Panel, Source};
use super::FetchError;
pub trait DataStorage {
fn add_panel(&self, name:&str);
}
pub struct SQLiteDataStore {
conn: Connection,
pub(crate) panels: Mutex<Vec<Panel>>,
}
impl SQLiteDataStore {
pub fn new(path: std::path::PathBuf) -> Result<Self, rusqlite::Error> {
let conn = Connection::open(path)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS panels (
id INTEGER PRIMARY KEY,
name TEXT UNIQUE,
view_scroll BOOL,
view_size INT,
width INT,
height INT
);",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS sources (
id INTEGER PRIMARY KEY,
name TEXT UNIQUE,
url TEXT,
interval INT,
query_x TEXT,
query_y TEXT,
panel_id INT
);",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS points (
id INTEGER PRIMARY KEY,
panel_id INT,
source_id INT,
x FLOAT,
y FLOAT
);",
[],
)?;
let mut store = SQLiteDataStore {
conn,
panels: Mutex::new(Vec::new()),
};
store.load_panels()?;
return Ok(store);
}
fn load_values(&self, panel_id:i32, source_id:i32) -> rusqlite::Result<Vec<Value>> {
let mut values : Vec<Value> = Vec::new();
let mut statement = self.conn.prepare("SELECT x, y FROM points WHERE panel_id = ? AND source_id = ?")?;
let values_iter = statement.query_map(params![panel_id, source_id], |row| {
Ok(Value{ x: row.get(0)?, y: row.get(1)? })
})?;
for value in values_iter {
if let Ok(v) = value {
values.push(v);
}
}
Ok(values)
}
fn put_value(&self, panel_id:i32, source_id:i32, v:Value) -> rusqlite::Result<usize> {
self.conn.execute(
"INSERT INTO points(panel_id, source_id, x, y) VALUES (?, ?, ?, ?)",
params![panel_id, source_id, v.x, v.y],
)
}
fn load_sources(&self, panel_id:i32) -> rusqlite::Result<Vec<Source>> {
let mut sources : Vec<Source> = Vec::new();
let mut statement = self.conn.prepare("SELECT * FROM sources WHERE panel_id = ?")?;
let sources_iter = statement.query_map(params![panel_id], |row| {
Ok(Source{
id: row.get(0)?,
name: row.get(1)?,
url: row.get(2)?,
interval: row.get(3)?,
last_fetch: Utc.ymd(1970, 1, 1).and_hms(0, 0, 0),
query_x: row.get(4)?,
// compiled_query_x: Arc::new(Mutex::new(jq_rs::compile(row.get::<usize, String>(4)?.as_str()).unwrap())),
query_y: row.get(5)?,
// compiled_query_y: Arc::new(Mutex::new(jq_rs::compile(row.get::<usize, String>(5)?.as_str()).unwrap())),
panel_id: row.get(6)?,
data: Mutex::new(Vec::new()),
})
})?;
for source in sources_iter {
if let Ok(mut s) = source {
s.data = Mutex::new(self.load_values(panel_id, s.id)?);
sources.push(s);
}
}
Ok(sources)
}
fn put_source(&self, panel_id:i32, s:Source) -> rusqlite::Result<usize> {
self.conn.execute(
"INSERT INTO sources(id, name, url, interval, query_x, query_y, panel_id) VALUES (?, ?, ?, ?, ?, ?, ?)",
params![s.id, s.name, s.url, s.interval, s.query_x, s.query_y, panel_id],
)
}
fn load_panels(&self) -> rusqlite::Result<Vec<Panel>> {
let mut panels : Vec<Panel> = Vec::new();
let mut statement = self.conn.prepare("SELECT * FROM panels")?;
let panels_iter = statement.query_map([], |row| {
Ok(Panel{
id: row.get(0)?,
name: row.get(1)?,
view_scroll: row.get(2)?,
view_size: row.get(3)?,
width: row.get(4)?,
height: row.get(5)?,
sources: Mutex::new(Vec::new()),
})
})?;
for panel in panels_iter {
if let Ok(mut p) = panel {
p.sources = Mutex::new(self.load_sources(p.id)?);
panels.push(p);
}
}
Ok(panels)
}
fn put_panel(&self, name:&str, view_scroll:bool, view_size:i32, width:i32, height:i32) -> rusqlite::Result<usize> {
self.conn.execute(
"INSERT INTO panels (name, view_scroll, view_size, width, height) VALUES (?, ?, ?, ?, ?)",
params![name, view_scroll, view_size, width, height]
)
}
// jank! TODO make it not jank!
fn new_panel(&self, name:&str) -> rusqlite::Result<Panel> {
self.put_panel(name, true, 100, 400, 280)?;
let mut statement = self.conn.prepare("SELECT * FROM panels WHERE name = ?")?;
for panel in statement.query_map(params![name], |row| {
Ok(Panel{
id: row.get(0)?,
name: row.get(1)?,
view_scroll: row.get(2)?,
view_size: row.get(3)?,
width: row.get(4)?,
height: row.get(5)?,
sources: Mutex::new(Vec::new()),
})
})? {
if let Ok(p) = panel {
return Ok(p);
} else {
println!("WTF");
}
}
Err(rusqlite::Error::QueryReturnedNoRows)
}
pub async fn fetch_all(&self) -> Result<(), FetchError> {
let panels = &*self.panels.lock().unwrap();
for i in 0..panels.len() {
let sources = &*panels[i].sources.lock().unwrap();
for j in 0..sources.len() {
if !sources[j].valid() {
let v = sources[j].fetch().await?;
self.put_value(panels[i].id, sources[j].id, v)?;
sources[j].data.lock().unwrap().push(v);
}
}
}
Ok(())
}
}
impl DataStorage for SQLiteDataStore {
fn add_panel(&self, name:&str) {
let panel = self.new_panel(name).unwrap();
self.panels.lock().unwrap().push(panel);
}
}

View file

@ -1,202 +0,0 @@
use std::sync::{Arc, Mutex};
use rand::Rng;
use std::io::{Write, Read};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize, de::{DeserializeOwned}};
use eframe::egui::{plot::Value, Context};
pub fn native_save(name: &str, data:String) -> std::io::Result<()> {
let mut file = std::fs::File::create(name)?;
file.write_all(data.as_bytes())?;
return Ok(());
}
pub struct DataSource {
data : Arc<Mutex<Vec<Value>>>,
}
#[derive(Serialize, Deserialize)]
struct SerializableValue {
x : f64,
y : f64,
}
impl DataSource {
pub fn new() -> Self {
Self{ data: Arc::new(Mutex::new(Vec::new())) }
}
pub fn view(&self) -> Vec<Value> { // TODO handle errors
return self.data.lock().unwrap().clone();
}
pub fn serialize(&self) -> String {
let mut out : Vec<SerializableValue> = Vec::new();
for value in self.view() {
out.push(SerializableValue { x: value.x, y: value.y });
}
return serde_json::to_string(&out).unwrap();
}
}
pub trait PlotValue {
fn as_value(&self) -> Value;
}
pub trait Data {
fn load_remote(&mut self, url:&str, ctx:Context);
fn load_local(&mut self, file:&str, ctx:Context);
fn read(&mut self, file:&str, storage:Arc<Mutex<Vec<Value>>>, ctx:Context) -> std::io::Result<()> {
let mut file = std::fs::File::open(file)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
let data : Vec<SerializableValue> = serde_json::from_str(contents.as_str())?;
for v in data {
storage.lock().unwrap().push(Value { x: v.x, y: v.y });
}
ctx.request_repaint();
Ok(())
}
fn fetch<T>(&mut self, base:&str, endpoint:&str, storage:Arc<Mutex<Vec<Value>>>, ctx:Context)
where T : DeserializeOwned + PlotValue {
let request = ehttp::Request::get(format!("{}/{}", base, endpoint));
ehttp::fetch(request, move |result: ehttp::Result<ehttp::Response>| {
let data : T = serde_json::from_slice(result.unwrap().bytes.as_slice()).unwrap();
storage.lock().unwrap().push(data.as_value());
ctx.request_repaint();
});
}
}
pub struct TpsData {
pub ds: DataSource,
load_interval : i64,
last_load : DateTime<Utc>,
}
#[derive(Serialize, Deserialize)]
struct TpsResponseData {
tps: f64
}
impl PlotValue for TpsResponseData {
fn as_value(&self) -> Value {
Value { x: Utc::now().timestamp() as f64, y: self.tps }
}
}
impl TpsData {
pub fn new(load_interval:i64) -> Self {
Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
}
}
impl Data for TpsData{
fn load_remote(&mut self, url:&str, ctx:Context) {
if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
self.last_load = Utc::now();
self.fetch::<TpsResponseData>(url, "tps", self.ds.data.clone(), ctx);
}
fn load_local(&mut self, file:&str, ctx:Context) {
self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
}
}
pub struct ChatData {
pub ds : DataSource,
load_interval : i64,
last_load : DateTime<Utc>,
}
#[derive(Serialize, Deserialize)]
struct ChatResponseData {
volume: f64
}
impl PlotValue for ChatResponseData {
fn as_value(&self) -> Value {
Value { x:Utc::now().timestamp() as f64, y: self.volume }
}
}
impl ChatData {
pub fn new(load_interval:i64) -> Self {
Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
}
}
impl Data for ChatData{
fn load_remote(&mut self, url:&str, ctx:Context) {
if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
self.last_load = Utc::now();
self.fetch::<ChatResponseData>(url, "chat_activity", self.ds.data.clone(), ctx);
}
fn load_local(&mut self, file:&str, ctx:Context) {
self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
}
}
pub struct PlayerCountData {
pub ds : DataSource,
load_interval : i64,
last_load : DateTime<Utc>,
}
#[derive(Serialize, Deserialize)]
struct PlayerCountResponseData {
count: i32
}
impl PlotValue for PlayerCountResponseData {
fn as_value(&self) -> Value {
Value { x:Utc::now().timestamp() as f64, y: self.count as f64 }
}
}
impl PlayerCountData {
pub fn new(load_interval:i64) -> Self {
Self { ds: DataSource::new() , last_load: Utc::now(), load_interval }
}
}
impl Data for PlayerCountData{
fn load_remote(&mut self, url:&str, ctx:Context) {
if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
self.last_load = Utc::now();
self.fetch::<PlayerCountResponseData>(url, "player_count", self.ds.data.clone(), ctx);
}
fn load_local(&mut self, file:&str, ctx:Context) {
self.read(file, self.ds.data.clone(), ctx).unwrap_or_else(|_err| println!("Could not load {}", file));
}
}
pub struct RandomData {
pub ds : DataSource,
load_interval : i64,
last_load : DateTime<Utc>,
rng: rand::rngs::ThreadRng,
}
impl RandomData {
#[allow(dead_code)]
pub fn new(load_interval:i64) -> Self {
Self { ds: DataSource::new() , last_load: Utc::now(), load_interval, rng : rand::thread_rng() }
}
}
impl Data for RandomData{
fn load_remote(&mut self, _url:&str, ctx:Context) {
if (Utc::now() - self.last_load).num_seconds() < self.load_interval { return; }
self.last_load = Utc::now();
self.ds.data.lock().unwrap().push(Value {x:Utc::now().timestamp() as f64, y:self.rng.gen()});
ctx.request_repaint();
}
fn load_local(&mut self, _file:&str, _ctx:Context) {}
}

View file

@ -1,12 +1,17 @@
mod datasource;
pub mod data;
use std::sync::{Arc, Mutex};
use chrono::{DateTime, NaiveDateTime, Utc};
use datasource::{ChatData, PlayerCountData, TpsData, Data, native_save};
use data::source::{ChatData, PlayerCountData, TpsData, Data, native_save};
use eframe::egui;
use eframe::egui::plot::{Line, Plot, Values};
use crate::app::data::store::DataStorage;
use self::data::store::SQLiteDataStore;
pub struct App {
servers : Vec<ServerOptions>,
// data : SQLiteDataStore,
data : Arc<SQLiteDataStore>,
}
struct ServerOptions {
@ -19,83 +24,20 @@ struct ServerOptions {
}
impl App {
pub fn new(_cc: &eframe::CreationContext) -> Self {
let mut servers = Vec::new();
servers.push(ServerOptions::new("9b9t", "https://alemi.dev/mcbots/9b"));
servers.push(ServerOptions::new("const", "https://alemi.dev/mcbots/const"));
servers.push(ServerOptions::new("of", "https://alemi.dev/mcbots/of"));
Self { servers }
}
}
impl ServerOptions {
fn new(title:&str, url:&str) -> Self {
Self {
title: title.to_string(),
url: url.to_string(),
player_count: PlayerCountData::new(60),
tps: TpsData::new(15),
chat: ChatData::new(30),
sync_time: false,
}
}
fn display(&mut self, ui:&mut eframe::egui::Ui) {
ui.vertical(|ui| {
ui.horizontal(|ui| {
ui.heading(self.title.as_str());
ui.checkbox(&mut self.sync_time, "Lock X to now");
});
let mut p = Plot::new(format!("plot-{}", self.title)).x_axis_formatter(|x, _range| {
format!(
"{}",
DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc)
.format("%Y/%m/%d %H:%M:%S")
)
}).center_x_axis(false).height(260.0); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do...
if self.sync_time {
p = p.include_x(Utc::now().timestamp() as f64);
}
p.show(ui, |plot_ui| {
plot_ui.line(
Line::new(Values::from_values(self.player_count.ds.view())).name("Player Count"),
);
plot_ui.line(Line::new(Values::from_values(self.tps.ds.view())).name("TPS over 15s"));
plot_ui.line(
Line::new(Values::from_values(self.chat.ds.view()))
.name("Chat messages per minute"),
);
});
});
// pub fn new(_cc: &eframe::CreationContext, data: SQLiteDataStore) -> Self {
pub fn new(_cc: &eframe::CreationContext, data: Arc<SQLiteDataStore>) -> Self {
Self { data }
}
}
impl eframe::App for App {
fn update(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) {
for server in &mut self.servers {
server.tps.load_remote(server.url.as_str(), ctx.clone());
server.player_count.load_remote(server.url.as_str(), ctx.clone());
server.chat.load_remote(server.url.as_str(), ctx.clone());
}
egui::TopBottomPanel::top("??? wtf").show(ctx, |ui| {
ui.horizontal(|ui| {
egui::widgets::global_dark_light_mode_switch(ui);
ui.heading("nnbot dashboard");
if ui.button("save").clicked() {
for server in &self.servers {
native_save(format!("{}-tps.json", server.title).as_str(), server.tps.ds.serialize()).unwrap();
native_save(format!("{}-chat.json", server.title).as_str(), server.chat.ds.serialize()).unwrap();
native_save(format!("{}-players.json", server.title).as_str(), server.player_count.ds.serialize()).unwrap();
}
}
if ui.button("load").clicked() {
for server in &mut self.servers {
server.tps.load_local(format!("{}-tps.json", server.title).as_str(), ctx.clone());
server.chat.load_local(format!("{}-chat.json", server.title).as_str(), ctx.clone());
server.player_count.load_local(format!("{}-players.json", server.title).as_str(), ctx.clone());
}
ui.heading("dashboard");
if ui.button("test add").clicked() {
self.data.add_panel("test panel");
}
ui.with_layout(egui::Layout::top_down(egui::Align::RIGHT), |ui| {
if ui.button("x").clicked() {
@ -105,15 +47,36 @@ impl eframe::App for App {
});
});
egui::CentralPanel::default().show(ctx, |ui| {
ui.group(|v_ui| {
self.servers[0].display(v_ui);
});
ui.group(|v_ui| {
self.servers[1].display(v_ui);
});
ui.group(|v_ui| {
self.servers[2].display(v_ui);
});
let panels = &*self.data.panels.lock().unwrap();
for i in 0..panels.len() {
// for panel in self.data.view() {
ui.group(|ui| {
ui.vertical(|ui| {
ui.horizontal(|ui| {
ui.heading(panels[i].name.as_str());
// ui.checkbox(&mut panel.view_scroll, "autoscroll");
});
let mut p = Plot::new(format!("plot-{}", panels[i].name)).x_axis_formatter(|x, _range| {
format!(
"{}",
DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(x as i64, 0), Utc)
.format("%Y/%m/%d %H:%M:%S")
)
}).center_x_axis(false).height(panels[i].height as f32); // TODO make it fucking reactive! It fills the whole screen with 1 plot no matter what I do...
if panels[i].view_scroll {
p = p.include_x(Utc::now().timestamp() as f64);
}
p.show(ui, |plot_ui| {
let sources = &*panels[i].sources.lock().unwrap();
for j in 0..sources.len() {
plot_ui.line(Line::new(sources[j].values()).name(sources[j].name.as_str()));
}
});
});
});
}
});
ctx.request_repaint(); // TODO super jank way to sorta keep drawing
}