From 192ce94ac6f14ca7d03682f3ab67a6d0559fcb1f Mon Sep 17 00:00:00 2001 From: alemidev Date: Tue, 18 Oct 2022 23:08:08 +0200 Subject: [PATCH 01/29] fix: where did this come from?? --- plugin/codemp.vim | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/plugin/codemp.vim b/plugin/codemp.vim index e601b00..f0d492d 100644 --- a/plugin/codemp.vim +++ b/plugin/codemp.vim @@ -1,11 +1,3 @@ -" Copyright 2017 Justin Charette -" -" Licensed under the Apache License, Version 2.0 or the MIT license -" , at your -" option. This file may not be copied, modified, or distributed -" except according to those terms. - if ! exists('s:jobid') let s:jobid = 0 endif @@ -95,6 +87,6 @@ function codemp#cursor() endfunction function s:OnStderr(id, data, event) dict - let g:msg = 'codemp: stderr: ' . join(a:data, "\n") + let g:msg = 'codemp: ' . join(a:data, "\n") echo g:msg endfunction From ebbca24a99c8b0488981ffdddfa8101bca3669bd Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 7 Apr 2023 03:05:21 +0200 Subject: [PATCH 02/29] chore: dramatically simplified everything working on this was really hard, so i'm making simple things first. removed almost everything except bare buffer changes, and not even done in a smart way, but should be a working PoC? now trying to make a working client to test it out and actually work on a real prototype --- Cargo.toml | 40 +++-- build.rs | 2 - proto/buffer.proto | 29 ++-- proto/session.proto | 25 ---- proto/workspace.proto | 51 ------- src/client/cli/main.rs | 46 ++++++ src/client/dispatcher.rs | 96 ------------ src/client/main.rs | 16 -- src/client/nvim/main.rs | 91 +++++++++++ src/client/nvim/mod.rs | 133 ---------------- src/lib/events.rs | 100 ------------- src/lib/lib.rs | 6 +- src/lib/proto.rs | 1 + src/lib/user.rs | 27 ---- src/server/actor/buffer.rs | 67 --------- src/server/actor/mod.rs | 3 - src/server/actor/state.rs | 106 ------------- src/server/actor/workspace.rs | 228 ---------------------------- src/server/buffer/actor.rs | 83 ++++++++++ src/server/buffer/mod.rs | 2 + src/server/buffer/service.rs | 105 +++++++++++++ src/server/main.rs | 16 +- src/server/service/buffer.rs | 156 ------------------- src/server/service/mod.rs | 3 - src/server/service/session.rs | 59 -------- src/server/service/workspace.rs | 258 -------------------------------- 26 files changed, 371 insertions(+), 1378 deletions(-) delete mode 100644 proto/session.proto delete mode 100644 proto/workspace.proto create mode 100644 src/client/cli/main.rs delete mode 100644 src/client/dispatcher.rs delete mode 100644 src/client/main.rs create mode 100644 src/client/nvim/main.rs delete mode 100644 src/client/nvim/mod.rs delete mode 100644 src/lib/events.rs create mode 100644 src/lib/proto.rs delete mode 100644 src/lib/user.rs delete mode 100644 src/server/actor/buffer.rs delete mode 100644 src/server/actor/mod.rs delete mode 100644 src/server/actor/state.rs delete mode 100644 src/server/actor/workspace.rs create mode 100644 src/server/buffer/actor.rs create mode 100644 src/server/buffer/mod.rs create mode 100644 src/server/buffer/service.rs delete mode 100644 src/server/service/buffer.rs delete mode 100644 src/server/service/mod.rs delete mode 100644 src/server/service/session.rs delete mode 100644 src/server/service/workspace.rs diff --git a/Cargo.toml b/Cargo.toml index ba0fdc2..86b27e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,9 @@ name = "codemp" version = "0.1.0" edition = "2021" -[features] -default = ["nvim"] -nvim = [] +# [features] +# default = ["nvim"] +# nvim = [] [lib] name = "library" @@ -15,23 +15,35 @@ path = "src/lib/lib.rs" name = "server" path = "src/server/main.rs" -[[bin]] # Bin to run the CodeMP gRPC client -name = "client-neovim" -path = "src/client/main.rs" +[[bin]] +name = "client-cli" +path = "src/client/cli/main.rs" +required-features = ["cli"] + +[[bin]] +name = "client-nvim" +path = "src/client/nvim/main.rs" +required-features = ["nvim"] [dependencies] tracing = "0.1" tracing-subscriber = "0.3" -tonic = "0.7" -prost = "0.10" -futures = "0.3" +tonic = "0.9" tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "full"] } tokio-stream = "0.1" rmpv = "1" -operational-transform = "0.6" -nvim-rs = { version = "0.4", features = ["use_tokio"] } # TODO put this behind a conditional feature -uuid = { version = "1", features = ["v4", "fast-rng", "macro-diagnostics"] } -rand = "0.8.5" +serde = "1" +serde_json = "1" +operational-transform = { version = "0.6", features = ["serde"] } +md5 = "0.7.0" +prost = "0.11.8" +clap = { version = "4.2.1", features = ["derive"], optional = true } +nvim-rs = { version = "0.5", features = ["use_tokio"], optional = true } [build-dependencies] -tonic-build = "0.7" +tonic-build = "0.9" + +[features] +default = [] +cli = ["dep:clap"] +nvim = ["dep:nvim-rs"] diff --git a/build.rs b/build.rs index 2b8ba6f..838368e 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,4 @@ fn main() -> Result<(), Box> { - tonic_build::compile_protos("proto/session.proto")?; - tonic_build::compile_protos("proto/workspace.proto")?; tonic_build::compile_protos("proto/buffer.proto")?; Ok(()) } diff --git a/proto/buffer.proto b/proto/buffer.proto index 8e3b3a3..671b991 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -2,35 +2,26 @@ syntax = "proto3"; package buffer; service Buffer { - rpc Attach (stream Operation) returns (stream Operation); - rpc Push (BufferPayload) returns (BufferResponse); - rpc Pull (BufferPayload) returns (BufferPayload); + rpc Attach (BufferPayload) returns (stream RawOp); + rpc Edit (OperationRequest) returns (BufferResponse); + rpc Create (BufferPayload) returns (BufferResponse); } -message Operation { - int64 opId = 1; +message RawOp { + string opseq = 1; +} - enum Action { - RETAIN = 0; - INSERT = 1; - DELETE = 2; - }; - Action action = 2; - - int32 row = 3; - int32 column = 4; - - optional string text = 5; +message OperationRequest { + string path = 1; + string hash = 2; + string opseq = 3; } message BufferPayload { - string sessionKey = 1; string path = 2; optional string content = 3; } message BufferResponse { - string sessionKey = 1; - string path = 2; bool accepted = 3; } diff --git a/proto/session.proto b/proto/session.proto deleted file mode 100644 index 7273946..0000000 --- a/proto/session.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; -package session; - -service Session { - // rpc Authenticate(SessionRequest) returns (SessionResponse); - // rpc ListWorkspaces(SessionRequest) returns (WorkspaceList); - rpc CreateWorkspace(WorkspaceBuilderRequest) returns (SessionResponse); -} - -message SessionRequest { - string sessionKey = 1; -} - -message SessionResponse { - string sessionKey = 1; - bool accepted = 2; -} - -message WorkspaceBuilderRequest { - string name = 1; -} - -message WorkspaceList { - repeated string name = 1; // TODO add more fields -} diff --git a/proto/workspace.proto b/proto/workspace.proto deleted file mode 100644 index ae17813..0000000 --- a/proto/workspace.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; -package workspace; - -service Workspace { - rpc Join (JoinRequest) returns (stream WorkspaceEvent); - rpc Subscribe (stream CursorUpdate) returns (stream CursorUpdate); - rpc ListUsers (WorkspaceRequest) returns (UsersList); - rpc Buffers (WorkspaceRequest) returns (BufferList); - rpc NewBuffer (BufferRequest) returns (WorkspaceResponse); - rpc RemoveBuffer (BufferRequest) returns (WorkspaceResponse); -} - -message JoinRequest { - string name = 1; -} - -message WorkspaceEvent { - int32 id = 1; - optional string body = 2; -} - -// nvim-rs passes everything as i64, so having them as i64 in the packet itself is convenient -// TODO can we make them i32 and save some space? -message CursorUpdate { - string username = 1; - int64 buffer = 2; - int64 col = 3; - int64 row = 4; -} - -message WorkspaceRequest { - string sessionKey = 1; -} - -message BufferRequest { - string sessionKey = 1; - string path = 2; -} - -message WorkspaceResponse { - bool accepted = 1; -} - -message BufferList { - repeated string path = 1; -} - -message UsersList { - repeated string name = 1; -} - diff --git a/src/client/cli/main.rs b/src/client/cli/main.rs new file mode 100644 index 0000000..8bf722d --- /dev/null +++ b/src/client/cli/main.rs @@ -0,0 +1,46 @@ +use clap::Parser; +use library::proto::{buffer_client::BufferClient, BufferPayload}; +use tokio_stream::StreamExt; + +#[derive(Parser, Debug)] +struct CliArgs { + /// path of buffer to create + path: String, + + /// initial content for buffer + #[arg(short, long)] + content: Option, + + /// attach instead of creating a new buffer + #[arg(long, default_value_t = false)] + attach: bool, + + /// host to connect to + #[arg(long, default_value = "http://[::1]:50051")] + host: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = CliArgs::parse(); + + let mut client = BufferClient::connect(args.host).await?; + + let request = BufferPayload { + path: args.path, + content: args.content, + }; + + if !args.attach { + client.create(request.clone()).await.unwrap(); + } + + let mut stream = client.attach(request).await.unwrap().into_inner(); + + while let Some(item) = stream.next().await { + println!("> {:?}", item); + } + + Ok(()) +} + diff --git a/src/client/dispatcher.rs b/src/client/dispatcher.rs deleted file mode 100644 index a56aa73..0000000 --- a/src/client/dispatcher.rs +++ /dev/null @@ -1,96 +0,0 @@ -pub mod proto { - tonic::include_proto!("session"); - tonic::include_proto!("workspace"); - tonic::include_proto!("buffer"); -} -use std::sync::Arc; -use tracing::error; - -use tokio::sync::{mpsc, Mutex}; -use tokio_stream::StreamExt; -use tokio_stream::wrappers::ReceiverStream; - -use proto::{ - workspace_client::WorkspaceClient, - session_client::SessionClient, - buffer_client::BufferClient, - WorkspaceBuilderRequest, JoinRequest, SessionResponse, CursorUpdate -}; -use tonic::{transport::Channel, Status, Request, Response}; - -#[derive(Clone)] -pub struct Dispatcher { - name: String, - dp: Arc>, // TODO use channels and don't lock -} - -struct DispatcherWorker { - // TODO do I need all three? Did I design the server badly? - session: SessionClient, - workspace: WorkspaceClient, - _buffers: BufferClient, -} - -impl Dispatcher { - pub async fn connect(addr:String) -> Result { - let (s, w, b) = tokio::join!( - SessionClient::connect(addr.clone()), - WorkspaceClient::connect(addr.clone()), - BufferClient::connect(addr.clone()), - ); - Ok( - Dispatcher { - name: format!("User#{}", rand::random::()), - dp: Arc::new( - Mutex::new( - DispatcherWorker { session: s?, workspace: w?, _buffers: b? } - ) - ) - } - ) - } - - pub async fn create_workspace(&self, name:String) -> Result, Status> { - self.dp.lock().await.session.create_workspace( - Request::new(WorkspaceBuilderRequest { name }) - ).await - } - - pub async fn join_workspace(&self, session_id:String) -> Result<(), Status> { - let mut req = Request::new(JoinRequest { name: self.name.clone() }); - req.metadata_mut().append("workspace", session_id.parse().unwrap()); - let mut stream = self.dp.lock().await.workspace.join(req).await?.into_inner(); - - let _worker = tokio::spawn(async move { - while let Some(pkt) = stream.next().await { - match pkt { - Ok(_event) => { - // TODO do something with events when they will mean something! - }, - Err(e) => error!("Error receiving event | {}", e), - } - } - }); - - Ok(()) - } - - pub async fn start_cursor_worker(&self, session_id:String, feed:mpsc::Receiver) -> Result, Status> { - let mut in_stream = Request::new(ReceiverStream::new(feed)); - in_stream.metadata_mut().append("workspace", session_id.parse().unwrap()); - - let mut stream = self.dp.lock().await.workspace.subscribe(in_stream).await?.into_inner(); - let (tx, rx) = mpsc::channel(50); - - let _worker = tokio::spawn(async move { - while let Some(pkt) = stream.next().await { - match pkt { - Ok(update) => tx.send(update).await.unwrap(), // TODO how to handle an error here? - Err(e) => error!("Error receiving cursor update | {}", e), - } - } - }); - - Ok(rx) - } -} diff --git a/src/client/main.rs b/src/client/main.rs deleted file mode 100644 index 37eae85..0000000 --- a/src/client/main.rs +++ /dev/null @@ -1,16 +0,0 @@ - -mod nvim; -pub mod dispatcher; - -use dispatcher::Dispatcher; - -#[tokio::main] -async fn main() -> Result<(), Box<(dyn std::error::Error + 'static)>> { - - let dispatcher = Dispatcher::connect("http://[::1]:50051".into()).await.unwrap(); - - #[cfg(feature = "nvim")] - crate::nvim::run_nvim_client(dispatcher).await?; - - Ok(()) -} diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs new file mode 100644 index 0000000..f71ec8b --- /dev/null +++ b/src/client/nvim/main.rs @@ -0,0 +1,91 @@ +//! A basic example. Mainly for use in a test, but also shows off some basic +//! functionality. +use std::{env, error::Error, fs}; + +use rmpv::Value; + +use tokio::io::Stdout; + +use nvim_rs::{ + compat::tokio::Compat, create::tokio as create, rpc::IntoVal, Handler, Neovim, +}; +use tonic::async_trait; + +#[derive(Clone)] +struct NeovimHandler { + +} + +#[async_trait] +impl Handler for NeovimHandler { + type Writer = Compat; + + async fn handle_request( + &self, + name: String, + args: Vec, + nvim: Neovim>, + ) -> Result { + match name.as_ref() { + "ping" => Ok(Value::from("pong")), + _ => unimplemented!(), + } + } + + async fn handle_notify( + &self, + name: String, + args: Vec, + nvim: Neovim>, + ) { + } +} + +#[tokio::main] +async fn main() { + let handler: NeovimHandler = NeovimHandler {}; + let (nvim, io_handler) = create::new_parent(handler).await; + let curbuf = nvim.get_current_buf().await.unwrap(); + + let mut envargs = env::args(); + let _ = envargs.next(); + let testfile = envargs.next().unwrap(); + + fs::write(testfile, &format!("{:?}", curbuf.into_val())).unwrap(); + + // Any error should probably be logged, as stderr is not visible to users. + match io_handler.await { + Err(joinerr) => eprintln!("Error joining IO loop: '{}'", joinerr), + Ok(Err(err)) => { + if !err.is_reader_error() { + // One last try, since there wasn't an error with writing to the + // stream + nvim + .err_writeln(&format!("Error: '{}'", err)) + .await + .unwrap_or_else(|e| { + // We could inspect this error to see what was happening, and + // maybe retry, but at this point it's probably best + // to assume the worst and print a friendly and + // supportive message to our users + eprintln!("Well, dang... '{}'", e); + }); + } + + if !err.is_channel_closed() { + // Closed channel usually means neovim quit itself, or this plugin was + // told to quit by closing the channel, so it's not always an error + // condition. + eprintln!("Error: '{}'", err); + + let mut source = err.source(); + + while let Some(e) = source { + eprintln!("Caused by: '{}'", e); + source = e.source(); + } + } + } + Ok(Ok(())) => {} + } +} diff --git a/src/client/nvim/mod.rs b/src/client/nvim/mod.rs deleted file mode 100644 index 5db4a62..0000000 --- a/src/client/nvim/mod.rs +++ /dev/null @@ -1,133 +0,0 @@ -use std::sync::Arc; - -use rmpv::Value; - -use tokio::io::Stdout; - -use nvim_rs::{compat::tokio::Compat, Handler, Neovim}; -use nvim_rs::create::tokio::new_parent; -use tokio::sync::{mpsc, Mutex}; - -use crate::dispatcher::{Dispatcher, proto::CursorUpdate}; - -#[derive(Clone)] -pub struct NeovimHandler { - dispatcher: Dispatcher, - sink: Arc>>>, -} - -impl NeovimHandler { - pub fn new(dispatcher: Dispatcher) -> Self { - NeovimHandler { - dispatcher, - sink: Arc::new(Mutex::new(None)), - } - } -} - -#[tonic::async_trait] -impl Handler for NeovimHandler { - type Writer = Compat; - - async fn handle_request( - &self, - name: String, - args: Vec, - neovim: Neovim>, - ) -> Result { - match name.as_ref() { - "ping" => Ok(Value::from("pong")), - "create" => { - if args.len() < 1 { - return Err(Value::from("[!] no session key")); - } - let res = self.dispatcher.create_workspace(args[0].to_string()) - .await - .map_err(|e| Value::from(e.to_string()))? - .into_inner(); - - Ok(res.session_key.into()) - }, - "join" => { - if args.len() < 1 { - return Err(Value::from("[!] no session key")); - } - - self.dispatcher.join_workspace( - args[0].as_str().unwrap().to_string(), // TODO throw err if it's not a string? - ).await.map_err(|e| Value::from(e.to_string()))?; - - Ok("OK".into()) - }, - "cursor-start" => { - if args.len() < 1 { - return Err(Value::from("[!] no session key")); - } - let (tx, stream) = mpsc::channel(50); - let mut rx = self.dispatcher.start_cursor_worker( - args[0].as_str().unwrap().to_string(), stream - ).await.map_err(|e| Value::from(e.to_string()))?; - let sink = self.sink.clone(); - sink.lock().await.replace(tx); - let _worker = tokio::spawn(async move { - let mut col : i64; - let mut row : i64; - let ns = neovim.create_namespace("Cursor").await.unwrap(); - while let Some(update) = rx.recv().await { - neovim.exec_lua(format!("print('{:?}')", update).as_str(), vec![]).await.unwrap(); - let buf = neovim.get_current_buf().await.unwrap(); - buf.clear_namespace(ns, 0, -1).await.unwrap(); - row = update.row as i64; - col = update.col as i64; - buf.add_highlight(ns, "ErrorMsg", row-1, col-1, col).await.unwrap(); - } - sink.lock().await.take(); - }); - Ok("OK".into()) - }, - _ => { - eprintln!("[!] unexpected call"); - Ok(Value::from("")) - }, - } - } - - async fn handle_notify( - &self, - name: String, - args: Vec, - _neovim: Neovim>, - ) { - match name.as_ref() { - "insert" => {}, - "cursor" => { - if args.len() >= 3 { - if let Some(sink) = self.sink.lock().await.as_ref() { - sink.send(CursorUpdate { - buffer: args[0].as_i64().unwrap(), - row: args[1].as_i64().unwrap(), - col: args[2].as_i64().unwrap(), - username: "root".into() - }).await.unwrap(); - } - } - }, - "tick" => eprintln!("tock"), - _ => eprintln!("[!] unexpected notify",) - } - } -} - -pub async fn run_nvim_client(dispatcher: Dispatcher) -> Result<(), Box> { - let handler: NeovimHandler = NeovimHandler::new(dispatcher); - let (_nvim, io_handler) = new_parent(handler).await; - - // Any error should probably be logged, as stderr is not visible to users. - match io_handler.await { - Err(err) => eprintln!("Error joining IO loop: {:?}", err), - Ok(Err(err)) => eprintln!("Process ended with error: {:?}", err), - Ok(Ok(())) => eprintln!("Finished"), - } - - Ok(()) -} diff --git a/src/lib/events.rs b/src/lib/events.rs deleted file mode 100644 index 2d01ad8..0000000 --- a/src/lib/events.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::fmt::Display; -use crate::user::User; - -#[derive(Debug, Clone)] -pub enum Event { - UserJoin { user: User }, - UserLeave { name: String }, - BufferNew { path: String }, - BufferDelete { path: String }, -} - -impl Display for Event { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UserJoin { user } => write!(f, "UserJoin(user:{})", user), - Self::UserLeave { name } => write!(f, "UserLeave(user:{})", name), - Self::BufferNew { path } => write!(f, "BufferNew(path:{})", path), - Self::BufferDelete { path } => write!(f, "BufferDelete(path:{})", path), - } - } -} - -// pub type Event = Box; -// -// pub trait EventInterface { -// fn class(&self) -> EventClass; -// fn unwrap(e: Event) -> Option where Self: Sized; -// -// fn wrap(self) -> Event { -// Box::new(self) -// } -// } -// -// -// // User joining workspace -// -// pub struct UserJoinEvent { -// user: User, -// } -// -// impl EventInterface for UserJoinEvent { -// fn class(&self) -> EventClass { EventClass::UserJoin } -// fn unwrap(e: Event) -> Option where Self: Sized { -// if matches!(e.class(), EventClass::UserJoin) { -// return Some(*e); -// } -// None -// } -// } -// -// -// // User leaving workspace -// -// pub struct UserLeaveEvent { -// name: String, -// } -// -// impl EventInterface for UserLeaveEvent { -// fn class(&self) -> EventClass { EventClass::UserLeave } -// } -// -// -// // Cursor movement -// -// pub struct CursorEvent { -// user: String, -// cursor: UserCursor, -// } -// -// impl EventInterface for CursorEvent { -// fn class(&self) -> EventClass { EventClass::Cursor } -// } -// -// impl CursorEvent { -// pub fn new(user:String, cursor: UserCursor) -> Self { -// CursorEvent { user, cursor } -// } -// } -// -// -// // Buffer added -// -// pub struct BufferNewEvent { -// path: String, -// } -// -// impl EventInterface for BufferNewEvent { -// fn class(&self) -> EventClass { EventClass::BufferNew } -// } -// -// -// // Buffer deleted -// -// pub struct BufferDeleteEvent { -// path: String, -// } -// -// impl EventInterface for BufferDeleteEvent { -// fn class(&self) -> EventClass { EventClass::BufferDelete } -// } diff --git a/src/lib/lib.rs b/src/lib/lib.rs index 84e0b84..b7446bb 100644 --- a/src/lib/lib.rs +++ b/src/lib/lib.rs @@ -1,2 +1,4 @@ -pub mod events; -pub mod user; +pub mod proto; + +pub use tonic; +pub use tokio; diff --git a/src/lib/proto.rs b/src/lib/proto.rs new file mode 100644 index 0000000..4e168ef --- /dev/null +++ b/src/lib/proto.rs @@ -0,0 +1 @@ +tonic::include_proto!("buffer"); diff --git a/src/lib/user.rs b/src/lib/user.rs deleted file mode 100644 index 0e80f53..0000000 --- a/src/lib/user.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::fmt::Display; - -#[derive(Debug, Clone)] -pub struct UserCursor{ - pub buffer: i64, - pub x: i64, - pub y: i64 -} - -impl Display for UserCursor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Cursor(buffer:{}, x:{}, y:{})", self.buffer, self.x, self.y) - } -} - - -#[derive(Debug, Clone)] -pub struct User { - pub name: String, - pub cursor: UserCursor, -} - -impl Display for User { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "User(name:{}, cursor:{})", self.name, self.cursor) - } -} diff --git a/src/server/actor/buffer.rs b/src/server/actor/buffer.rs deleted file mode 100644 index 9e28b6d..0000000 --- a/src/server/actor/buffer.rs +++ /dev/null @@ -1,67 +0,0 @@ -use operational_transform::OperationSeq; -use tokio::sync::{broadcast, mpsc, watch}; -use tracing::error; - -use library::events::Event; - -#[derive(Debug, Clone)] -/// A view of a buffer, with references to access value and send operations -pub struct BufferView { - pub name: String, - pub content: watch::Receiver, - op_tx: mpsc::Sender, -} - -impl BufferView { - pub async fn op(&self, op: OperationSeq) -> Result<(), mpsc::error::SendError> { - self.op_tx.send(op).await - } -} - -#[derive(Debug)] -pub struct Buffer { - view: BufferView, - run: watch::Sender, -} - -impl Drop for Buffer { - fn drop(&mut self) { - self.run.send(false).unwrap_or_else(|e| { - error!("Could not stop Buffer worker task: {:?}", e); - }); - } -} - -impl Buffer { - pub fn new(name: String, _bus: broadcast::Sender) -> Self { - let (op_tx, mut op_rx) = mpsc::channel(32); - let (stop_tx, stop_rx) = watch::channel(true); - let (content_tx, content_rx) = watch::channel(String::new()); - - let b = Buffer { - run: stop_tx, - view: BufferView { - name: name.clone(), - op_tx, - content: content_rx, - }, - }; - - tokio::spawn(async move { - let mut content = String::new(); - while stop_rx.borrow().to_owned() { - // TODO handle these errors!! - let op = op_rx.recv().await.unwrap(); - content = op.apply(content.as_str()).unwrap(); - // bus.send((name.clone(), op)).unwrap(); // TODO fails when there are no receivers subscribed - content_tx.send(content.clone()).unwrap(); - } - }); - - return b; - } - - pub fn view(&self) -> BufferView { - return self.view.clone(); - } -} diff --git a/src/server/actor/mod.rs b/src/server/actor/mod.rs deleted file mode 100644 index 42110ea..0000000 --- a/src/server/actor/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod buffer; -pub mod workspace; -pub mod state; diff --git a/src/server/actor/state.rs b/src/server/actor/state.rs deleted file mode 100644 index a4b8dae..0000000 --- a/src/server/actor/state.rs +++ /dev/null @@ -1,106 +0,0 @@ - -use std::{collections::HashMap, sync::Arc}; -use tokio::sync::{mpsc, watch}; -use tracing::error; -use uuid::Uuid; - -use crate::actor::workspace::Workspace; - - -#[derive(Debug)] -enum WorkspaceAction { - ADD { - key: Uuid, - w: Box, - }, - REMOVE { - key: Uuid - }, -} - -#[derive(Debug, Clone)] -pub struct WorkspacesView { - watch: watch::Receiver>>, - op: mpsc::Sender, -} - -impl WorkspacesView { - pub fn borrow(&self) -> watch::Ref>> { - self.watch.borrow() - } - - pub async fn add(&mut self, w: Workspace) { - self.op.send(WorkspaceAction::ADD { key: w.id, w: Box::new(w) }).await.unwrap(); - } - - pub async fn remove(&mut self, key: Uuid) { - self.op.send(WorkspaceAction::REMOVE { key }).await.unwrap(); - } -} - -#[derive(Debug)] -pub struct StateManager { - pub workspaces: WorkspacesView, - pub run: watch::Receiver, - run_tx: watch::Sender, -} - -impl Drop for StateManager { - fn drop(&mut self) { - self.run_tx.send(false).unwrap_or_else(|e| { - error!("Could not stop StateManager worker: {:?}", e); - }) - } -} - -impl StateManager { - pub fn new() -> Self { - let (tx, rx) = mpsc::channel(32); // TODO quantify backpressure - let (workspaces_tx, workspaces_rx) = watch::channel(HashMap::new()); - let (run_tx, run_rx) = watch::channel(true); - - let s = StateManager { - workspaces: WorkspacesView { watch: workspaces_rx, op: tx }, - run_tx, run: run_rx, - }; - - s.workspaces_worker(rx, workspaces_tx); - - return s; - } - - fn workspaces_worker(&self, mut rx: mpsc::Receiver, tx: watch::Sender>>) { - let run = self.run.clone(); - tokio::spawn(async move { - let mut store = HashMap::new(); - - while run.borrow().to_owned() { - if let Some(event) = rx.recv().await { - match event { - WorkspaceAction::ADD { key, w } => { - store.insert(key, Arc::new(*w)); // TODO put in hashmap - }, - WorkspaceAction::REMOVE { key } => { - store.remove(&key); - }, - } - tx.send(store.clone()).unwrap(); - } else { - break - } - } - }); - } - - pub fn view(&self) -> WorkspacesView { - return self.workspaces.clone(); - } - - /// get a workspace Arc directly, without passing by the WorkspacesView - pub fn get(&self, key: &Uuid) -> Option> { - if let Some(w) = self.workspaces.borrow().get(key) { - return Some(w.clone()); - } - return None; - } -} diff --git a/src/server/actor/workspace.rs b/src/server/actor/workspace.rs deleted file mode 100644 index df5a0a8..0000000 --- a/src/server/actor/workspace.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::collections::HashMap; - -use tokio::sync::{broadcast, mpsc, watch::{self, Ref}}; -use tracing::{warn, info}; - -use library::{events::Event, user::{User, UserCursor}}; - -use crate::service::workspace::proto::CursorUpdate; - -use super::buffer::{BufferView, Buffer}; - -#[derive(Debug, Clone)] -pub struct UsersView { - watch: watch::Receiver>, - op: mpsc::Sender, -} - -impl UsersView { // TODO don't unwrap everything! - pub fn borrow(&self) -> Ref> { - return self.watch.borrow(); - } - - pub async fn add(&mut self, user: User) { - self.op.send(UserAction::ADD{ user }).await.unwrap(); - } - - pub async fn remove(&mut self, name: String) { - self.op.send(UserAction::REMOVE{ name }).await.unwrap(); - } - - pub async fn update(&mut self, user_name: String, cursor: UserCursor) { - self.op.send(UserAction::CURSOR { name: user_name, cursor }).await.unwrap(); - } -} - -#[derive(Debug, Clone)] -pub struct BuffersTreeView { - watch: watch::Receiver>, - op: mpsc::Sender, -} - -impl BuffersTreeView { - pub fn borrow(&self) -> Ref> { - return self.watch.borrow(); - } - - pub async fn add(&mut self, buffer: Buffer) { - self.op.send(BufferAction::ADD { buffer }).await.unwrap(); - } - - pub async fn remove(&mut self, path: String) { - self.op.send(BufferAction::REMOVE { path }).await.unwrap(); - } -} - -pub struct WorkspaceView { - rx: broadcast::Receiver, - pub users: UsersView, - pub buffers: BuffersTreeView, -} - -impl WorkspaceView { - pub async fn event(&mut self) -> Result { - self.rx.recv().await - } -} - -// Must be clonable, containing references to the actual state maybe? Or maybe give everyone an Arc, idk -#[derive(Debug)] -pub struct Workspace { - pub id: uuid::Uuid, - pub name: String, - pub bus: broadcast::Sender, - pub cursors: broadcast::Sender, - - pub buffers: BuffersTreeView, - pub users: UsersView, - - run_tx: watch::Sender, - run_rx: watch::Receiver, -} - -impl Drop for Workspace { - fn drop(&mut self) { - self.run_tx.send(false).unwrap_or_else(|e| warn!("could not stop workspace worker: {:?}", e)); - } -} - -impl Workspace { - pub fn new(name: String) -> Self { - let (op_buf_tx, op_buf_rx) = mpsc::channel::(32); - let (op_usr_tx, op_usr_rx) = mpsc::channel::(32); - let (run_tx, run_rx) = watch::channel::(true); - let (buffer_tx, buffer_rx) = watch::channel::>(HashMap::new()); - let (users_tx, users_rx) = watch::channel(HashMap::new()); - let (broadcast_tx, _broadcast_rx) = broadcast::channel::(32); - let (cursors_tx, _cursors_rx) = broadcast::channel::(32); - - let w = Workspace { - id: uuid::Uuid::new_v4(), - name, - bus: broadcast_tx, - cursors: cursors_tx, - buffers: BuffersTreeView{ op: op_buf_tx, watch: buffer_rx }, - users: UsersView{ op: op_usr_tx, watch: users_rx }, - run_tx, - run_rx, - }; - - w.users_worker(op_usr_rx, users_tx); // spawn worker to handle users - w.buffers_worker(op_buf_rx, buffer_tx); // spawn worker to handle buffers - // - info!("new workspace created: {}[{}]", w.name, w.id); - - return w; - } - - fn buffers_worker(&self, mut rx: mpsc::Receiver, tx: watch::Sender>) { - let bus = self.bus.clone(); - let run = self.run_rx.clone(); - tokio::spawn(async move { - let mut buffers : HashMap = HashMap::new(); - - while run.borrow().to_owned() { - // TODO handle these errors!! - let action = rx.recv().await.unwrap(); - match action { - BufferAction::ADD { buffer } => { - let view = buffer.view(); - buffers.insert(view.name.clone(), buffer); - bus.send(Event::BufferNew { path: view.name }).unwrap(); - } - BufferAction::REMOVE { path } => { - buffers.remove(&path); - bus.send(Event::BufferDelete { path }).unwrap(); - } - } - tx.send( - buffers.iter() - .map(|(k, v)| (k.clone(), v.view())) - .collect() - ).unwrap(); - } - }); - } - - fn users_worker(&self, mut rx: mpsc::Receiver, tx: watch::Sender>) { - let bus = self.bus.clone(); - let cursors_tx = self.cursors.clone(); - let run = self.run_rx.clone(); - tokio::spawn(async move { - let mut cursors_rx = cursors_tx.subscribe(); - let mut users : HashMap = HashMap::new(); - - while run.borrow().to_owned() { - tokio::select!{ - action = rx.recv() => { - match action.unwrap() { - UserAction::ADD { user } => { - users.insert(user.name.clone(), user.clone()); - bus.send(Event::UserJoin { user }).unwrap(); - }, - UserAction::REMOVE { name } => { - if let None = users.remove(&name) { - continue; // don't update channel since this was a no-op - } else { - bus.send(Event::UserLeave { name }).unwrap(); - } - }, - UserAction::CURSOR { name, cursor } => { - if let Some(user) = users.get_mut(&name) { - user.cursor = cursor.clone(); - } else { - continue; // don't update channel since this was a no-op - } - }, - }; - }, - cursor = cursors_rx.recv() => { - let cursor = cursor.unwrap(); - if let Some(user) = users.get_mut(&cursor.username) { - user.cursor = UserCursor { buffer: cursor.buffer, x:cursor.col, y:cursor.row }; - } - } - } - - tx.send( - users.iter() - .map(|(k, u)| (k.clone(), u.clone())) - .collect() - ).unwrap(); - } - }); - } - - pub fn view(&self) -> WorkspaceView { - WorkspaceView { - rx: self.bus.subscribe(), - users: self.users.clone(), - buffers: self.buffers.clone(), - } - } -} - -#[derive(Debug)] -enum UserAction { - ADD { - user: User, - }, - REMOVE { - name: String, - }, - CURSOR { - name: String, - cursor: UserCursor, - }, -} - -#[derive(Debug)] -enum BufferAction { - ADD { - buffer: Buffer, - }, - REMOVE { - path: String, // TODO remove by id? - }, -} - diff --git a/src/server/buffer/actor.rs b/src/server/buffer/actor.rs new file mode 100644 index 0000000..d5b1d26 --- /dev/null +++ b/src/server/buffer/actor.rs @@ -0,0 +1,83 @@ +use tokio::sync::{mpsc, broadcast, watch}; +use tracing::error; +use md5::Digest; + +use operational_transform::OperationSeq; + +pub trait BufferStore { + fn get(&self, key: &T) -> Option<&BufferHandle>; + fn put(&mut self, key: T, handle: BufferHandle) -> Option; + + fn handle(&mut self, key: T, content: Option) { + let handle = BufferHandle::new(content); + self.put(key, handle); + } +} + +#[derive(Clone)] +pub struct BufferHandle { + pub edit: mpsc::Sender, + events: broadcast::Sender, + pub digest: watch::Receiver, +} + +impl BufferHandle { + fn new(init: Option) -> Self { + let init_val = init.unwrap_or("".into()); + let (edits_tx, edits_rx) = mpsc::channel(64); // TODO hardcoded size + let (events_tx, _events_rx) = broadcast::channel(64); // TODO hardcoded size + let (digest_tx, digest_rx) = watch::channel(md5::compute(&init_val)); + + let events_tx_clone = events_tx.clone(); + + tokio::spawn(async move { + let worker = BufferWorker { + content: init_val, + edits: edits_rx, + events: events_tx_clone, + digest: digest_tx, + }; + worker.work().await + }); + + BufferHandle { + edit: edits_tx, + events: events_tx, + digest: digest_rx, + } + } + + pub fn subscribe(&self) -> broadcast::Receiver { + self.events.subscribe() + + } +} + +struct BufferWorker { + content: String, + edits: mpsc::Receiver, + events: broadcast::Sender, + digest: watch::Sender, +} + +impl BufferWorker { + async fn work(mut self) { + loop { + match self.edits.recv().await { + None => break, + Some(v) => { + match v.apply(&self.content) { + Ok(res) => { + self.content = res; + self.digest.send(md5::compute(&self.content)).unwrap(); + if let Err(e) = self.events.send(v) { + error!("could not broadcast OpSeq: {}", e); + } + }, + Err(e) => error!("coult not apply OpSeq '{:?}' on '{}' : {}", v, self.content, e), + } + }, + } + } + } +} diff --git a/src/server/buffer/mod.rs b/src/server/buffer/mod.rs new file mode 100644 index 0000000..5c38de9 --- /dev/null +++ b/src/server/buffer/mod.rs @@ -0,0 +1,2 @@ +pub mod actor; +pub mod service; diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs new file mode 100644 index 0000000..4e8e9cb --- /dev/null +++ b/src/server/buffer/service.rs @@ -0,0 +1,105 @@ +use std::{pin::Pin, sync::{Arc, RwLock}, collections::HashMap}; + +use operational_transform::OperationSeq; +use tokio::sync::mpsc; +use tonic::{Request, Response, Status}; + +use tokio_stream::{Stream, wrappers::ReceiverStream}; // TODO example used this? + +use library::proto::{buffer_server::{Buffer, BufferServer}, RawOp, BufferPayload, BufferResponse, OperationRequest}; +use tracing::info; + +use super::actor::{BufferHandle, BufferStore}; + +type OperationStream = Pin> + Send>>; + +struct BufferMap { + store: HashMap, +} + +impl From::> for BufferMap { + fn from(value: HashMap) -> Self { + BufferMap { store: value } + } +} + +impl BufferStore for BufferMap { + fn get(&self, key: &String) -> Option<&BufferHandle> { + self.store.get(key) + } + fn put(&mut self, key: String, handle: BufferHandle) -> Option { + self.store.insert(key, handle) + } +} + +pub struct BufferService { + map: Arc>, +} + +#[tonic::async_trait] +impl Buffer for BufferService { + type AttachStream = OperationStream; + + async fn attach(&self, req: Request) -> Result, Status> { + let request = req.into_inner(); + match self.map.read().unwrap().get(&request.path) { + Some(handle) => { + let (tx, rx) = mpsc::channel(128); + let mut sub = handle.subscribe(); + tokio::spawn(async move { + loop { + match sub.recv().await { + Ok(v) => { + let snd = RawOp { opseq: serde_json::to_string(&v).unwrap() }; + tx.send(Ok(snd)).await.unwrap(); + } + Err(_e) => break, + } + } + }); + let output_stream = ReceiverStream::new(rx); + info!("registered new subscriber on buffer"); + Ok(Response::new(Box::pin(output_stream))) + }, + None => Err(Status::not_found("path not found")), + } + } + + async fn edit(&self, req:Request) -> Result, Status> { + let request = req.into_inner(); + let tx = match self.map.read().unwrap().get(&request.path) { + Some(handle) => { + if format!("{:x}", *handle.digest.borrow()) != request.hash { + return Ok(Response::new(BufferResponse { accepted : false } )); + } + handle.edit.clone() + }, + None => return Err(Status::not_found("path not found")), + }; + let opseq : OperationSeq = serde_json::from_str(&request.opseq).unwrap(); + tx.send(opseq).await.unwrap(); + info!("sent edit to buffer"); + Ok(Response::new(BufferResponse { accepted: true })) + } + + async fn create(&self, req:Request) -> Result, Status> { + let request = req.into_inner(); + let _handle = self.map.write().unwrap().handle(request.path, request.content); + info!("created new buffer"); + let answ = BufferResponse { accepted: true }; + Ok(Response::new(answ)) + } + +} + +impl BufferService { + pub fn new() -> BufferService { + BufferService { + map: Arc::new(RwLock::new(HashMap::new().into())), + } + } + + pub fn server(self) -> BufferServer { + BufferServer::new(self) + } +} diff --git a/src/server/main.rs b/src/server/main.rs index e74133d..377e346 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -4,19 +4,13 @@ //! all clients and synching everyone's cursor. //! -pub mod actor; -pub mod service; - -use std::sync::Arc; +mod buffer; use tracing::info; use tonic::transport::Server; -use crate::{ - actor::state::StateManager, - service::{buffer::BufferService, workspace::WorkspaceService, session::SessionService}, -}; +use crate::buffer::service::BufferService; #[tokio::main] async fn main() -> Result<(), Box> { @@ -24,14 +18,10 @@ async fn main() -> Result<(), Box> { let addr = "[::1]:50051".parse()?; - let state = Arc::new(StateManager::new()); - info!("Starting server"); Server::builder() - .add_service(SessionService::new(state.clone()).server()) - .add_service(WorkspaceService::new(state.clone()).server()) - .add_service(BufferService::new(state.clone()).server()) + .add_service(BufferService::new().server()) .serve(addr) .await?; diff --git a/src/server/service/buffer.rs b/src/server/service/buffer.rs deleted file mode 100644 index 4d6f7a4..0000000 --- a/src/server/service/buffer.rs +++ /dev/null @@ -1,156 +0,0 @@ -use std::collections::VecDeque; -use std::{pin::Pin, sync::Arc}; - -use uuid::Uuid; - -use tokio_stream::wrappers::ReceiverStream; -use tracing::error; - -use operational_transform::OperationSeq; -use tonic::{Request, Response, Status}; - -pub mod proto { - tonic::include_proto!("buffer"); -} - -use library::events::Event; - -use tokio::sync::{broadcast, mpsc}; -use tokio_stream::{Stream, StreamExt}; // TODO example used this? - -use proto::buffer_server::{Buffer, BufferServer}; -use proto::Operation; - -use tonic::Streaming; -//use futures::{Stream, StreamExt}; - -use crate::actor::{buffer::BufferView, state::StateManager}; - -use self::proto::{BufferPayload, BufferResponse}; // TODO fuck x2! - -type OperationStream = Pin> + Send>>; - -pub struct BufferService { - state: Arc, -} - -fn op_seq(_o: &Operation) -> OperationSeq { - todo!() -} -fn _op_net(_o: &OperationSeq) -> Operation { - todo!() -} - -// async fn buffer_worker(tx: mpsc::Sender>, mut rx:Streaming, mut rx_core: mpsc::Receiver) { -async fn buffer_worker( - bv: BufferView, - mut client_rx: Streaming, - _tx_client: mpsc::Sender>, - mut rx_core: broadcast::Receiver, -) { - let mut queue: VecDeque = VecDeque::new(); - loop { - tokio::select! { - client_op = client_rx.next() => { - if let Some(result) = client_op { - match result { - Ok(op) => { - bv.op(op_seq(&op)).await.unwrap(); // TODO make OpSeq from network Operation pkt! - queue.push_back(op); - }, - Err(status) => { - error!("error receiving op from client: {:?}", status); - break; - } - } - } - }, - - server_op = rx_core.recv() => { - if let Ok(_oop) = server_op { - let mut send_op = true; - for (i, _op) in queue.iter().enumerate() { - if true { // TODO must compare underlying OperationSeq here! (op.equals(server_op)) - queue.remove(i); - send_op = false; - break; - } else { - // serv_op.transform(op); // TODO transform OpSeq ! - } - } - if send_op { - // tx_client.send(Ok(op_net(&oop.1))).await.unwrap(); - } - } - } - } - } -} - -#[tonic::async_trait] -impl Buffer for BufferService { - // type ServerStreamingEchoStream = ResponseStream; - type AttachStream = OperationStream; - - async fn attach( - &self, - req: Request>, - ) -> Result, Status> { - let session_id: String; - if let Some(sid) = req.metadata().get("session_id") { - session_id = sid.to_str().unwrap().to_string(); - } else { - return Err(Status::failed_precondition( - "Missing metadata key 'session_id'", - )); - } - - let path: String; - if let Some(p) = req.metadata().get("path") { - path = p.to_str().unwrap().to_string(); - } else { - return Err(Status::failed_precondition("Missing metadata key 'path'")); - } - // TODO make these above nicer? more concise? idk - - if let Some(workspace) = self.state.workspaces.borrow().get(&Uuid::parse_str(session_id.as_str()).unwrap()) { - let in_stream = req.into_inner(); - let (tx_og, rx) = mpsc::channel::>(128); - - let b: BufferView = workspace.buffers.borrow().get(&path).unwrap().clone(); - let w = workspace.clone(); - tokio::spawn(async move { - buffer_worker(b, in_stream, tx_og, w.bus.subscribe()).await; - }); - - // echo just write the same data that was received - let out_stream = ReceiverStream::new(rx); - - return Ok(Response::new(Box::pin(out_stream) as Self::AttachStream)); - } else { - return Err(Status::not_found(format!( - "Norkspace with session_id {}", - session_id - ))); - } - } - - async fn push(&self, _req:Request) -> Result, Status> { - todo!() - } - - async fn pull(&self, _req:Request) -> Result, Status> { - todo!() - } - -} - -impl BufferService { - pub fn new(state: Arc) -> BufferService { - BufferService { state } - } - - pub fn server(self) -> BufferServer { - BufferServer::new(self) - } -} diff --git a/src/server/service/mod.rs b/src/server/service/mod.rs deleted file mode 100644 index a43d4c0..0000000 --- a/src/server/service/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod buffer; -pub mod session; -pub mod workspace; diff --git a/src/server/service/session.rs b/src/server/service/session.rs deleted file mode 100644 index c0aad35..0000000 --- a/src/server/service/session.rs +++ /dev/null @@ -1,59 +0,0 @@ -pub mod proto { - tonic::include_proto!("session"); -} - -use std::sync::Arc; - -use proto::{session_server::Session, WorkspaceBuilderRequest, SessionResponse}; -use tonic::{Request, Response, Status}; - - -use crate::actor::{ - state::StateManager, workspace::Workspace, // TODO fuck x2! -}; - -use self::proto::session_server::SessionServer; - -#[derive(Debug)] -pub struct SessionService { - state: Arc, -} - -#[tonic::async_trait] -impl Session for SessionService { - async fn create_workspace( - &self, - _req: Request, - ) -> Result, Status> { - // let name = req.extensions().get::().unwrap().id.clone(); - let w = Workspace::new("im lazy".into()); - let res = SessionResponse { accepted:true, session_key: w.id.to_string() }; - - self.state.view().add(w).await; - Ok(Response::new(res)) - } - - // async fn authenticate( - // &self, - // req: Request, - // ) -> Result, Status> { - // todo!() - // } - - // async fn list_workspaces( - // &self, - // req: Request, - // ) -> Result, Status> { - // todo!() - // } -} - -impl SessionService { - pub fn new(state: Arc) -> SessionService { - SessionService { state } - } - - pub fn server(self) -> SessionServer { - SessionServer::new(self) - } -} diff --git a/src/server/service/workspace.rs b/src/server/service/workspace.rs deleted file mode 100644 index b573bdf..0000000 --- a/src/server/service/workspace.rs +++ /dev/null @@ -1,258 +0,0 @@ -use std::{pin::Pin, sync::Arc}; - -use uuid::Uuid; -use tonic::codegen::InterceptedService; -use tonic::service::Interceptor; -use tracing::info; - -use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status, Streaming}; -use tokio::sync::{watch, mpsc}; - -pub mod proto { - tonic::include_proto!("workspace"); -} - -use library::user::User; - -use tokio_stream::{Stream, StreamExt}; // TODO example used this? - -use proto::workspace_server::{Workspace, WorkspaceServer}; -use proto::{BufferList, WorkspaceEvent, WorkspaceRequest, WorkspaceResponse, UsersList, BufferRequest, CursorUpdate, JoinRequest}; - -use library::user::UserCursor; -use crate::actor::{buffer::Buffer, state::StateManager}; // TODO fuck x2! - -pub struct WorkspaceExtension { - pub id: String -} - -#[derive(Debug, Clone)] -pub struct WorkspaceInterceptor { - state: Arc, -} - -impl Interceptor for WorkspaceInterceptor { - fn call(&mut self, mut req: tonic::Request<()>) -> Result, Status> { - // Set an extension that can be retrieved by `say_hello` - let id; - - // TODO this is kinda spaghetti but I can't borrow immutably and mutably req inside this match - // tree... - match req.metadata().get("workspace") { - Some(value) => { - info!("Metadata: {:?}", value); - match value.to_str() { - Ok(w_id) => { - id = w_id.to_string(); - }, - Err(_) => return Err(Status::invalid_argument("Workspace key is not valid")), - } - }, - None => return Err(Status::unauthenticated("No workspace key included in request")) - } - - info!("checking request : {}", id); - - let uid = match Uuid::parse_str(id.as_str()) { - Ok(id) => id, - Err(e) => { return Err(Status::invalid_argument(format!("Invalid uuid : {}", e))); }, - }; - - if !self.state.workspaces.borrow().contains_key(&uid) { - return Err(Status::not_found(format!("Workspace '{}' could not be found", id))); - } - - req.extensions_mut().insert(WorkspaceExtension { id }); - Ok(req) - } -} - - -type EventStream = Pin> + Send>>; -type CursorUpdateStream = Pin> + Send>>; - -#[derive(Debug)] -pub struct WorkspaceService { - state: Arc, -} - -#[tonic::async_trait] -impl Workspace for WorkspaceService { - type JoinStream = EventStream; - type SubscribeStream = CursorUpdateStream; - - async fn join( - &self, - req: Request, - ) -> Result, Status> { - let session_id = Uuid::parse_str(req.extensions().get::().unwrap().id.as_str()).unwrap(); - let r = req.into_inner(); - let run = self.state.run.clone(); - let user_name = r.name.clone(); - match self.state.get(&session_id) { - Some(w) => { - let (tx, rx) = mpsc::channel::>(128); - tokio::spawn(async move { - let mut event_receiver = w.bus.subscribe(); - w.view().users.add( - User { - name: r.name.clone(), - cursor: UserCursor { buffer:0, x:0, y:0 } - } - ).await; - info!("User {} joined workspace {}", r.name, w.id); - while run.borrow().to_owned() { - let res = event_receiver.recv().await.unwrap(); - let broadcasting = WorkspaceEvent { id: 1, body: Some(res.to_string()) }; // TODO actually process packet - tx.send(Ok(broadcasting)).await.unwrap(); - } - w.view().users.remove(user_name).await; - }); - return Ok(Response::new(Box::pin(ReceiverStream::new(rx)))); - }, - None => Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - session_id - ))) - } - } - - async fn subscribe( - &self, - req: tonic::Request>, - ) -> Result, Status> { - let s_id = Uuid::parse_str(req.extensions().get::().unwrap().id.as_str()).unwrap(); - let mut r = req.into_inner(); - match self.state.get(&s_id) { - Some(w) => { - let cursors_ref = w.cursors.clone(); - let (_stop_tx, stop_rx) = watch::channel(true); - let (tx, rx) = mpsc::channel::>(128); - tokio::spawn(async move { - let mut workspace_bus = cursors_ref.subscribe(); - while stop_rx.borrow().to_owned() { - tokio::select!{ - remote = workspace_bus.recv() => { - if let Ok(cur) = remote { - info!("Sending cursor update : {:?}", cur); - tx.send(Ok(cur)).await.unwrap(); - } - }, - local = r.next() => { - match local { - Some(request) => { - info!("Received cursor update : {:?}", request); - match request { - Ok(cur) => { - cursors_ref.send(cur).unwrap(); - }, - Err(_e) => {}, - } - }, - None => {}, - } - }, - } - } - }); - return Ok(Response::new(Box::pin(ReceiverStream::new(rx)))); - }, - None => Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - s_id - ))) - } - } - - async fn buffers( - &self, - req: Request, - ) -> Result, Status> { - let r = req.into_inner(); - match self.state.get(&Uuid::parse_str(r.session_key.as_str()).unwrap()) { - Some(w) => { - let mut out = Vec::new(); - for (_k, v) in w.buffers.borrow().iter() { - out.push(v.name.clone()); - } - Ok(Response::new(BufferList { path: out })) - } - None => Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - r.session_key - ))), - } - } - - async fn new_buffer( - &self, - req: Request, - ) -> Result, Status> { - let session_id = req.extensions().get::().unwrap().id.clone(); - let r = req.into_inner(); - if let Some(w) = self.state.get(&Uuid::parse_str(session_id.as_str()).unwrap()) { - let mut view = w.view(); - let buf = Buffer::new(r.path, w.bus.clone()); - view.buffers.add(buf).await; - - Ok(Response::new(WorkspaceResponse { accepted: true })) - } else { - return Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - r.session_key - ))); - } - } - - async fn remove_buffer( - &self, - req: Request, - ) -> Result, Status> { - let session_id = req.extensions().get::().unwrap().id.clone(); - let r = req.into_inner(); - match self.state.get(&Uuid::parse_str(session_id.as_str()).unwrap()) { - Some(w) => { - w.view().buffers.remove(r.path).await; - Ok(Response::new(WorkspaceResponse { accepted: true })) - } - None => Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - r.session_key - ))), - } - } - - async fn list_users( - &self, - req: Request, - ) -> Result, Status> { - let session_id = req.extensions().get::().unwrap().id.clone(); - let r = req.into_inner(); - match self.state.get(&Uuid::parse_str(session_id.as_str()).unwrap()) { - Some(w) => { - let mut out = Vec::new(); - for (_k, v) in w.users.borrow().iter() { - out.push(v.name.clone()); - } - Ok(Response::new(UsersList { name: out })) - }, - None => Err(Status::not_found(format!( - "No active workspace with session_key '{}'", - r.session_key - ))), - } - } - -} - -impl WorkspaceService { - pub fn new(state: Arc) -> WorkspaceService { - WorkspaceService { state } - } - - pub fn server(self) -> InterceptedService, WorkspaceInterceptor> { - let state = self.state.clone(); - WorkspaceServer::with_interceptor(self, WorkspaceInterceptor { state }) - } -} From 14e9a1e86eae4615093fcdf4f8a04300b0f5d55d Mon Sep 17 00:00:00 2001 From: alemi Date: Mon, 10 Apr 2023 01:41:22 +0200 Subject: [PATCH 03/29] feat: added Operation Factory struct --- src/lib/lib.rs | 1 + src/lib/opfactory.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 src/lib/opfactory.rs diff --git a/src/lib/lib.rs b/src/lib/lib.rs index b7446bb..97146d3 100644 --- a/src/lib/lib.rs +++ b/src/lib/lib.rs @@ -1,4 +1,5 @@ pub mod proto; +pub mod opfactory; pub use tonic; pub use tokio; diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs new file mode 100644 index 0000000..2743eff --- /dev/null +++ b/src/lib/opfactory.rs @@ -0,0 +1,56 @@ +use operational_transform::{OperationSeq, OTError}; + + +#[derive(Clone)] +pub struct OperationFactory { + content: String, +} + +impl OperationFactory { + pub fn new(init: Option) -> Self { + OperationFactory { content: init.unwrap_or(String::new()) } + } + + pub fn check(&self, txt: &str) -> bool { + self.content == txt + } + + pub fn replace(&mut self, txt: &str) -> OperationSeq { + let out = OperationSeq::default(); + if self.content == txt { + return out; // nothing to do + } + + todo!() + } + + pub fn insert(&mut self, txt: &str, pos: u64) -> Result { + let mut out = OperationSeq::default(); + out.retain(pos); + out.insert(txt); + self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + Ok(out) + } + + pub fn delete(&mut self, pos: u64, count: u64) -> Result { + let mut out = OperationSeq::default(); + out.retain(pos - count); + out.delete(count); + self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + Ok(out) + } + + pub fn cancel(&mut self, pos: u64, count: u64) -> Result { + let mut out = OperationSeq::default(); + out.retain(pos); + out.delete(count); + self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + Ok(out) + } + + pub fn process(&mut self, op: OperationSeq) -> Result<(), OTError> { + self.content = op.apply(&self.content)?; + Ok(()) + } + +} From 9bf12b8bc3d4913b88af972af9bd80e31463c018 Mon Sep 17 00:00:00 2001 From: alemi Date: Mon, 10 Apr 2023 20:24:11 +0200 Subject: [PATCH 04/29] feat: add user to msgs, pass msgs directly --- proto/buffer.proto | 7 +++++-- src/client/cli/main.rs | 2 +- src/server/buffer/actor.rs | 20 +++++++++++++------- src/server/buffer/service.rs | 19 +++++++++---------- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/proto/buffer.proto b/proto/buffer.proto index 671b991..36c26ee 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -9,19 +9,22 @@ service Buffer { message RawOp { string opseq = 1; + string user = 2; } message OperationRequest { string path = 1; string hash = 2; string opseq = 3; + string user = 4; } message BufferPayload { - string path = 2; + string path = 1; + string user = 2; optional string content = 3; } message BufferResponse { - bool accepted = 3; + bool accepted = 1; } diff --git a/src/client/cli/main.rs b/src/client/cli/main.rs index 8bf722d..ad27332 100644 --- a/src/client/cli/main.rs +++ b/src/client/cli/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use library::proto::{buffer_client::BufferClient, BufferPayload}; +use codemp::proto::{buffer_client::BufferClient, BufferPayload}; use tokio_stream::StreamExt; #[derive(Parser, Debug)] diff --git a/src/server/buffer/actor.rs b/src/server/buffer/actor.rs index d5b1d26..0b67c07 100644 --- a/src/server/buffer/actor.rs +++ b/src/server/buffer/actor.rs @@ -1,3 +1,4 @@ +use codemp::proto::{RawOp, OperationRequest}; use tokio::sync::{mpsc, broadcast, watch}; use tracing::error; use md5::Digest; @@ -16,8 +17,8 @@ pub trait BufferStore { #[derive(Clone)] pub struct BufferHandle { - pub edit: mpsc::Sender, - events: broadcast::Sender, + pub edit: mpsc::Sender, + events: broadcast::Sender, pub digest: watch::Receiver, } @@ -47,7 +48,7 @@ impl BufferHandle { } } - pub fn subscribe(&self) -> broadcast::Receiver { + pub fn subscribe(&self) -> broadcast::Receiver { self.events.subscribe() } @@ -55,8 +56,8 @@ impl BufferHandle { struct BufferWorker { content: String, - edits: mpsc::Receiver, - events: broadcast::Sender, + edits: mpsc::Receiver, + events: broadcast::Sender, digest: watch::Sender, } @@ -66,11 +67,16 @@ impl BufferWorker { match self.edits.recv().await { None => break, Some(v) => { - match v.apply(&self.content) { + let op : OperationSeq = serde_json::from_str(&v.opseq).unwrap(); + match op.apply(&self.content) { Ok(res) => { self.content = res; self.digest.send(md5::compute(&self.content)).unwrap(); - if let Err(e) = self.events.send(v) { + let msg = RawOp { + opseq: v.opseq, + user: v.user + }; + if let Err(e) = self.events.send(msg) { error!("could not broadcast OpSeq: {}", e); } }, diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs index 4e8e9cb..ba9adf9 100644 --- a/src/server/buffer/service.rs +++ b/src/server/buffer/service.rs @@ -1,12 +1,11 @@ use std::{pin::Pin, sync::{Arc, RwLock}, collections::HashMap}; -use operational_transform::OperationSeq; use tokio::sync::mpsc; use tonic::{Request, Response, Status}; use tokio_stream::{Stream, wrappers::ReceiverStream}; // TODO example used this? -use library::proto::{buffer_server::{Buffer, BufferServer}, RawOp, BufferPayload, BufferResponse, OperationRequest}; +use codemp::proto::{buffer_server::{Buffer, BufferServer}, RawOp, BufferPayload, BufferResponse, OperationRequest}; use tracing::info; use super::actor::{BufferHandle, BufferStore}; @@ -42,6 +41,7 @@ impl Buffer for BufferService { async fn attach(&self, req: Request) -> Result, Status> { let request = req.into_inner(); + let myself = request.user; match self.map.read().unwrap().get(&request.path) { Some(handle) => { let (tx, rx) = mpsc::channel(128); @@ -50,8 +50,8 @@ impl Buffer for BufferService { loop { match sub.recv().await { Ok(v) => { - let snd = RawOp { opseq: serde_json::to_string(&v).unwrap() }; - tx.send(Ok(snd)).await.unwrap(); + if v.user == myself { continue } + tx.send(Ok(v)).await.unwrap(); // TODO unnecessary channel? } Err(_e) => break, } @@ -69,16 +69,15 @@ impl Buffer for BufferService { let request = req.into_inner(); let tx = match self.map.read().unwrap().get(&request.path) { Some(handle) => { - if format!("{:x}", *handle.digest.borrow()) != request.hash { - return Ok(Response::new(BufferResponse { accepted : false } )); - } + // if format!("{:x}", *handle.digest.borrow()) != request.hash { + // return Ok(Response::new(BufferResponse { accepted : false } )); + // } handle.edit.clone() }, None => return Err(Status::not_found("path not found")), }; - let opseq : OperationSeq = serde_json::from_str(&request.opseq).unwrap(); - tx.send(opseq).await.unwrap(); - info!("sent edit to buffer"); + info!("sending edit to buffer: {}", request.opseq); + tx.send(request).await.unwrap(); Ok(Response::new(BufferResponse { accepted: true })) } From 665b8ea2e085caa2e8ba8d91d0f82ca31526e24e Mon Sep 17 00:00:00 2001 From: alemi Date: Mon, 10 Apr 2023 20:25:47 +0200 Subject: [PATCH 05/29] fix: renamed --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 86b27e7..0c9f819 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" # nvim = [] [lib] -name = "library" +name = "codemp" path = "src/lib/lib.rs" [[bin]] # Bin to run the CodeMP gRPC server From 4f43573aa035bd0752852de92527c7172effeb84 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 06:20:40 +0200 Subject: [PATCH 06/29] feat: basic nvim RPC client --- Cargo.toml | 5 +- src/client/nvim/client.rs | 104 ++++++++++++++++++++++++++++ src/client/nvim/codemp.lua | 24 +++++++ src/client/nvim/main.rs | 137 +++++++++++++++++++++++++++++++------ src/lib/opfactory.rs | 17 +++-- 5 files changed, 258 insertions(+), 29 deletions(-) create mode 100644 src/client/nvim/client.rs create mode 100644 src/client/nvim/codemp.lua diff --git a/Cargo.toml b/Cargo.toml index 0c9f819..98e3c18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,11 +39,12 @@ md5 = "0.7.0" prost = "0.11.8" clap = { version = "4.2.1", features = ["derive"], optional = true } nvim-rs = { version = "0.5", features = ["use_tokio"], optional = true } +uuid = { version = "1.3.1", features = ["v4"] } [build-dependencies] tonic-build = "0.9" [features] -default = [] +default = ["nvim"] cli = ["dep:clap"] -nvim = ["dep:nvim-rs"] +nvim = ["dep:nvim-rs", "dep:clap"] diff --git a/src/client/nvim/client.rs b/src/client/nvim/client.rs new file mode 100644 index 0000000..2fab297 --- /dev/null +++ b/src/client/nvim/client.rs @@ -0,0 +1,104 @@ +use std::sync::{Arc, Mutex}; + +use codemp::{proto::{buffer_client::BufferClient, BufferPayload, RawOp, OperationRequest}, tonic::{transport::Channel, Status, Streaming}, opfactory::OperationFactory}; +use tracing::{error, warn}; +use uuid::Uuid; + +type FactoryHandle = Arc>; + +impl From::> for CodempClient { + fn from(x: BufferClient) -> CodempClient { + CodempClient { + id: Uuid::new_v4(), + client:x, + factory: Arc::new(Mutex::new(OperationFactory::new(None))) + } + } +} + +#[derive(Clone)] +pub struct CodempClient { + id: Uuid, + client: BufferClient, + factory: FactoryHandle, // TODO less jank solution than Arc +} + +impl CodempClient { + pub async fn create(&mut self, path: String, content: Option) -> Result { + Ok( + self.client.create( + BufferPayload { + path, + content, + user: self.id.to_string(), + } + ) + .await? + .into_inner() + .accepted + ) + } + + pub async fn insert(&mut self, path: String, txt: String, pos: u64) -> Result { + let res = { self.factory.lock().unwrap().insert(&txt, pos) }; + match res { + Ok(op) => { + Ok( + self.client.edit( + OperationRequest { + path, + hash: "".into(), + opseq: serde_json::to_string(&op).unwrap(), + user: self.id.to_string(), + } + ) + .await? + .into_inner() + .accepted + ) + }, + Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), + } + } + + pub async fn attach () + Send + 'static>(&mut self, path: String, callback: F) -> Result<(), Status> { + let stream = self.client.attach( + BufferPayload { + path, + content: None, + user: self.id.to_string(), + } + ) + .await? + .into_inner(); + + let factory = self.factory.clone(); + tokio::spawn(async move { Self::worker(stream, factory, callback).await } ); + + Ok(()) + } + + async fn worker ()>(mut stream: Streaming, factory: FactoryHandle, callback: F) { + loop { + match stream.message().await { + Ok(v) => match v { + Some(operation) => { + let op = serde_json::from_str(&operation.opseq).unwrap(); + let res = { factory.lock().unwrap().process(op) }; + match res { + Ok(x) => callback(x), + Err(e) => break error!("desynched: {}", e), + } + } + None => break warn!("stream closed"), + }, + Err(e) => break error!("error receiving change: {}", e), + } + } + } + + pub fn content(&self) -> String { + let factory = self.factory.lock().unwrap(); + factory.content() + } +} diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua new file mode 100644 index 0000000..3b55d74 --- /dev/null +++ b/src/client/nvim/codemp.lua @@ -0,0 +1,24 @@ +local BINARY = "/home/alemi/projects/codemp/target/debug/client-nvim --debug" + +if vim.g.codemp_jobid == nil then + vim.g.codemp_jobid = vim.fn.jobstart(BINARY, { rpc = true }) +end + +local M = {} +M.create = function(path, content) return vim.rpcrequest(vim.g.codemp_jobid, "create", path, content) end +M.insert = function(path, txt, pos) return vim.rpcrequest(vim.g.codemp_jobid, "insert", path, txt, pos) end +M.dump = function() return vim.rpcrequest(vim.g.codemp_jobid, "dump") end +M.attach = function(path) + vim.api.nvim_create_autocmd( + { "InsertCharPre" }, + { + callback = function() + local cursor = vim.api.nvim_win_get_cursor(0) + M.insert(path, vim.v.char, cursor[2]) + end, + } + ) + return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) +end + +return M diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index f71ec8b..51f2331 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -1,19 +1,28 @@ -//! A basic example. Mainly for use in a test, but also shows off some basic -//! functionality. -use std::{env, error::Error, fs}; +mod client; +use std::error::Error; + +use client::CodempClient; +use codemp::proto::buffer_client::BufferClient; use rmpv::Value; + use tokio::io::Stdout; +use clap::Parser; use nvim_rs::{ - compat::tokio::Compat, create::tokio as create, rpc::IntoVal, Handler, Neovim, + compat::tokio::Compat, create::tokio as create, Handler, Neovim, }; use tonic::async_trait; +use tracing::{error, warn, debug}; #[derive(Clone)] struct NeovimHandler { - + client: CodempClient, +} + +fn nullable_optional_str(args: &Vec, index: usize) -> Option { + Some(args.get(index)?.as_str()?.to_string()) } #[async_trait] @@ -28,34 +37,116 @@ impl Handler for NeovimHandler { ) -> Result { match name.as_ref() { "ping" => Ok(Value::from("pong")), - _ => unimplemented!(), + + "dump" => Ok(Value::from(self.client.content())), + + "create" => { + if args.len() < 1 { + return Err(Value::from("no path given")); + } + let path = args.get(0).unwrap().as_str().unwrap().into(); + let content = nullable_optional_str(&args, 1); + let mut c = self.client.clone(); + match c.create(path, content).await { + Ok(r) => match r { + true => Ok(Value::from("accepted")), + false => Err(Value::from("rejected")), + }, + Err(e) => Err(Value::from(format!("could not create buffer: {}", e))), + } + }, + + "insert" => { + if args.len() < 3 { + return Err(Value::from("not enough arguments")); + } + let path = args.get(0).unwrap().as_str().unwrap().into(); + let txt = args.get(1).unwrap().as_str().unwrap().into(); + let pos = args.get(2).unwrap().as_u64().unwrap(); + + let mut c = self.client.clone(); + match c.insert(path, txt, pos).await { + Ok(res) => match res { + true => Ok(Value::from("accepted")), + false => Err(Value::from("rejected")), + }, + Err(e) => Err(Value::from(format!("could not send insert: {}", e))), + } + }, + + "attach" => { + if args.len() < 1 { + return Err(Value::from("no path given")); + } + let path = args.get(0).unwrap().as_str().unwrap().into(); + let buf = nvim.get_current_buf().await.unwrap(); + let mut c = self.client.clone(); + + match c.attach(path, move |x| { + let lines : Vec = x.split("\n").map(|x| x.to_string()).collect(); + let b = buf.clone(); + tokio::spawn(async move { + if let Err(e) = b.set_lines(0, lines.len() as i64, false, lines).await { + error!("could not update buffer: {}", e); + } + }); + }).await { + Ok(()) => Ok(Value::from("spawned worker")), + Err(e) => Err(Value::from(format!("could not attach to stream: {}", e))), + } + }, + + _ => Err(Value::from("unimplemented")), } } async fn handle_notify( &self, - name: String, - args: Vec, - nvim: Neovim>, + _name: String, + _args: Vec, + _nvim: Neovim>, ) { + warn!("notify not handled"); } } +#[derive(Parser, Debug)] +struct CliArgs { + /// server host to connect to + #[arg(long, default_value = "http://[::1]:50051")] + host: String, + + /// show debug level logs + #[arg(long, default_value_t = false)] + debug: bool, +} + + #[tokio::main] -async fn main() { - let handler: NeovimHandler = NeovimHandler {}; +async fn main() -> Result<(), tonic::transport::Error> { + + let args = CliArgs::parse(); + + tracing_subscriber::fmt() + .compact() + .without_time() + .with_ansi(false) + .with_writer(std::io::stderr) + .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) + .init(); + + let client = BufferClient::connect(args.host).await?; + debug!("client connected"); + + let handler: NeovimHandler = NeovimHandler { + client: client.into(), + }; + let (nvim, io_handler) = create::new_parent(handler).await; - let curbuf = nvim.get_current_buf().await.unwrap(); - - let mut envargs = env::args(); - let _ = envargs.next(); - let testfile = envargs.next().unwrap(); - - fs::write(testfile, &format!("{:?}", curbuf.into_val())).unwrap(); // Any error should probably be logged, as stderr is not visible to users. match io_handler.await { - Err(joinerr) => eprintln!("Error joining IO loop: '{}'", joinerr), + Err(joinerr) => error!("Error joining IO loop: '{}'", joinerr), Ok(Err(err)) => { if !err.is_reader_error() { // One last try, since there wasn't an error with writing to the @@ -68,7 +159,7 @@ async fn main() { // maybe retry, but at this point it's probably best // to assume the worst and print a friendly and // supportive message to our users - eprintln!("Well, dang... '{}'", e); + error!("Well, dang... '{}'", e); }); } @@ -76,16 +167,18 @@ async fn main() { // Closed channel usually means neovim quit itself, or this plugin was // told to quit by closing the channel, so it's not always an error // condition. - eprintln!("Error: '{}'", err); + error!("Error: '{}'", err); let mut source = err.source(); while let Some(e) = source { - eprintln!("Caused by: '{}'", e); + error!("Caused by: '{}'", e); source = e.source(); } } } Ok(Ok(())) => {} } + + Ok(()) } diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index 2743eff..b327120 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -1,4 +1,5 @@ use operational_transform::{OperationSeq, OTError}; +use tracing::{debug, info}; #[derive(Clone)] @@ -11,6 +12,11 @@ impl OperationFactory { OperationFactory { content: init.unwrap_or(String::new()) } } + // TODO remove the need for this + pub fn content(&self) -> String { + self.content.clone() + } + pub fn check(&self, txt: &str) -> bool { self.content == txt } @@ -25,10 +31,11 @@ impl OperationFactory { } pub fn insert(&mut self, txt: &str, pos: u64) -> Result { + info!("inserting {} at {}", txt, pos); let mut out = OperationSeq::default(); out.retain(pos); out.insert(txt); - self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } @@ -36,7 +43,7 @@ impl OperationFactory { let mut out = OperationSeq::default(); out.retain(pos - count); out.delete(count); - self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } @@ -44,13 +51,13 @@ impl OperationFactory { let mut out = OperationSeq::default(); out.retain(pos); out.delete(count); - self.content = out.apply(&self.content)?; // TODO does aplying mutate the OpSeq itself? + self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } - pub fn process(&mut self, op: OperationSeq) -> Result<(), OTError> { + pub fn process(&mut self, op: OperationSeq) -> Result { self.content = op.apply(&self.content)?; - Ok(()) + Ok(self.content.clone()) } } From 2472164350af6b83253fffa93d5a86fea45fba4c Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:00:49 +0200 Subject: [PATCH 07/29] chore: removed cli client --- Cargo.toml | 5 -- src/client/cli/main.rs | 46 ----------------- src/client/nvim/client.rs | 104 -------------------------------------- 3 files changed, 155 deletions(-) delete mode 100644 src/client/cli/main.rs delete mode 100644 src/client/nvim/client.rs diff --git a/Cargo.toml b/Cargo.toml index 98e3c18..f19676a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,6 @@ path = "src/lib/lib.rs" name = "server" path = "src/server/main.rs" -[[bin]] -name = "client-cli" -path = "src/client/cli/main.rs" -required-features = ["cli"] - [[bin]] name = "client-nvim" path = "src/client/nvim/main.rs" diff --git a/src/client/cli/main.rs b/src/client/cli/main.rs deleted file mode 100644 index ad27332..0000000 --- a/src/client/cli/main.rs +++ /dev/null @@ -1,46 +0,0 @@ -use clap::Parser; -use codemp::proto::{buffer_client::BufferClient, BufferPayload}; -use tokio_stream::StreamExt; - -#[derive(Parser, Debug)] -struct CliArgs { - /// path of buffer to create - path: String, - - /// initial content for buffer - #[arg(short, long)] - content: Option, - - /// attach instead of creating a new buffer - #[arg(long, default_value_t = false)] - attach: bool, - - /// host to connect to - #[arg(long, default_value = "http://[::1]:50051")] - host: String, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args = CliArgs::parse(); - - let mut client = BufferClient::connect(args.host).await?; - - let request = BufferPayload { - path: args.path, - content: args.content, - }; - - if !args.attach { - client.create(request.clone()).await.unwrap(); - } - - let mut stream = client.attach(request).await.unwrap().into_inner(); - - while let Some(item) = stream.next().await { - println!("> {:?}", item); - } - - Ok(()) -} - diff --git a/src/client/nvim/client.rs b/src/client/nvim/client.rs deleted file mode 100644 index 2fab297..0000000 --- a/src/client/nvim/client.rs +++ /dev/null @@ -1,104 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use codemp::{proto::{buffer_client::BufferClient, BufferPayload, RawOp, OperationRequest}, tonic::{transport::Channel, Status, Streaming}, opfactory::OperationFactory}; -use tracing::{error, warn}; -use uuid::Uuid; - -type FactoryHandle = Arc>; - -impl From::> for CodempClient { - fn from(x: BufferClient) -> CodempClient { - CodempClient { - id: Uuid::new_v4(), - client:x, - factory: Arc::new(Mutex::new(OperationFactory::new(None))) - } - } -} - -#[derive(Clone)] -pub struct CodempClient { - id: Uuid, - client: BufferClient, - factory: FactoryHandle, // TODO less jank solution than Arc -} - -impl CodempClient { - pub async fn create(&mut self, path: String, content: Option) -> Result { - Ok( - self.client.create( - BufferPayload { - path, - content, - user: self.id.to_string(), - } - ) - .await? - .into_inner() - .accepted - ) - } - - pub async fn insert(&mut self, path: String, txt: String, pos: u64) -> Result { - let res = { self.factory.lock().unwrap().insert(&txt, pos) }; - match res { - Ok(op) => { - Ok( - self.client.edit( - OperationRequest { - path, - hash: "".into(), - opseq: serde_json::to_string(&op).unwrap(), - user: self.id.to_string(), - } - ) - .await? - .into_inner() - .accepted - ) - }, - Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), - } - } - - pub async fn attach () + Send + 'static>(&mut self, path: String, callback: F) -> Result<(), Status> { - let stream = self.client.attach( - BufferPayload { - path, - content: None, - user: self.id.to_string(), - } - ) - .await? - .into_inner(); - - let factory = self.factory.clone(); - tokio::spawn(async move { Self::worker(stream, factory, callback).await } ); - - Ok(()) - } - - async fn worker ()>(mut stream: Streaming, factory: FactoryHandle, callback: F) { - loop { - match stream.message().await { - Ok(v) => match v { - Some(operation) => { - let op = serde_json::from_str(&operation.opseq).unwrap(); - let res = { factory.lock().unwrap().process(op) }; - match res { - Ok(x) => callback(x), - Err(e) => break error!("desynched: {}", e), - } - } - None => break warn!("stream closed"), - }, - Err(e) => break error!("error receiving change: {}", e), - } - } - } - - pub fn content(&self) -> String { - let factory = self.factory.lock().unwrap(); - factory.content() - } -} From b12b6dc68fcbd72e1eb1a13f3de813f1053a1340 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:01:11 +0200 Subject: [PATCH 08/29] feat: added reference client in lib --- src/lib/client.rs | 110 +++++++++++++++++++++++++++++++++++++++++++ src/lib/lib.rs | 1 + src/lib/opfactory.rs | 2 +- 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/lib/client.rs diff --git a/src/lib/client.rs b/src/lib/client.rs new file mode 100644 index 0000000..071f178 --- /dev/null +++ b/src/lib/client.rs @@ -0,0 +1,110 @@ +/// TODO better name for this file + +use std::sync::{Arc, Mutex}; +use tracing::{error, warn}; +use uuid::Uuid; + +use crate::{ + opfactory::OperationFactory, + proto::{buffer_client::BufferClient, BufferPayload, OperationRequest, RawOp}, + tonic::{transport::Channel, Status, Streaming}, +}; + +type FactoryHandle = Arc>; + +impl From::> for CodempClient { + fn from(x: BufferClient) -> CodempClient { + CodempClient { + id: Uuid::new_v4(), + client:x, + factory: Arc::new(Mutex::new(OperationFactory::new(None))) + } + } +} + +#[derive(Clone)] +pub struct CodempClient { + id: Uuid, + client: BufferClient, + factory: FactoryHandle, // TODO less jank solution than Arc +} + +impl CodempClient { + pub async fn create(&mut self, path: String, content: Option) -> Result { + Ok( + self.client.create( + BufferPayload { + path, + content, + user: self.id.to_string(), + } + ) + .await? + .into_inner() + .accepted + ) + } + + pub async fn insert(&mut self, path: String, txt: String, pos: u64) -> Result { + let res = { self.factory.lock().unwrap().insert(&txt, pos) }; + match res { + Ok(op) => { + Ok( + self.client.edit( + OperationRequest { + path, + hash: "".into(), + opseq: serde_json::to_string(&op).unwrap(), + user: self.id.to_string(), + } + ) + .await? + .into_inner() + .accepted + ) + }, + Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), + } + } + + pub async fn attach () + Send + 'static>(&mut self, path: String, callback: F) -> Result<(), Status> { + let stream = self.client.attach( + BufferPayload { + path, + content: None, + user: self.id.to_string(), + } + ) + .await? + .into_inner(); + + let factory = self.factory.clone(); + tokio::spawn(async move { Self::worker(stream, factory, callback).await } ); + + Ok(()) + } + + async fn worker ()>(mut stream: Streaming, factory: FactoryHandle, callback: F) { + loop { + match stream.message().await { + Ok(v) => match v { + Some(operation) => { + let op = serde_json::from_str(&operation.opseq).unwrap(); + let res = { factory.lock().unwrap().process(op) }; + match res { + Ok(x) => callback(x), + Err(e) => break error!("desynched: {}", e), + } + } + None => break warn!("stream closed"), + }, + Err(e) => break error!("error receiving change: {}", e), + } + } + } + + pub fn content(&self) -> String { + let factory = self.factory.lock().unwrap(); + factory.content() + } +} diff --git a/src/lib/lib.rs b/src/lib/lib.rs index 97146d3..82b6ce5 100644 --- a/src/lib/lib.rs +++ b/src/lib/lib.rs @@ -1,5 +1,6 @@ pub mod proto; pub mod opfactory; +pub mod client; pub use tonic; pub use tokio; diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index b327120..190c017 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -1,5 +1,5 @@ use operational_transform::{OperationSeq, OTError}; -use tracing::{debug, info}; +use tracing::info; #[derive(Clone)] From f1f65aafdb46d35427fbb904e5f882d4dabd5236 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:01:55 +0200 Subject: [PATCH 09/29] chore: cleanup nvim main --- src/client/nvim/main.rs | 49 +++++------------------------------------ 1 file changed, 6 insertions(+), 43 deletions(-) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 51f2331..715d9ea 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -1,8 +1,4 @@ -mod client; - -use std::error::Error; - -use client::CodempClient; +use codemp::client::CodempClient; use codemp::proto::buffer_client::BufferClient; use rmpv::Value; @@ -10,9 +6,7 @@ use rmpv::Value; use tokio::io::Stdout; use clap::Parser; -use nvim_rs::{ - compat::tokio::Compat, create::tokio as create, Handler, Neovim, -}; +use nvim_rs::{compat::tokio::Compat, create::tokio as create, Handler, Neovim}; use tonic::async_trait; use tracing::{error, warn, debug}; @@ -123,8 +117,7 @@ struct CliArgs { #[tokio::main] -async fn main() -> Result<(), tonic::transport::Error> { - +async fn main() -> Result<(), Box> { let args = CliArgs::parse(); tracing_subscriber::fmt() @@ -144,40 +137,10 @@ async fn main() -> Result<(), tonic::transport::Error> { let (nvim, io_handler) = create::new_parent(handler).await; - // Any error should probably be logged, as stderr is not visible to users. - match io_handler.await { - Err(joinerr) => error!("Error joining IO loop: '{}'", joinerr), - Ok(Err(err)) => { - if !err.is_reader_error() { - // One last try, since there wasn't an error with writing to the - // stream - nvim - .err_writeln(&format!("Error: '{}'", err)) - .await - .unwrap_or_else(|e| { - // We could inspect this error to see what was happening, and - // maybe retry, but at this point it's probably best - // to assume the worst and print a friendly and - // supportive message to our users - error!("Well, dang... '{}'", e); - }); - } + nvim.out_write("[*] codemp loaded").await?; - if !err.is_channel_closed() { - // Closed channel usually means neovim quit itself, or this plugin was - // told to quit by closing the channel, so it's not always an error - // condition. - error!("Error: '{}'", err); - - let mut source = err.source(); - - while let Some(e) = source { - error!("Caused by: '{}'", e); - source = e.source(); - } - } - } - Ok(Ok(())) => {} + if let Err(e) = io_handler.await? { + error!("[!] worker stopped with error: {}", e); } Ok(()) From 9a0311eb38fa081c15ab15d4b82624a92581bd3f Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:02:03 +0200 Subject: [PATCH 10/29] chore: cargo features --- Cargo.toml | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f19676a..54d1bde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ path = "src/lib/lib.rs" [[bin]] # Bin to run the CodeMP gRPC server name = "server" path = "src/server/main.rs" +required-features = ["server"] [[bin]] name = "client-nvim" @@ -21,25 +22,33 @@ path = "src/client/nvim/main.rs" required-features = ["nvim"] [dependencies] +# core tracing = "0.1" -tracing-subscriber = "0.3" tonic = "0.9" -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "full"] } -tokio-stream = "0.1" -rmpv = "1" -serde = "1" -serde_json = "1" -operational-transform = { version = "0.6", features = ["serde"] } -md5 = "0.7.0" prost = "0.11.8" +md5 = "0.7.0" +uuid = { version = "1.3.1", features = ["v4"] } +operational-transform = { version = "0.6", features = ["serde"] } +# can these be optional? +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "full"], optional = false } +tokio-stream = { version = "0.1", optional = false } +serde = { version = "1", optional = false } +serde_json = { version = "1", optional = false } +# runtime +# logs +tracing-subscriber = { version = "0.3", optional = true } +# nvim +rmpv = { version = "1", optional = true } clap = { version = "4.2.1", features = ["derive"], optional = true } nvim-rs = { version = "0.5", features = ["use_tokio"], optional = true } -uuid = { version = "1.3.1", features = ["v4"] } [build-dependencies] tonic-build = "0.9" [features] -default = ["nvim"] -cli = ["dep:clap"] -nvim = ["dep:nvim-rs", "dep:clap"] +default = [] +logs = ["dep:tracing-subscriber"] +# runtime = ["dep:tokio", "dep:tokio-stream"] +# serde = ["dep:serde", "dep:serde_json"] +server = ["logs"] +nvim = ["logs", "dep:nvim-rs", "dep:clap", "dep:rmpv"] From 532de6639fbcb98bf06e24d734e3c418e4829df0 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:02:45 +0200 Subject: [PATCH 11/29] feat: pass back stderr --- src/client/nvim/codemp.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 3b55d74..c6c34e5 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -1,7 +1,13 @@ local BINARY = "/home/alemi/projects/codemp/target/debug/client-nvim --debug" if vim.g.codemp_jobid == nil then - vim.g.codemp_jobid = vim.fn.jobstart(BINARY, { rpc = true }) + vim.g.codemp_jobid = vim.fn.jobstart( + BINARY, + { + rpc = true, + on_stderr = function(_, data, _) print(vim.fn.join(data, "\n")) end, + } + ) end local M = {} From ca4f68c5ec8662bc46198bb3bb87577ad8b9a9ce Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:24:40 +0200 Subject: [PATCH 12/29] feat: added delete fn, handle CR and BS --- src/client/nvim/codemp.lua | 13 ++++++++++++- src/client/nvim/main.rs | 18 ++++++++++++++++++ src/lib/client.rs | 22 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index c6c34e5..111a151 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -1,4 +1,4 @@ -local BINARY = "/home/alemi/projects/codemp/target/debug/client-nvim --debug" +local BINARY = "/home/alemi/source/codemp/target/debug/client-nvim --debug" if vim.g.codemp_jobid == nil then vim.g.codemp_jobid = vim.fn.jobstart( @@ -13,6 +13,7 @@ end local M = {} M.create = function(path, content) return vim.rpcrequest(vim.g.codemp_jobid, "create", path, content) end M.insert = function(path, txt, pos) return vim.rpcrequest(vim.g.codemp_jobid, "insert", path, txt, pos) end +M.delete = function(path, pos, count) return vim.rpcrequest(vim.g.codemp_jobid, "delete", path, pos, count) end M.dump = function() return vim.rpcrequest(vim.g.codemp_jobid, "dump") end M.attach = function(path) vim.api.nvim_create_autocmd( @@ -24,6 +25,16 @@ M.attach = function(path) end, } ) + vim.keymap.set('i', '', function() + local cursor = vim.api.nvim_win_get_cursor(0) + M.delete(path, cursor[2], 1) + return '' + end, {expr = true}) + vim.keymap.set('i', '', function() + local cursor = vim.api.nvim_win_get_cursor(0) + M.insert(path, "\n", cursor[2]) + return '' + end, {expr = true}) return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) end diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 715d9ea..138c808 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -68,6 +68,24 @@ impl Handler for NeovimHandler { } }, + "delete" => { + if args.len() < 3 { + return Err(Value::from("not enough arguments")); + } + let path = args.get(0).unwrap().as_str().unwrap().into(); + let pos = args.get(1).unwrap().as_u64().unwrap(); + let count = args.get(2).unwrap().as_u64().unwrap(); + + let mut c = self.client.clone(); + match c.delete(path, pos, count).await { + Ok(res) => match res { + true => Ok(Value::from("accepted")), + false => Err(Value::from("rejected")), + }, + Err(e) => Err(Value::from(format!("could not send insert: {}", e))), + } + }, + "attach" => { if args.len() < 1 { return Err(Value::from("no path given")); diff --git a/src/lib/client.rs b/src/lib/client.rs index 071f178..5dae970 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -67,6 +67,28 @@ impl CodempClient { } } + pub async fn delete(&mut self, path: String, pos: u64, count: u64) -> Result { + let res = { self.factory.lock().unwrap().delete(pos, count) }; + match res { + Ok(op) => { + Ok( + self.client.edit( + OperationRequest { + path, + hash: "".into(), + opseq: serde_json::to_string(&op).unwrap(), + user: self.id.to_string(), + } + ) + .await? + .into_inner() + .accepted + ) + }, + Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), + } + } + pub async fn attach () + Send + 'static>(&mut self, path: String, callback: F) -> Result<(), Status> { let stream = self.client.attach( BufferPayload { From de153c798cbe12866401fd485cc795e8a37043e9 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 14:24:53 +0200 Subject: [PATCH 13/29] fix: retain ending chars --- src/lib/opfactory.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index 190c017..707c951 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -1,6 +1,4 @@ use operational_transform::{OperationSeq, OTError}; -use tracing::info; - #[derive(Clone)] pub struct OperationFactory { @@ -31,26 +29,31 @@ impl OperationFactory { } pub fn insert(&mut self, txt: &str, pos: u64) -> Result { - info!("inserting {} at {}", txt, pos); let mut out = OperationSeq::default(); + let len = self.content.len() as u64; out.retain(pos); out.insert(txt); + out.retain(len - pos); self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } pub fn delete(&mut self, pos: u64, count: u64) -> Result { let mut out = OperationSeq::default(); + let len = self.content.len() as u64; out.retain(pos - count); out.delete(count); + out.retain(len - pos); self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } pub fn cancel(&mut self, pos: u64, count: u64) -> Result { let mut out = OperationSeq::default(); + let len = self.content.len() as u64; out.retain(pos); out.delete(count); + out.retain(len - (pos+count)); self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? Ok(out) } @@ -59,5 +62,4 @@ impl OperationFactory { self.content = op.apply(&self.content)?; Ok(self.content.clone()) } - } From 228f6a54f03a27dba605a80e346a72e8256bce24 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 17:12:22 +0200 Subject: [PATCH 14/29] fix: catch some edge cases --- src/client/nvim/codemp.lua | 11 ++++++-- src/client/nvim/main.rs | 56 ++++++++++++++++++++++++++++---------- src/lib/client.rs | 14 ++++++---- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 111a151..90fac65 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -21,18 +21,23 @@ M.attach = function(path) { callback = function() local cursor = vim.api.nvim_win_get_cursor(0) - M.insert(path, vim.v.char, cursor[2]) + local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 + M.insert(path, vim.v.char, off) end, } ) vim.keymap.set('i', '', function() local cursor = vim.api.nvim_win_get_cursor(0) - M.delete(path, cursor[2], 1) + local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 + if off > 0 then + M.delete(path, off, 1) + end return '' end, {expr = true}) vim.keymap.set('i', '', function() local cursor = vim.api.nvim_win_get_cursor(0) - M.insert(path, "\n", cursor[2]) + local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 + M.insert(path, "\n", off) return '' end, {expr = true}) return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 138c808..7a8d392 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -1,3 +1,5 @@ +use std::{net::TcpStream, sync::Mutex}; + use codemp::client::CodempClient; use codemp::proto::buffer_client::BufferClient; use rmpv::Value; @@ -19,6 +21,18 @@ fn nullable_optional_str(args: &Vec, index: usize) -> Option { Some(args.get(index)?.as_str()?.to_string()) } +fn default_empty_str(args: &Vec, index: usize) -> String { + nullable_optional_str(args, index).unwrap_or("".into()) +} + +fn nullable_optional_number(args: &Vec, index: usize) -> Option { + Some(args.get(index)?.as_u64()?) +} + +fn default_zero_number(args: &Vec, index: usize) -> u64 { + nullable_optional_number(args, index).unwrap_or(0) +} + #[async_trait] impl Handler for NeovimHandler { type Writer = Compat; @@ -29,6 +43,7 @@ impl Handler for NeovimHandler { args: Vec, nvim: Neovim>, ) -> Result { + debug!("processing '{}' - {:?}", name, args); match name.as_ref() { "ping" => Ok(Value::from("pong")), @@ -38,7 +53,7 @@ impl Handler for NeovimHandler { if args.len() < 1 { return Err(Value::from("no path given")); } - let path = args.get(0).unwrap().as_str().unwrap().into(); + let path = default_empty_str(&args, 0); let content = nullable_optional_str(&args, 1); let mut c = self.client.clone(); match c.create(path, content).await { @@ -54,9 +69,9 @@ impl Handler for NeovimHandler { if args.len() < 3 { return Err(Value::from("not enough arguments")); } - let path = args.get(0).unwrap().as_str().unwrap().into(); - let txt = args.get(1).unwrap().as_str().unwrap().into(); - let pos = args.get(2).unwrap().as_u64().unwrap(); + let path = default_empty_str(&args, 0); + let txt = default_empty_str(&args, 1); + let pos = default_zero_number(&args, 2); let mut c = self.client.clone(); match c.insert(path, txt, pos).await { @@ -72,9 +87,9 @@ impl Handler for NeovimHandler { if args.len() < 3 { return Err(Value::from("not enough arguments")); } - let path = args.get(0).unwrap().as_str().unwrap().into(); - let pos = args.get(1).unwrap().as_u64().unwrap(); - let count = args.get(2).unwrap().as_u64().unwrap(); + let path = default_empty_str(&args, 0); + let pos = default_zero_number(&args, 1); + let count = default_zero_number(&args, 2); let mut c = self.client.clone(); match c.delete(path, pos, count).await { @@ -90,8 +105,12 @@ impl Handler for NeovimHandler { if args.len() < 1 { return Err(Value::from("no path given")); } - let path = args.get(0).unwrap().as_str().unwrap().into(); - let buf = nvim.get_current_buf().await.unwrap(); + let path = default_empty_str(&args, 0); + let buf = match nvim.get_current_buf().await { + Ok(b) => b, + Err(e) => return Err(Value::from(format!("could not get current buffer: {}", e))), + }; + let mut c = self.client.clone(); match c.attach(path, move |x| { @@ -138,13 +157,22 @@ struct CliArgs { async fn main() -> Result<(), Box> { let args = CliArgs::parse(); - tracing_subscriber::fmt() + let sub = tracing_subscriber::fmt() .compact() .without_time() - .with_ansi(false) - .with_writer(std::io::stderr) - .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) - .init(); + .with_ansi(false); + match TcpStream::connect("127.0.0.1:6969") { + Ok(stream) => { + sub.with_writer(Mutex::new(stream)) + .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) + .init(); + }, + Err(_) => { + sub.with_writer(std::io::stderr) + .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) + .init(); + }, + } let client = BufferClient::connect(args.host).await?; debug!("client connected"); diff --git a/src/lib/client.rs b/src/lib/client.rs index 5dae970..89b499e 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -111,11 +111,15 @@ impl CodempClient { match stream.message().await { Ok(v) => match v { Some(operation) => { - let op = serde_json::from_str(&operation.opseq).unwrap(); - let res = { factory.lock().unwrap().process(op) }; - match res { - Ok(x) => callback(x), - Err(e) => break error!("desynched: {}", e), + match serde_json::from_str(&operation.opseq) { + Ok(op) => { + let res = { factory.lock().unwrap().process(op) }; + match res { + Ok(x) => callback(x), + Err(e) => break error!("desynched: {}", e), + } + }, + Err(e) => break error!("could not deserialize opseq: {}", e), } } None => break warn!("stream closed"), From b891c0d2f082a8e99dac6c2b8168334301dd696b Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 22:35:10 +0200 Subject: [PATCH 15/29] feat: added sync rpc --- proto/buffer.proto | 2 ++ src/client/nvim/main.rs | 52 +++++++++++++++++++++++++++--------- src/server/buffer/actor.rs | 19 ++++++++----- src/server/buffer/service.rs | 17 +++++++++--- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/proto/buffer.proto b/proto/buffer.proto index 36c26ee..3a322f5 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -5,6 +5,7 @@ service Buffer { rpc Attach (BufferPayload) returns (stream RawOp); rpc Edit (OperationRequest) returns (BufferResponse); rpc Create (BufferPayload) returns (BufferResponse); + rpc Sync (BufferPayload) returns (BufferResponse); } message RawOp { @@ -27,4 +28,5 @@ message BufferPayload { message BufferResponse { bool accepted = 1; + optional string content = 2; } diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 7a8d392..1b4e10b 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -10,7 +10,7 @@ use clap::Parser; use nvim_rs::{compat::tokio::Compat, create::tokio as create, Handler, Neovim}; use tonic::async_trait; -use tracing::{error, warn, debug}; +use tracing::{error, warn, debug, info}; #[derive(Clone)] struct NeovimHandler { @@ -47,7 +47,7 @@ impl Handler for NeovimHandler { match name.as_ref() { "ping" => Ok(Value::from("pong")), - "dump" => Ok(Value::from(self.client.content())), + "error" => Err(Value::from("user-requested error")), "create" => { if args.len() < 1 { @@ -72,12 +72,15 @@ impl Handler for NeovimHandler { let path = default_empty_str(&args, 0); let txt = default_empty_str(&args, 1); let pos = default_zero_number(&args, 2); - let mut c = self.client.clone(); + info!("correctly parsed arguments: {} - {} - {}", path, txt, pos); match c.insert(path, txt, pos).await { - Ok(res) => match res { - true => Ok(Value::from("accepted")), - false => Err(Value::from("rejected")), + Ok(res) => { + info!("RPC 'insert' completed"); + match res { + true => Ok(Value::from("accepted")), + false => Err(Value::from("rejected")), + } }, Err(e) => Err(Value::from(format!("could not send insert: {}", e))), } @@ -101,6 +104,28 @@ impl Handler for NeovimHandler { } }, + "sync" => { + if args.len() < 1 { + return Err(Value::from("no path given")); + } + let path = default_empty_str(&args, 0); + + let mut c = self.client.clone(); + match c.sync(path).await { + Err(e) => Err(Value::from(format!("could not sync: {}", e))), + Ok(content) => match nvim.get_current_buf().await { + Err(e) => return Err(Value::from(format!("could not get current buffer: {}", e))), + Ok(b) => { + let lines : Vec = content.split("\n").map(|x| x.to_string()).collect(); + match b.set_lines(0, -1, false, lines).await { + Err(e) => Err(Value::from(format!("failed sync: {}", e))), + Ok(()) => Ok(Value::from("synched")), + } + }, + }, + } + } + "attach" => { if args.len() < 1 { return Err(Value::from("no path given")); @@ -117,7 +142,7 @@ impl Handler for NeovimHandler { let lines : Vec = x.split("\n").map(|x| x.to_string()).collect(); let b = buf.clone(); tokio::spawn(async move { - if let Err(e) = b.set_lines(0, lines.len() as i64, false, lines).await { + if let Err(e) = b.set_lines(0, -1, false, lines).await { error!("could not update buffer: {}", e); } }); @@ -157,18 +182,19 @@ struct CliArgs { async fn main() -> Result<(), Box> { let args = CliArgs::parse(); - let sub = tracing_subscriber::fmt() - .compact() - .without_time() - .with_ansi(false); - match TcpStream::connect("127.0.0.1:6969") { + let sub = tracing_subscriber::fmt(); + match TcpStream::connect("127.0.0.1:6969") { // TODO get rid of this Ok(stream) => { sub.with_writer(Mutex::new(stream)) .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) .init(); }, Err(_) => { - sub.with_writer(std::io::stderr) + sub + .compact() + .without_time() + .with_ansi(false) + .with_writer(std::io::stderr) .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) .init(); }, diff --git a/src/server/buffer/actor.rs b/src/server/buffer/actor.rs index 0b67c07..ddbc1fb 100644 --- a/src/server/buffer/actor.rs +++ b/src/server/buffer/actor.rs @@ -20,6 +20,7 @@ pub struct BufferHandle { pub edit: mpsc::Sender, events: broadcast::Sender, pub digest: watch::Receiver, + pub content: watch::Receiver, } impl BufferHandle { @@ -28,15 +29,17 @@ impl BufferHandle { let (edits_tx, edits_rx) = mpsc::channel(64); // TODO hardcoded size let (events_tx, _events_rx) = broadcast::channel(64); // TODO hardcoded size let (digest_tx, digest_rx) = watch::channel(md5::compute(&init_val)); + let (content_tx, content_rx) = watch::channel(init_val.clone()); let events_tx_clone = events_tx.clone(); tokio::spawn(async move { let worker = BufferWorker { - content: init_val, + store: init_val, edits: edits_rx, events: events_tx_clone, digest: digest_tx, + content: content_tx, }; worker.work().await }); @@ -45,20 +48,21 @@ impl BufferHandle { edit: edits_tx, events: events_tx, digest: digest_rx, + content: content_rx, } } pub fn subscribe(&self) -> broadcast::Receiver { self.events.subscribe() - } } struct BufferWorker { - content: String, + store: String, edits: mpsc::Receiver, events: broadcast::Sender, digest: watch::Sender, + content: watch::Sender, } impl BufferWorker { @@ -68,10 +72,11 @@ impl BufferWorker { None => break, Some(v) => { let op : OperationSeq = serde_json::from_str(&v.opseq).unwrap(); - match op.apply(&self.content) { + match op.apply(&self.store) { Ok(res) => { - self.content = res; - self.digest.send(md5::compute(&self.content)).unwrap(); + self.store = res; + self.digest.send(md5::compute(&self.store)).unwrap(); + self.content.send(self.store.clone()).unwrap(); let msg = RawOp { opseq: v.opseq, user: v.user @@ -80,7 +85,7 @@ impl BufferWorker { error!("could not broadcast OpSeq: {}", e); } }, - Err(e) => error!("coult not apply OpSeq '{:?}' on '{}' : {}", v, self.content, e), + Err(e) => error!("coult not apply OpSeq '{:?}' on '{}' : {}", v, self.store, e), } }, } diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs index ba9adf9..a7f894b 100644 --- a/src/server/buffer/service.rs +++ b/src/server/buffer/service.rs @@ -78,17 +78,28 @@ impl Buffer for BufferService { }; info!("sending edit to buffer: {}", request.opseq); tx.send(request).await.unwrap(); - Ok(Response::new(BufferResponse { accepted: true })) + Ok(Response::new(BufferResponse { accepted: true, content: None })) } async fn create(&self, req:Request) -> Result, Status> { let request = req.into_inner(); let _handle = self.map.write().unwrap().handle(request.path, request.content); info!("created new buffer"); - let answ = BufferResponse { accepted: true }; + let answ = BufferResponse { accepted: true, content: None }; Ok(Response::new(answ)) } - + + async fn sync(&self, req: Request) -> Result, Status> { + let request = req.into_inner(); + match self.map.read().unwrap().get(&request.path) { + None => Err(Status::not_found("requested buffer does not exist")), + Some(buf) => { + info!("synching buffer"); + let answ = BufferResponse { accepted: true, content: Some(buf.content.borrow().clone()) }; + Ok(Response::new(answ)) + } + } + } } impl BufferService { From 0a464296cd1a1308ced8baebf99f21edca1c8f3a Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 22:35:27 +0200 Subject: [PATCH 16/29] feat: added Join/Share commands --- src/client/nvim/codemp.lua | 78 +++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 90fac65..d418bae 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -1,4 +1,4 @@ -local BINARY = "/home/alemi/source/codemp/target/debug/client-nvim --debug" +local BINARY = "/home/alemi/projects/codemp/target/debug/client-nvim --debug" if vim.g.codemp_jobid == nil then vim.g.codemp_jobid = vim.fn.jobstart( @@ -14,33 +14,57 @@ local M = {} M.create = function(path, content) return vim.rpcrequest(vim.g.codemp_jobid, "create", path, content) end M.insert = function(path, txt, pos) return vim.rpcrequest(vim.g.codemp_jobid, "insert", path, txt, pos) end M.delete = function(path, pos, count) return vim.rpcrequest(vim.g.codemp_jobid, "delete", path, pos, count) end +M.sync = function(path) return vim.rpcrequest(vim.g.codemp_jobid, "sync", path) end M.dump = function() return vim.rpcrequest(vim.g.codemp_jobid, "dump") end -M.attach = function(path) - vim.api.nvim_create_autocmd( - { "InsertCharPre" }, - { - callback = function() - local cursor = vim.api.nvim_win_get_cursor(0) - local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 - M.insert(path, vim.v.char, off) - end, - } - ) - vim.keymap.set('i', '', function() - local cursor = vim.api.nvim_win_get_cursor(0) - local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 - if off > 0 then - M.delete(path, off, 1) - end - return '' - end, {expr = true}) - vim.keymap.set('i', '', function() - local cursor = vim.api.nvim_win_get_cursor(0) - local off = vim.fn.line2byte(cursor[1]) + cursor[2] - 1 - M.insert(path, "\n", off) - return '' - end, {expr = true}) - return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) +M.attach = function(path) return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) end + +local function cursor_offset() + local cursor = vim.api.nvim_win_get_cursor(0) + return vim.fn.line2byte(cursor[1]) + cursor[2] - 1 end +local function hook_callbacks(path) + vim.api.nvim_create_autocmd( + { "InsertCharPre" }, + { callback = function() M.insert(path, vim.v.char, cursor_offset()) end } + ) + vim.keymap.set('i', '', function() + local off = cursor_offset() + pcall(M.delete, path, off, 1) + return '' + end, {expr = true}) + vim.keymap.set('i', '', function() + pcall(M.cancel, path, cursor_offset(), 1) + return '' + end, {expr = true}) + vim.keymap.set('i', '', function() + pcall(M.insertpath, "\n", cursor_offset()) + return '' + end, {expr = true}) +end + +vim.api.nvim_create_user_command( + 'Share', + function(args) + local path = args.fargs[1] + local bufnr = vim.api.nvim_get_current_buf() + local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + M.create(path, vim.fn.join(lines, "\n")) + hook_callbacks(path) + M.attach(path) + end, + {nargs=1} +) + +vim.api.nvim_create_user_command( + 'Join', + function(args) + local path = args.fargs[1] + M.sync(path) + hook_callbacks(path) + M.attach(path) + end, + {nargs=1} +) + return M From 8e2f41a1c8fac456b650a466c67ed503c83249d8 Mon Sep 17 00:00:00 2001 From: alemi Date: Tue, 11 Apr 2023 22:35:37 +0200 Subject: [PATCH 17/29] chore: made OperationFactory async and mutexless --- src/lib/client.rs | 47 ++++++++-------- src/lib/opfactory.rs | 128 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 30 deletions(-) diff --git a/src/lib/client.rs b/src/lib/client.rs index 89b499e..d022955 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -1,23 +1,21 @@ /// TODO better name for this file -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use tracing::{error, warn}; use uuid::Uuid; use crate::{ - opfactory::OperationFactory, + opfactory::AsyncFactory, proto::{buffer_client::BufferClient, BufferPayload, OperationRequest, RawOp}, tonic::{transport::Channel, Status, Streaming}, }; -type FactoryHandle = Arc>; - impl From::> for CodempClient { fn from(x: BufferClient) -> CodempClient { CodempClient { id: Uuid::new_v4(), client:x, - factory: Arc::new(Mutex::new(OperationFactory::new(None))) + factory: Arc::new(AsyncFactory::new(None)), } } } @@ -26,7 +24,7 @@ impl From::> for CodempClient { pub struct CodempClient { id: Uuid, client: BufferClient, - factory: FactoryHandle, // TODO less jank solution than Arc + factory: Arc, } impl CodempClient { @@ -46,8 +44,7 @@ impl CodempClient { } pub async fn insert(&mut self, path: String, txt: String, pos: u64) -> Result { - let res = { self.factory.lock().unwrap().insert(&txt, pos) }; - match res { + match self.factory.insert(txt, pos).await { Ok(op) => { Ok( self.client.edit( @@ -68,8 +65,7 @@ impl CodempClient { } pub async fn delete(&mut self, path: String, pos: u64, count: u64) -> Result { - let res = { self.factory.lock().unwrap().delete(pos, count) }; - match res { + match self.factory.delete(pos, count).await { Ok(op) => { Ok( self.client.edit( @@ -106,31 +102,32 @@ impl CodempClient { Ok(()) } - async fn worker ()>(mut stream: Streaming, factory: FactoryHandle, callback: F) { + pub async fn sync(&mut self, path: String) -> Result { + let res = self.client.sync( + BufferPayload { + path, content: None, user: self.id.to_string(), + } + ).await?; + Ok(res.into_inner().content.unwrap_or("".into())) + } + + async fn worker ()>(mut stream: Streaming, factory: Arc, callback: F) { loop { match stream.message().await { + Err(e) => break error!("error receiving change: {}", e), Ok(v) => match v { + None => break warn!("stream closed"), Some(operation) => { match serde_json::from_str(&operation.opseq) { - Ok(op) => { - let res = { factory.lock().unwrap().process(op) }; - match res { - Ok(x) => callback(x), - Err(e) => break error!("desynched: {}", e), - } - }, Err(e) => break error!("could not deserialize opseq: {}", e), + Ok(op) => match factory.process(op).await { + Err(e) => break error!("desynched: {}", e), + Ok(x) => callback(x), + }, } } - None => break warn!("stream closed"), }, - Err(e) => break error!("error receiving change: {}", e), } } } - - pub fn content(&self) -> String { - let factory = self.factory.lock().unwrap(); - factory.content() - } } diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index 707c951..8c92a8b 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -1,4 +1,6 @@ use operational_transform::{OperationSeq, OTError}; +use tokio::sync::{mpsc, watch, oneshot}; +use tracing::error; #[derive(Clone)] pub struct OperationFactory { @@ -30,11 +32,11 @@ impl OperationFactory { pub fn insert(&mut self, txt: &str, pos: u64) -> Result { let mut out = OperationSeq::default(); - let len = self.content.len() as u64; + let total = self.content.len() as u64; out.retain(pos); out.insert(txt); - out.retain(len - pos); - self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? + out.retain(total - pos); + self.content = out.apply(&self.content)?; Ok(out) } @@ -44,7 +46,7 @@ impl OperationFactory { out.retain(pos - count); out.delete(count); out.retain(len - pos); - self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? + self.content = out.apply(&self.content)?; Ok(out) } @@ -54,7 +56,7 @@ impl OperationFactory { out.retain(pos); out.delete(count); out.retain(len - (pos+count)); - self.content = out.apply(&self.content)?; // TODO does applying mutate the OpSeq itself? + self.content = out.apply(&self.content)?; Ok(out) } @@ -63,3 +65,119 @@ impl OperationFactory { Ok(self.content.clone()) } } + + +pub struct AsyncFactory { + run: watch::Sender, + ops: mpsc::Sender, + content: watch::Receiver, +} + +impl Drop for AsyncFactory { + fn drop(&mut self) { + self.run.send(false).unwrap_or(()); + } +} + +impl AsyncFactory { + pub fn new(init: Option) -> Self { + let (run_tx, run_rx) = watch::channel(true); + let (ops_tx, ops_rx) = mpsc::channel(64); // TODO hardcoded size + let (txt_tx, txt_rx) = watch::channel("".into()); + + let worker = AsyncFactoryWorker { + factory: OperationFactory::new(init), + ops: ops_rx, + run: run_rx, + content: txt_tx, + }; + + tokio::spawn(async move { worker.work().await }); + + AsyncFactory { run: run_tx, ops: ops_tx, content: txt_rx } + } + + pub async fn insert(&self, txt: String, pos: u64) -> Result { + let (tx, rx) = oneshot::channel(); + self.ops.send(OpMsg::Exec(OpWrapper::Insert(txt, pos), tx)).await.unwrap(); + rx.await.unwrap() + } + + pub async fn delete(&self, pos: u64, count: u64) -> Result { + let (tx, rx) = oneshot::channel(); + self.ops.send(OpMsg::Exec(OpWrapper::Delete(pos, count), tx)).await.unwrap(); + rx.await.unwrap() + } + + pub async fn cancel(&self, pos: u64, count: u64) -> Result { + let (tx, rx) = oneshot::channel(); + self.ops.send(OpMsg::Exec(OpWrapper::Cancel(pos, count), tx)).await.unwrap(); + rx.await.unwrap() + } + + pub async fn process(&self, opseq: OperationSeq) -> Result { + let (tx, rx) = oneshot::channel(); + self.ops.send(OpMsg::Process(opseq, tx)).await.unwrap(); + rx.await.unwrap() + } +} + + + + + +#[derive(Debug)] +enum OpMsg { + Exec(OpWrapper, oneshot::Sender>), + Process(OperationSeq, oneshot::Sender>), +} + +#[derive(Debug)] +enum OpWrapper { + Insert(String, u64), + Delete(u64, u64), + Cancel(u64, u64), +} + +struct AsyncFactoryWorker { + factory: OperationFactory, + ops: mpsc::Receiver, + run: watch::Receiver, + content: watch::Sender +} + +impl AsyncFactoryWorker { + async fn work(mut self) { + while *self.run.borrow() { + tokio::select! { // periodically check run so that we stop cleanly + + recv = self.ops.recv() => { + match recv { + Some(msg) => { + match msg { + OpMsg::Exec(op, tx) => tx.send(self.exec(op)).unwrap_or(()), + OpMsg::Process(opseq, tx) => tx.send(self.factory.process(opseq)).unwrap_or(()), + } + if let Err(e) = self.content.send(self.factory.content()) { + error!("error updating content: {}", e); + break; + } + }, + None => break, + } + }, + + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => {}, + + }; + } + } + + fn exec(&mut self, op: OpWrapper) -> Result { + match op { + OpWrapper::Insert(txt, pos) => Ok(self.factory.insert(&txt, pos)?), + OpWrapper::Delete(pos, count) => Ok(self.factory.delete(pos, count)?), + OpWrapper::Cancel(pos, count) => Ok(self.factory.cancel(pos, count)?), + } + } +} From 1eec71f3b27dd9c8e0d2a81c9d363edf4ef21030 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 00:31:59 +0200 Subject: [PATCH 18/29] fix: callbacks local to buffer, local bufnr + path --- src/client/nvim/codemp.lua | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index d418bae..3388353 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -14,8 +14,6 @@ local M = {} M.create = function(path, content) return vim.rpcrequest(vim.g.codemp_jobid, "create", path, content) end M.insert = function(path, txt, pos) return vim.rpcrequest(vim.g.codemp_jobid, "insert", path, txt, pos) end M.delete = function(path, pos, count) return vim.rpcrequest(vim.g.codemp_jobid, "delete", path, pos, count) end -M.sync = function(path) return vim.rpcrequest(vim.g.codemp_jobid, "sync", path) end -M.dump = function() return vim.rpcrequest(vim.g.codemp_jobid, "dump") end M.attach = function(path) return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) end local function cursor_offset() @@ -23,24 +21,27 @@ local function cursor_offset() return vim.fn.line2byte(cursor[1]) + cursor[2] - 1 end -local function hook_callbacks(path) +local function hook_callbacks(path, buffer) vim.api.nvim_create_autocmd( { "InsertCharPre" }, - { callback = function() M.insert(path, vim.v.char, cursor_offset()) end } + { + callback = function(_) M.insert(path, vim.v.char, cursor_offset()) end, + buffer = buffer, + } ) vim.keymap.set('i', '', function() local off = cursor_offset() - pcall(M.delete, path, off, 1) + M.delete(path, off, 1) return '' - end, {expr = true}) + end, {expr = true, buffer = buffer}) vim.keymap.set('i', '', function() - pcall(M.cancel, path, cursor_offset(), 1) + M.delete(path, cursor_offset(), 1) return '' - end, {expr = true}) + end, {expr = true, buffer = buffer}) vim.keymap.set('i', '', function() - pcall(M.insertpath, "\n", cursor_offset()) + M.insert(path, "\n", cursor_offset()) return '' - end, {expr = true}) + end, {expr = true, buffer = buffer}) end vim.api.nvim_create_user_command( @@ -50,7 +51,7 @@ vim.api.nvim_create_user_command( local bufnr = vim.api.nvim_get_current_buf() local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) M.create(path, vim.fn.join(lines, "\n")) - hook_callbacks(path) + hook_callbacks(path, bufnr) M.attach(path) end, {nargs=1} @@ -60,8 +61,8 @@ vim.api.nvim_create_user_command( 'Join', function(args) local path = args.fargs[1] - M.sync(path) - hook_callbacks(path) + local bufnr = vim.api.nvim_get_current_buf() + hook_callbacks(path, bufnr) M.attach(path) end, {nargs=1} From 3827ab066d477f3ba4f3139f66866ac8a2feed22 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 00:32:39 +0200 Subject: [PATCH 19/29] fix: one factory per buffer, create on attach --- src/client/nvim/main.rs | 33 +++------- src/lib/client.rs | 143 ++++++++++++++++++++-------------------- 2 files changed, 81 insertions(+), 95 deletions(-) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 1b4e10b..3e16d56 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -104,40 +104,19 @@ impl Handler for NeovimHandler { } }, - "sync" => { - if args.len() < 1 { - return Err(Value::from("no path given")); - } - let path = default_empty_str(&args, 0); - - let mut c = self.client.clone(); - match c.sync(path).await { - Err(e) => Err(Value::from(format!("could not sync: {}", e))), - Ok(content) => match nvim.get_current_buf().await { - Err(e) => return Err(Value::from(format!("could not get current buffer: {}", e))), - Ok(b) => { - let lines : Vec = content.split("\n").map(|x| x.to_string()).collect(); - match b.set_lines(0, -1, false, lines).await { - Err(e) => Err(Value::from(format!("failed sync: {}", e))), - Ok(()) => Ok(Value::from("synched")), - } - }, - }, - } - } - "attach" => { if args.len() < 1 { return Err(Value::from("no path given")); } let path = default_empty_str(&args, 0); - let buf = match nvim.get_current_buf().await { + let buffer = match nvim.get_current_buf().await { Ok(b) => b, Err(e) => return Err(Value::from(format!("could not get current buffer: {}", e))), }; let mut c = self.client.clone(); + let buf = buffer.clone(); match c.attach(path, move |x| { let lines : Vec = x.split("\n").map(|x| x.to_string()).collect(); let b = buf.clone(); @@ -147,8 +126,14 @@ impl Handler for NeovimHandler { } }); }).await { - Ok(()) => Ok(Value::from("spawned worker")), Err(e) => Err(Value::from(format!("could not attach to stream: {}", e))), + Ok(content) => { + let lines : Vec = content.split("\n").map(|x| x.to_string()).collect(); + if let Err(e) = buffer.set_lines(0, -1, false, lines).await { + error!("could not update buffer: {}", e); + } + Ok(Value::from("spawned worker")) + }, } }, diff --git a/src/lib/client.rs b/src/lib/client.rs index d022955..436fe89 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -1,6 +1,6 @@ /// TODO better name for this file -use std::sync::Arc; +use std::{sync::{Arc, RwLock}, collections::BTreeMap}; use tracing::{error, warn}; use uuid::Uuid; @@ -10,12 +10,14 @@ use crate::{ tonic::{transport::Channel, Status, Streaming}, }; +pub type FactoryStore = Arc>>>; + impl From::> for CodempClient { fn from(x: BufferClient) -> CodempClient { CodempClient { id: Uuid::new_v4(), client:x, - factory: Arc::new(AsyncFactory::new(None)), + factories: Arc::new(RwLock::new(BTreeMap::new())), } } } @@ -24,85 +26,85 @@ impl From::> for CodempClient { pub struct CodempClient { id: Uuid, client: BufferClient, - factory: Arc, + factories: FactoryStore, } impl CodempClient { + fn get_factory(&self, path: &String) -> Result, Status> { + match self.factories.read().unwrap().get(path) { + Some(f) => Ok(f.clone()), + None => Err(Status::not_found("no active buffer for given path")), + } + } + + pub fn add_factory(&self, path: String, factory:Arc) { + self.factories.write().unwrap().insert(path, factory); + } + pub async fn create(&mut self, path: String, content: Option) -> Result { - Ok( - self.client.create( - BufferPayload { - path, - content, - user: self.id.to_string(), - } - ) - .await? - .into_inner() - .accepted - ) + let req = BufferPayload { + path: path.clone(), + content: content.clone(), + user: self.id.to_string(), + }; + + let res = self.client.create(req).await?.into_inner(); + + Ok(res.accepted) } pub async fn insert(&mut self, path: String, txt: String, pos: u64) -> Result { - match self.factory.insert(txt, pos).await { - Ok(op) => { - Ok( - self.client.edit( - OperationRequest { - path, - hash: "".into(), - opseq: serde_json::to_string(&op).unwrap(), - user: self.id.to_string(), - } - ) - .await? - .into_inner() - .accepted - ) - }, + let factory = self.get_factory(&path)?; + match factory.insert(txt, pos).await { Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), + Ok(op) => { + let req = OperationRequest { + path, + hash: "".into(), + user: self.id.to_string(), + opseq: serde_json::to_string(&op) + .map_err(|_| Status::invalid_argument("could not serialize opseq"))?, + }; + let res = self.client.edit(req).await?.into_inner(); + Ok(res.accepted) + }, } } pub async fn delete(&mut self, path: String, pos: u64, count: u64) -> Result { - match self.factory.delete(pos, count).await { - Ok(op) => { - Ok( - self.client.edit( - OperationRequest { - path, - hash: "".into(), - opseq: serde_json::to_string(&op).unwrap(), - user: self.id.to_string(), - } - ) - .await? - .into_inner() - .accepted - ) - }, + let factory = self.get_factory(&path)?; + match factory.delete(pos, count).await { Err(e) => Err(Status::internal(format!("invalid operation: {}", e))), + Ok(op) => { + let req = OperationRequest { + path, + hash: "".into(), + user: self.id.to_string(), + opseq: serde_json::to_string(&op) + .map_err(|_| Status::invalid_argument("could not serialize opseq"))?, + }; + let res = self.client.edit(req).await?.into_inner(); + Ok(res.accepted) + }, } } - pub async fn attach () + Send + 'static>(&mut self, path: String, callback: F) -> Result<(), Status> { - let stream = self.client.attach( - BufferPayload { - path, - content: None, - user: self.id.to_string(), - } - ) - .await? - .into_inner(); - - let factory = self.factory.clone(); + pub async fn attach(&mut self, path: String, callback: F) -> Result + where F : Fn(String) -> () + Send + 'static { + let content = self.sync(path.clone()).await?; + let factory = Arc::new(AsyncFactory::new(Some(content.clone()))); + self.add_factory(path.clone(), factory.clone()); + let req = BufferPayload { + path, + content: None, + user: self.id.to_string(), + }; + let stream = self.client.attach(req).await?.into_inner(); tokio::spawn(async move { Self::worker(stream, factory, callback).await } ); - - Ok(()) + Ok(content) } - pub async fn sync(&mut self, path: String) -> Result { + async fn sync(&mut self, path: String) -> Result { let res = self.client.sync( BufferPayload { path, content: None, user: self.id.to_string(), @@ -111,20 +113,19 @@ impl CodempClient { Ok(res.into_inner().content.unwrap_or("".into())) } - async fn worker ()>(mut stream: Streaming, factory: Arc, callback: F) { + async fn worker(mut stream: Streaming, factory: Arc, callback: F) + where F : Fn(String) -> () { loop { match stream.message().await { Err(e) => break error!("error receiving change: {}", e), Ok(v) => match v { None => break warn!("stream closed"), - Some(operation) => { - match serde_json::from_str(&operation.opseq) { - Err(e) => break error!("could not deserialize opseq: {}", e), - Ok(op) => match factory.process(op).await { - Err(e) => break error!("desynched: {}", e), - Ok(x) => callback(x), - }, - } + Some(operation) => match serde_json::from_str(&operation.opseq) { + Err(e) => break error!("could not deserialize opseq: {}", e), + Ok(op) => match factory.process(op).await { + Err(e) => break error!("desynched: {}", e), + Ok(x) => callback(x), + }, } }, } From dd0acdad2f482de38f07dd0002267c572912da79 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 00:32:56 +0200 Subject: [PATCH 20/29] fix: map errors --- src/lib/opfactory.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index 8c92a8b..99ea1d9 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -99,26 +99,26 @@ impl AsyncFactory { pub async fn insert(&self, txt: String, pos: u64) -> Result { let (tx, rx) = oneshot::channel(); - self.ops.send(OpMsg::Exec(OpWrapper::Insert(txt, pos), tx)).await.unwrap(); - rx.await.unwrap() + self.ops.send(OpMsg::Exec(OpWrapper::Insert(txt, pos), tx)).await.map_err(|_| OTError)?; + rx.await.map_err(|_| OTError)? } pub async fn delete(&self, pos: u64, count: u64) -> Result { let (tx, rx) = oneshot::channel(); - self.ops.send(OpMsg::Exec(OpWrapper::Delete(pos, count), tx)).await.unwrap(); - rx.await.unwrap() + self.ops.send(OpMsg::Exec(OpWrapper::Delete(pos, count), tx)).await.map_err(|_| OTError)?; + rx.await.map_err(|_| OTError)? } pub async fn cancel(&self, pos: u64, count: u64) -> Result { let (tx, rx) = oneshot::channel(); - self.ops.send(OpMsg::Exec(OpWrapper::Cancel(pos, count), tx)).await.unwrap(); - rx.await.unwrap() + self.ops.send(OpMsg::Exec(OpWrapper::Cancel(pos, count), tx)).await.map_err(|_| OTError)?; + rx.await.map_err(|_| OTError)? } pub async fn process(&self, opseq: OperationSeq) -> Result { let (tx, rx) = oneshot::channel(); - self.ops.send(OpMsg::Process(opseq, tx)).await.unwrap(); - rx.await.unwrap() + self.ops.send(OpMsg::Process(opseq, tx)).await.map_err(|_| OTError)?; + rx.await.map_err(|_| OTError)? } } From c1b7073e89523b61ccc388e7788d0a4b12037fa5 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 00:33:14 +0200 Subject: [PATCH 21/29] fix: better error handling --- src/server/buffer/actor.rs | 20 ++++++++++++-------- src/server/buffer/service.rs | 6 ++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/server/buffer/actor.rs b/src/server/buffer/actor.rs index ddbc1fb..f65e5f6 100644 --- a/src/server/buffer/actor.rs +++ b/src/server/buffer/actor.rs @@ -1,6 +1,6 @@ use codemp::proto::{RawOp, OperationRequest}; use tokio::sync::{mpsc, broadcast, watch}; -use tracing::error; +use tracing::{error, warn}; use md5::Digest; use operational_transform::OperationSeq; @@ -69,23 +69,27 @@ impl BufferWorker { async fn work(mut self) { loop { match self.edits.recv().await { - None => break, - Some(v) => { - let op : OperationSeq = serde_json::from_str(&v.opseq).unwrap(); - match op.apply(&self.store) { + None => break warn!("channel closed"), + Some(v) => match serde_json::from_str::(&v.opseq) { + Err(e) => break error!("could not deserialize opseq: {}", e), + Ok(op) => match op.apply(&self.store) { + Err(e) => error!("coult not apply OpSeq '{:?}' on '{}' : {}", v, self.store, e), Ok(res) => { self.store = res; - self.digest.send(md5::compute(&self.store)).unwrap(); - self.content.send(self.store.clone()).unwrap(); let msg = RawOp { opseq: v.opseq, user: v.user }; + if let Err(e) = self.digest.send(md5::compute(&self.store)) { + error!("could not update digest: {}", e); + } + if let Err(e) = self.content.send(self.store.clone()) { + error!("could not update content: {}", e); + } if let Err(e) = self.events.send(msg) { error!("could not broadcast OpSeq: {}", e); } }, - Err(e) => error!("coult not apply OpSeq '{:?}' on '{}' : {}", v, self.store, e), } }, } diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs index a7f894b..6d26dc0 100644 --- a/src/server/buffer/service.rs +++ b/src/server/buffer/service.rs @@ -77,8 +77,10 @@ impl Buffer for BufferService { None => return Err(Status::not_found("path not found")), }; info!("sending edit to buffer: {}", request.opseq); - tx.send(request).await.unwrap(); - Ok(Response::new(BufferResponse { accepted: true, content: None })) + match tx.send(request).await { + Ok(()) => Ok(Response::new(BufferResponse { accepted: true, content: None })), + Err(e) => Err(Status::internal(format!("error sending edit to buffer actor: {}", e))), + } } async fn create(&self, req:Request) -> Result, Status> { From 55c4ddb93ab44e434246ac4a69c4b1d17f06a89f Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 01:38:38 +0200 Subject: [PATCH 22/29] feat: improved Lua API for plugin added commands to connect/disconnect, jobid is kept internally, commands to hook/unhook callbacks, refactored stuff a little --- src/client/nvim/codemp.lua | 100 ++++++++++++++++++++++++------------- src/client/nvim/main.rs | 10 ++++ src/lib/client.rs | 8 ++- 3 files changed, 81 insertions(+), 37 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 3388353..63fb064 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -1,51 +1,74 @@ -local BINARY = "/home/alemi/projects/codemp/target/debug/client-nvim --debug" - -if vim.g.codemp_jobid == nil then - vim.g.codemp_jobid = vim.fn.jobstart( - BINARY, - { - rpc = true, - on_stderr = function(_, data, _) print(vim.fn.join(data, "\n")) end, - } - ) -end +local BINARY = vim.g.codemp_binary or "/home/alemi/projects/codemp/target/debug/client-nvim" local M = {} -M.create = function(path, content) return vim.rpcrequest(vim.g.codemp_jobid, "create", path, content) end -M.insert = function(path, txt, pos) return vim.rpcrequest(vim.g.codemp_jobid, "insert", path, txt, pos) end -M.delete = function(path, pos, count) return vim.rpcrequest(vim.g.codemp_jobid, "delete", path, pos, count) end -M.attach = function(path) return vim.rpcrequest(vim.g.codemp_jobid, "attach", path) end +M.jobid = nil +M.create = function(path, content) return vim.rpcrequest(M.jobid, "create", path, content) end +M.insert = function(path, txt, pos) return vim.rpcrequest(M.jobid, "insert", path, txt, pos) end +M.delete = function(path, pos, count) return vim.rpcrequest(M.jobid, "delete", path, pos, count) end +M.attach = function(path) return vim.rpcrequest(M.jobid, "attach", path) end +M.detach = function(path) return vim.rpcrequest(M.jobid, "detach", path) end local function cursor_offset() local cursor = vim.api.nvim_win_get_cursor(0) return vim.fn.line2byte(cursor[1]) + cursor[2] - 1 end +local codemp_autocmds = vim.api.nvim_create_augroup("CodempAuGroup", { clear = true }) + local function hook_callbacks(path, buffer) vim.api.nvim_create_autocmd( { "InsertCharPre" }, { callback = function(_) M.insert(path, vim.v.char, cursor_offset()) end, buffer = buffer, + group = codemp_autocmds, } ) - vim.keymap.set('i', '', function() - local off = cursor_offset() - M.delete(path, off, 1) - return '' - end, {expr = true, buffer = buffer}) - vim.keymap.set('i', '', function() - M.delete(path, cursor_offset(), 1) - return '' - end, {expr = true, buffer = buffer}) - vim.keymap.set('i', '', function() - M.insert(path, "\n", cursor_offset()) - return '' - end, {expr = true, buffer = buffer}) + vim.keymap.set('i', '', function() M.delete(path, cursor_offset(), 1) return '' end, {expr = true, buffer = buffer}) + vim.keymap.set('i', '', function() M.delete(path, cursor_offset() + 1, 1) return '' end, {expr = true, buffer = buffer}) + vim.keymap.set('i', '', function() M.insert(path, "\n", cursor_offset()) return ''end, {expr = true, buffer = buffer}) end -vim.api.nvim_create_user_command( - 'Share', +local function unhook_callbacks(buffer) + vim.api.nvim_clear_autocmds({ group = codemp_autocmds, buffer = buffer }) + vim.keymap.del('i', '', { buffer = buffer }) + vim.keymap.del('i', '', { buffer = buffer }) + vim.keymap.del('i', '', { buffer = buffer }) +end + + +vim.api.nvim_create_user_command('Connect', + function(args) + if M.jobid ~= nil then + print("already connected, disconnect first") + return + end + local bin_args = { BINARY } + if #args.args > 0 then + table.insert(bin_args, "--host") + table.insert(bin_args, args.args[1]) + end + if args.bang then + table.insert(bin_args, "--debug") + end + M.jobid = vim.fn.jobstart( + bin_args, + { + rpc = true, + on_stderr = function(_, data, _) print(vim.fn.join(data, "\n")) end, + } + ) + end, +{ nargs='?', bang=true }) + +vim.api.nvim_create_user_command('Stop', + function(_) + vim.fn.jobstop(M.jobid) + M.jobid = nil + end, +{ bang=true }) + +vim.api.nvim_create_user_command('Share', function(args) local path = args.fargs[1] local bufnr = vim.api.nvim_get_current_buf() @@ -54,18 +77,23 @@ vim.api.nvim_create_user_command( hook_callbacks(path, bufnr) M.attach(path) end, - {nargs=1} -) +{ nargs=1 }) -vim.api.nvim_create_user_command( - 'Join', +vim.api.nvim_create_user_command('Join', function(args) local path = args.fargs[1] local bufnr = vim.api.nvim_get_current_buf() hook_callbacks(path, bufnr) M.attach(path) end, - {nargs=1} -) +{ nargs=1 }) + +vim.api.nvim_create_user_command('Detach', + function(args) + local bufnr = vim.api.nvim_get_current_buf() + unhook_callbacks(bufnr) + M.detach(args.fargs[1]) + end, +{ nargs=1 }) return M diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 3e16d56..4f59aeb 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -137,6 +137,16 @@ impl Handler for NeovimHandler { } }, + "detach" => { + if args.len() < 1 { + return Err(Value::from("no path given")); + } + let path = default_empty_str(&args, 0); + let mut c = self.client.clone(); + c.detach(path); + Ok(Value::Nil) + }, + _ => Err(Value::from("unimplemented")), } } diff --git a/src/lib/client.rs b/src/lib/client.rs index 436fe89..fd1b308 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -1,7 +1,7 @@ /// TODO better name for this file use std::{sync::{Arc, RwLock}, collections::BTreeMap}; -use tracing::{error, warn}; +use tracing::{error, warn, info}; use uuid::Uuid; use crate::{ @@ -104,6 +104,10 @@ impl CodempClient { Ok(content) } + pub fn detach(&mut self, path: String) { + self.factories.write().unwrap().remove(&path); + } + async fn sync(&mut self, path: String) -> Result { let res = self.client.sync( BufferPayload { @@ -115,6 +119,7 @@ impl CodempClient { async fn worker(mut stream: Streaming, factory: Arc, callback: F) where F : Fn(String) -> () { + info!("|> buffer worker started"); loop { match stream.message().await { Err(e) => break error!("error receiving change: {}", e), @@ -130,5 +135,6 @@ impl CodempClient { }, } } + info!("[] buffer worker stopped"); } } From e471a6dbc907e9b05656c3d4585522b4dbe059f9 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 01:38:47 +0200 Subject: [PATCH 23/29] chore: return nil --- src/client/nvim/main.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 4f59aeb..ec649a1 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -47,8 +47,6 @@ impl Handler for NeovimHandler { match name.as_ref() { "ping" => Ok(Value::from("pong")), - "error" => Err(Value::from("user-requested error")), - "create" => { if args.len() < 1 { return Err(Value::from("no path given")); @@ -58,7 +56,7 @@ impl Handler for NeovimHandler { let mut c = self.client.clone(); match c.create(path, content).await { Ok(r) => match r { - true => Ok(Value::from("accepted")), + true => Ok(Value::Nil), false => Err(Value::from("rejected")), }, Err(e) => Err(Value::from(format!("could not create buffer: {}", e))), @@ -78,7 +76,7 @@ impl Handler for NeovimHandler { Ok(res) => { info!("RPC 'insert' completed"); match res { - true => Ok(Value::from("accepted")), + true => Ok(Value::Nil), false => Err(Value::from("rejected")), } }, @@ -97,7 +95,7 @@ impl Handler for NeovimHandler { let mut c = self.client.clone(); match c.delete(path, pos, count).await { Ok(res) => match res { - true => Ok(Value::from("accepted")), + true => Ok(Value::Nil), false => Err(Value::from("rejected")), }, Err(e) => Err(Value::from(format!("could not send insert: {}", e))), @@ -132,7 +130,7 @@ impl Handler for NeovimHandler { if let Err(e) = buffer.set_lines(0, -1, false, lines).await { error!("could not update buffer: {}", e); } - Ok(Value::from("spawned worker")) + Ok(Value::Nil) }, } }, From 8ca5128ca9cf4b4a60b114ea36780f74ab9ef414 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 03:29:42 +0200 Subject: [PATCH 24/29] feat: very crude cursor implementation --- proto/buffer.proto | 9 ++++++ src/client/nvim/codemp.lua | 15 +++++++++ src/client/nvim/main.rs | 59 ++++++++++++++++++++++++++++++++---- src/lib/client.rs | 28 ++++++++++++++++- src/lib/opfactory.rs | 4 +-- src/server/buffer/service.rs | 45 +++++++++++++++++++++++++-- 6 files changed, 148 insertions(+), 12 deletions(-) diff --git a/proto/buffer.proto b/proto/buffer.proto index 3a322f5..f6bc495 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -6,6 +6,8 @@ service Buffer { rpc Edit (OperationRequest) returns (BufferResponse); rpc Create (BufferPayload) returns (BufferResponse); rpc Sync (BufferPayload) returns (BufferResponse); + rpc Cursor (CursorMov) returns (BufferResponse); + rpc Listen (BufferPayload) returns (stream CursorMov); } message RawOp { @@ -13,6 +15,13 @@ message RawOp { string user = 2; } +message CursorMov { + string user = 1; + string path = 2; + int64 row = 3; + int64 col = 4; +} + message OperationRequest { string path = 1; string hash = 2; diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 63fb064..405f61d 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -4,8 +4,10 @@ local M = {} M.jobid = nil M.create = function(path, content) return vim.rpcrequest(M.jobid, "create", path, content) end M.insert = function(path, txt, pos) return vim.rpcrequest(M.jobid, "insert", path, txt, pos) end +M.cursor = function(path, row, col) return vim.rpcrequest(M.jobid, "cursor", path, row, col) end M.delete = function(path, pos, count) return vim.rpcrequest(M.jobid, "delete", path, pos, count) end M.attach = function(path) return vim.rpcrequest(M.jobid, "attach", path) end +M.listen = function(path) return vim.rpcrequest(M.jobid, "listen", path) end M.detach = function(path) return vim.rpcrequest(M.jobid, "detach", path) end local function cursor_offset() @@ -24,6 +26,17 @@ local function hook_callbacks(path, buffer) group = codemp_autocmds, } ) + vim.api.nvim_create_autocmd( + { "CursorMoved" }, + { + callback = function(_) + local cursor = vim.api.nvim_win_get_cursor(0) + M.cursor(path, cursor[1], cursor[2]) + end, + buffer = buffer, + group = codemp_autocmds, + } + ) vim.keymap.set('i', '', function() M.delete(path, cursor_offset(), 1) return '' end, {expr = true, buffer = buffer}) vim.keymap.set('i', '', function() M.delete(path, cursor_offset() + 1, 1) return '' end, {expr = true, buffer = buffer}) vim.keymap.set('i', '', function() M.insert(path, "\n", cursor_offset()) return ''end, {expr = true, buffer = buffer}) @@ -76,6 +89,7 @@ vim.api.nvim_create_user_command('Share', M.create(path, vim.fn.join(lines, "\n")) hook_callbacks(path, bufnr) M.attach(path) + M.listen(path) end, { nargs=1 }) @@ -85,6 +99,7 @@ vim.api.nvim_create_user_command('Join', local bufnr = vim.api.nvim_get_current_buf() hook_callbacks(path, bufnr) M.attach(path) + M.listen(path) end, { nargs=1 }) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index ec649a1..8b46e49 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -25,11 +25,11 @@ fn default_empty_str(args: &Vec, index: usize) -> String { nullable_optional_str(args, index).unwrap_or("".into()) } -fn nullable_optional_number(args: &Vec, index: usize) -> Option { - Some(args.get(index)?.as_u64()?) +fn nullable_optional_number(args: &Vec, index: usize) -> Option { + Some(args.get(index)?.as_i64()?) } -fn default_zero_number(args: &Vec, index: usize) -> u64 { +fn default_zero_number(args: &Vec, index: usize) -> i64 { nullable_optional_number(args, index).unwrap_or(0) } @@ -69,7 +69,7 @@ impl Handler for NeovimHandler { } let path = default_empty_str(&args, 0); let txt = default_empty_str(&args, 1); - let pos = default_zero_number(&args, 2); + let pos = default_zero_number(&args, 2) as u64; let mut c = self.client.clone(); info!("correctly parsed arguments: {} - {} - {}", path, txt, pos); match c.insert(path, txt, pos).await { @@ -89,8 +89,8 @@ impl Handler for NeovimHandler { return Err(Value::from("not enough arguments")); } let path = default_empty_str(&args, 0); - let pos = default_zero_number(&args, 1); - let count = default_zero_number(&args, 2); + let pos = default_zero_number(&args, 1) as u64; + let count = default_zero_number(&args, 2) as u64; let mut c = self.client.clone(); match c.delete(path, pos, count).await { @@ -145,6 +145,53 @@ impl Handler for NeovimHandler { Ok(Value::Nil) }, + "listen" => { + if args.len() < 1 { + return Err(Value::from("no path given")); + } + let path = default_empty_str(&args, 0); + let mut c = self.client.clone(); + + let ns = nvim.create_namespace("Cursor").await + .map_err(|e| Value::from(format!("could not create namespace: {}", e)))?; + + let buf = nvim.get_current_buf().await + .map_err(|e| Value::from(format!("could not get current buf: {}", e)))?; + + match c.listen(path, move |cur| { + let _b = buf.clone(); + tokio::spawn(async move { + if let Err(e) = _b.clear_namespace(ns, 0, -1).await { + error!("could not clear previous cursor highlight: {}", e); + } + if let Err(e) = _b.add_highlight( + ns, "ErrorMsg", + cur.row as i64 - 1, cur.col as i64 - 1, cur.col as i64 + ).await { + error!("could not create highlight for cursor: {}", e); + } + }); + }).await { + Ok(()) => Ok(Value::Nil), + Err(e) => Err(Value::from(format!("could not listen cursors: {}", e))), + } + }, + + "cursor" => { + if args.len() < 3 { + return Err(Value::from("not enough args")); + } + let path = default_empty_str(&args, 0); + let row = default_zero_number(&args, 1); + let col = default_zero_number(&args, 2); + + let mut c = self.client.clone(); + match c.cursor(path, row, col).await { + Ok(()) => Ok(Value::Nil), + Err(e) => Err(Value::from(format!("could not send cursor update: {}", e))), + } + }, + _ => Err(Value::from("unimplemented")), } } diff --git a/src/lib/client.rs b/src/lib/client.rs index fd1b308..64e5d07 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -6,7 +6,7 @@ use uuid::Uuid; use crate::{ opfactory::AsyncFactory, - proto::{buffer_client::BufferClient, BufferPayload, OperationRequest, RawOp}, + proto::{buffer_client::BufferClient, BufferPayload, OperationRequest, RawOp, CursorMov}, tonic::{transport::Channel, Status, Streaming}, }; @@ -89,6 +89,32 @@ impl CodempClient { } } + pub async fn cursor(&mut self, path: String, row: i64, col: i64) -> Result<(), Status> { + let req = CursorMov { + path, user: self.id.to_string(), + row, col, + }; + let _res = self.client.cursor(req).await?.into_inner(); + Ok(()) + } + + pub async fn listen(&mut self, path: String, callback: F) -> Result<(), Status> + where F : Fn(CursorMov) -> () + Send + 'static { + let req = BufferPayload { + path, + content: None, + user: self.id.to_string(), + }; + let mut stream = self.client.listen(req).await?.into_inner(); + tokio::spawn(async move { + // TODO catch some errors + while let Ok(Some(x)) = stream.message().await { + callback(x) + } + }); + Ok(()) + } + pub async fn attach(&mut self, path: String, callback: F) -> Result where F : Fn(String) -> () + Send + 'static { let content = self.sync(path.clone()).await?; diff --git a/src/lib/opfactory.rs b/src/lib/opfactory.rs index 99ea1d9..1dd85a1 100644 --- a/src/lib/opfactory.rs +++ b/src/lib/opfactory.rs @@ -70,6 +70,7 @@ impl OperationFactory { pub struct AsyncFactory { run: watch::Sender, ops: mpsc::Sender, + #[allow(unused)] // TODO is this necessary? content: watch::Receiver, } @@ -123,9 +124,6 @@ impl AsyncFactory { } - - - #[derive(Debug)] enum OpMsg { Exec(OpWrapper, oneshot::Sender>), diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs index 6d26dc0..c0da738 100644 --- a/src/server/buffer/service.rs +++ b/src/server/buffer/service.rs @@ -1,16 +1,17 @@ use std::{pin::Pin, sync::{Arc, RwLock}, collections::HashMap}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, broadcast}; use tonic::{Request, Response, Status}; use tokio_stream::{Stream, wrappers::ReceiverStream}; // TODO example used this? -use codemp::proto::{buffer_server::{Buffer, BufferServer}, RawOp, BufferPayload, BufferResponse, OperationRequest}; +use codemp::proto::{buffer_server::{Buffer, BufferServer}, RawOp, BufferPayload, BufferResponse, OperationRequest, CursorMov}; use tracing::info; use super::actor::{BufferHandle, BufferStore}; type OperationStream = Pin> + Send>>; +type CursorStream = Pin> + Send>>; struct BufferMap { store: HashMap, @@ -33,11 +34,22 @@ impl BufferStore for BufferMap { pub struct BufferService { map: Arc>, + cursor: broadcast::Sender, +} + +impl BufferService { + fn get_buffer(&self, path: &String) -> Result { + match self.map.read().unwrap().get(path) { + Some(buf) => Ok(buf.clone()), + None => Err(Status::not_found("no buffer for given path")), + } + } } #[tonic::async_trait] impl Buffer for BufferService { type AttachStream = OperationStream; + type ListenStream = CursorStream; async fn attach(&self, req: Request) -> Result, Status> { let request = req.into_inner(); @@ -65,6 +77,33 @@ impl Buffer for BufferService { } } + async fn listen(&self, req: Request) -> Result, Status> { + let mut sub = self.cursor.subscribe(); + let myself = req.into_inner().user; + let (tx, rx) = mpsc::channel(128); + tokio::spawn(async move { + loop { + match sub.recv().await { + Ok(v) => { + if v.user == myself { continue } + tx.send(Ok(v)).await.unwrap(); // TODO unnecessary channel? + } + Err(_e) => break, + } + } + }); + let output_stream = ReceiverStream::new(rx); + info!("registered new subscriber to cursor updates"); + Ok(Response::new(Box::pin(output_stream))) + } + + async fn cursor(&self, req:Request) -> Result, Status> { + match self.cursor.send(req.into_inner()) { + Ok(_) => Ok(Response::new(BufferResponse { accepted: true, content: None})), + Err(e) => Err(Status::internal(format!("could not broadcast cursor update: {}", e))), + } + } + async fn edit(&self, req:Request) -> Result, Status> { let request = req.into_inner(); let tx = match self.map.read().unwrap().get(&request.path) { @@ -106,8 +145,10 @@ impl Buffer for BufferService { impl BufferService { pub fn new() -> BufferService { + let (cur_tx, _cur_rx) = broadcast::channel(64); // TODO hardcoded capacity BufferService { map: Arc::new(RwLock::new(HashMap::new().into())), + cursor: cur_tx, } } From a52f74d0927d6b70b8c63cf8640a9e588e0f8792 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 04:21:02 +0200 Subject: [PATCH 25/29] fix: move cursor in insert, error if not connected --- src/client/nvim/codemp.lua | 14 ++++++++++++-- src/client/nvim/main.rs | 5 +---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 405f61d..3e75531 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -27,7 +27,7 @@ local function hook_callbacks(path, buffer) } ) vim.api.nvim_create_autocmd( - { "CursorMoved" }, + { "CursorMoved", "CursorMovedI" }, { callback = function(_) local cursor = vim.api.nvim_win_get_cursor(0) @@ -49,7 +49,6 @@ local function unhook_callbacks(buffer) vim.keymap.del('i', '', { buffer = buffer }) end - vim.api.nvim_create_user_command('Connect', function(args) if M.jobid ~= nil then @@ -71,6 +70,9 @@ vim.api.nvim_create_user_command('Connect', on_stderr = function(_, data, _) print(vim.fn.join(data, "\n")) end, } ) + if M.jobid <= 0 then + print("[!] could not start codemp client") + end end, { nargs='?', bang=true }) @@ -83,6 +85,10 @@ vim.api.nvim_create_user_command('Stop', vim.api.nvim_create_user_command('Share', function(args) + if M.jobid <= 0 then + print("[!] connect to codemp server first") + return + end local path = args.fargs[1] local bufnr = vim.api.nvim_get_current_buf() local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) @@ -95,6 +101,10 @@ vim.api.nvim_create_user_command('Share', vim.api.nvim_create_user_command('Join', function(args) + if M.jobid <= 0 then + print("[!] connect to codemp server first") + return + end local path = args.fargs[1] local bufnr = vim.api.nvim_get_current_buf() hook_callbacks(path, bufnr) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 8b46e49..6ec0362 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -164,10 +164,7 @@ impl Handler for NeovimHandler { if let Err(e) = _b.clear_namespace(ns, 0, -1).await { error!("could not clear previous cursor highlight: {}", e); } - if let Err(e) = _b.add_highlight( - ns, "ErrorMsg", - cur.row as i64 - 1, cur.col as i64 - 1, cur.col as i64 - ).await { + if let Err(e) = _b.add_highlight(ns, "ErrorMsg", cur.row-1, cur.col, cur.col+1).await { error!("could not create highlight for cursor: {}", e); } }); From 2fde9659db1fec94745575fa661e9e5d4935311a Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 04:59:33 +0200 Subject: [PATCH 26/29] feat: parse CLI args in server too --- Cargo.toml | 2 +- src/server/buffer/service.rs | 1 + src/server/main.rs | 29 ++++++++++++++++++++++------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 54d1bde..71fef1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,5 +50,5 @@ default = [] logs = ["dep:tracing-subscriber"] # runtime = ["dep:tokio", "dep:tokio-stream"] # serde = ["dep:serde", "dep:serde_json"] -server = ["logs"] +server = ["logs", "dep:clap"] nvim = ["logs", "dep:nvim-rs", "dep:clap", "dep:rmpv"] diff --git a/src/server/buffer/service.rs b/src/server/buffer/service.rs index c0da738..141e96e 100644 --- a/src/server/buffer/service.rs +++ b/src/server/buffer/service.rs @@ -38,6 +38,7 @@ pub struct BufferService { } impl BufferService { + #[allow(unused)] fn get_buffer(&self, path: &String) -> Result { match self.map.read().unwrap().get(path) { Some(buf) => Ok(buf.clone()), diff --git a/src/server/main.rs b/src/server/main.rs index 377e346..8d57893 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -4,25 +4,40 @@ //! all clients and synching everyone's cursor. //! -mod buffer; - +use clap::Parser; use tracing::info; - use tonic::transport::Server; +mod buffer; + use crate::buffer::service::BufferService; +#[derive(Parser, Debug)] +struct CliArgs { + + /// address to listen on + #[arg(long, default_value = "[::1]:50051")] + host: String, + + /// enable debug log level + #[arg(long, default_value_t = false)] + debug: bool, +} + #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt::init(); + let args = CliArgs::parse(); - let addr = "[::1]:50051".parse()?; + tracing_subscriber::fmt() + .with_writer(std::io::stdout) + .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) + .init(); - info!("Starting server"); + info!("starting server"); Server::builder() .add_service(BufferService::new().server()) - .serve(addr) + .serve(args.host.parse()?) .await?; Ok(()) From a872c39d7f0b714c1bfa7b7e7b8d474f9c7dfc69 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 05:00:18 +0200 Subject: [PATCH 27/29] fix: properly print logs in nvim --- src/client/nvim/codemp.lua | 8 +++++++- src/client/nvim/main.rs | 7 +++---- src/lib/client.rs | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/client/nvim/codemp.lua b/src/client/nvim/codemp.lua index 3e75531..6f3f35b 100644 --- a/src/client/nvim/codemp.lua +++ b/src/client/nvim/codemp.lua @@ -67,7 +67,13 @@ vim.api.nvim_create_user_command('Connect', bin_args, { rpc = true, - on_stderr = function(_, data, _) print(vim.fn.join(data, "\n")) end, + on_stderr = function(_, data, _) + for _, line in pairs(data) do + print(line) + end + -- print(vim.fn.join(data, "\n")) + end, + stderr_buffered = false, } ) if M.jobid <= 0 then diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 6ec0362..737053b 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -238,18 +238,17 @@ async fn main() -> Result<(), Box> { } let client = BufferClient::connect(args.host).await?; - debug!("client connected"); let handler: NeovimHandler = NeovimHandler { client: client.into(), }; - let (nvim, io_handler) = create::new_parent(handler).await; + let (_nvim, io_handler) = create::new_parent(handler).await; - nvim.out_write("[*] codemp loaded").await?; + info!("++ codemp started"); if let Err(e) = io_handler.await? { - error!("[!] worker stopped with error: {}", e); + error!("worker stopped with error: {}", e); } Ok(()) diff --git a/src/lib/client.rs b/src/lib/client.rs index 64e5d07..f2ecb1d 100644 --- a/src/lib/client.rs +++ b/src/lib/client.rs @@ -132,6 +132,7 @@ impl CodempClient { pub fn detach(&mut self, path: String) { self.factories.write().unwrap().remove(&path); + info!("|| detached from buffer"); } async fn sync(&mut self, path: String) -> Result { From 77eae35bc306644567bb321a33f2794e2e074946 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 05:00:37 +0200 Subject: [PATCH 28/29] feat: allow to request remote tracing via socket --- src/client/nvim/main.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/client/nvim/main.rs b/src/client/nvim/main.rs index 737053b..5600d91 100644 --- a/src/client/nvim/main.rs +++ b/src/client/nvim/main.rs @@ -212,6 +212,10 @@ struct CliArgs { /// show debug level logs #[arg(long, default_value_t = false)] debug: bool, + + /// dump raw tracing logs into this TCP host + #[arg(long)] + remote_debug: Option, } @@ -219,22 +223,21 @@ struct CliArgs { async fn main() -> Result<(), Box> { let args = CliArgs::parse(); - let sub = tracing_subscriber::fmt(); - match TcpStream::connect("127.0.0.1:6969") { // TODO get rid of this - Ok(stream) => { - sub.with_writer(Mutex::new(stream)) + match args.remote_debug { + Some(host) => + tracing_subscriber::fmt() + .with_writer(Mutex::new(TcpStream::connect(host)?)) .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) - .init(); - }, - Err(_) => { - sub + .init(), + + None => + tracing_subscriber::fmt() .compact() .without_time() .with_ansi(false) .with_writer(std::io::stderr) .with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO }) - .init(); - }, + .init(), } let client = BufferClient::connect(args.host).await?; From 4ea92c46db0dcc22f39d53e4ad096adecb3c103e Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 12 Apr 2023 05:01:14 +0200 Subject: [PATCH 29/29] chore: version bump --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 71fef1a..cda7b1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codemp" -version = "0.1.0" +version = "0.2.0" edition = "2021" # [features]