From 98bab88e014e0fb8a9d689f9536f31316a09f9d9 Mon Sep 17 00:00:00 2001 From: frelodev Date: Sun, 3 Dec 2023 19:54:55 +0100 Subject: [PATCH 01/42] added user proto --- proto/user.proto | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 proto/user.proto diff --git a/proto/user.proto b/proto/user.proto new file mode 100644 index 0000000..1697d5d --- /dev/null +++ b/proto/user.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package user; + + +// payload identifying user +message UserIdentity { + // user identifier + string id = 1; +} + +service User{ + +} \ No newline at end of file From a3ed66521cae547f2e67a4907bf96991863d107d Mon Sep 17 00:00:00 2001 From: frelodev Date: Sun, 3 Dec 2023 19:56:52 +0100 Subject: [PATCH 02/42] updated cursor.proto --- proto/cursor.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/cursor.proto b/proto/cursor.proto index d48d7c5..ec53e17 100644 --- a/proto/cursor.proto +++ b/proto/cursor.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package codemp.cursor; +package cursor; // handle cursor events and broadcast to all users service Cursor { From f687fcb1ad65158c13710599f0510ef527fce75c Mon Sep 17 00:00:00 2001 From: frelodev Date: Sun, 3 Dec 2023 19:58:21 +0100 Subject: [PATCH 03/42] updated buffer proto with buffer metadata and buffer struct --- proto/buffer.proto | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/proto/buffer.proto b/proto/buffer.proto index 23f919f..5e5e719 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -1,6 +1,7 @@ syntax = "proto3"; -package codemp.buffer; +package buffer; +import "proto/user.proto"; // handle buffer changes, keep in sync users service Buffer { @@ -14,6 +15,20 @@ service Buffer { rpc Sync (BufferPayload) returns (BufferResponse); } +message BufferView { + int32 id = 1; + + BufferMetadata metadata = 2; +} + +message BufferMetadata { + string name = 2; + string path = 3; + uint64 created = 4; // unix timestamp + repeated user.UserIdentity attached_users = 5; +} + + // empty request message BufferCreateResponse {} From 18290d768cb49f4176fcb14fd2c5cb8d87a044f7 Mon Sep 17 00:00:00 2001 From: frelodev Date: Sun, 3 Dec 2023 20:16:43 +0100 Subject: [PATCH 04/42] features: create buffers from workspaces, list buffers and users --- proto/workspace.proto | 158 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 proto/workspace.proto diff --git a/proto/workspace.proto b/proto/workspace.proto new file mode 100644 index 0000000..3f0dd42 --- /dev/null +++ b/proto/workspace.proto @@ -0,0 +1,158 @@ +// Workspace effimero: sta in /tmp o proprio in memoria +// Workspace e` autenticato: come si decide mentre si rifa il server +// Workspace ha id univoco (stringa), usato per connettercisi +// Workspace implementera` access control: +// * accedere al workspace +// * i singoli buffer +// - i metadati maybe???? +// Workspace offre le seguenti features: +// * listare i buffer DONE +// * listare gli user connessi DONE +// * creare buffers DONE REPLACE THE ONE ON buffer.proto +// * NO ATTACH: responsabilita` del buffer service +// * contiene metadata dei buffers: +// * path +// * data creazione +// Buffer id NON E` il path DONE +// BufferService NON ha metadata: +// Workspace tiene traccia di utenti attached (nel futuro) DONE + + + + + + +syntax = "proto3"; + +package workspace; +import "proto/user.proto"; +import "proto/buffer.proto"; + +message Empty {} + +message WorkspaceEvent { + oneof event { + CursorEvent cursor = 1; + FileEvent file = 2; + UserEvent user = 3; + } +} + +message WorkspaceFileTree { + // list of strings may be more efficient but it's a lot more hassle + string payload = 1; // spappolata di json +} + +message WorkspaceUserList { + repeated user.UserIdentity user = 1; +} + +message WorkspaceMessage { + int32 id = 1; +} + + +message TreeRequest {} // empty +message UserRequest {} +message CursorResponse{} +message UserListRequest{} + +service Workspace { + + // + rpc Create (BufferPayload) returns (BufferCreateResponse); + + rpc ListBuffers (BufferListRequest) returns (BufferList); + + rpc ListUsers (UserListRequest) returns (UserList); + + // + rpc Join (user.UserIdentity) returns (stream WorkspaceEvent); + + // + rpc Tree (TreeRequest) returns (WorkspaceFileTree); + + // + rpc Users (UserRequest) returns (WorkspaceUserList); // TODO could be handled by cursor service + + // send cursor movement to server + rpc Cursor (CursorEvent) returns (CursorResponse); +} + +// a tuple indicating row and column +message RowCol { + int32 row = 1; + int32 col = 2; +} + +// cursor position object +message CursorPosition { + // cursor start position + RowCol start = 1; + // cursor end position + RowCol end = 2; +} + +// cursor event, with user id and cursor position +message CursorEvent { + // user moving the cursor + string user = 1; + // path of current buffer this cursor is into + string buffer = 2; + // new cursor position + repeated CursorPosition position = 3; +} + +enum FileEventType { + CREATE = 0; + DELETE = 1; + RENAME = 2; +} + +message FileEvent { + string buffer = 1; + + FileEventType type = 2; +} + +enum UserEventType { + JOIN = 0; + LEAVE = 1; +} + +message UserEvent { + user.UserIdentity user = 1; + + UserEventType type = 2; +} + + + +message BufferPayload { + // buffer path to operate onto + string path = 1; + + // user id that is requesting the operation + user.UserIdentity user = 2; + + // optional buffer full content for replacing + optional string content = 3; +} + +message BufferCreateResponse { + + string status = 1; +} + + +message BufferListRequest{ + +} + +message BufferList{ + repeated buffer.BufferView buffers = 1; +} + +message UserList{ + repeated user.UserIdentity users = 1; +} \ No newline at end of file From 2f1bfab1306928c2c0a7aad1bc908a30922ee315 Mon Sep 17 00:00:00 2001 From: frelodev Date: Mon, 18 Dec 2023 23:36:15 +0100 Subject: [PATCH 05/42] new changes to proto --- proto/buffer.proto | 75 ---------------- proto/buffer_service.proto | 15 ++++ proto/cursor.proto | 44 ---------- proto/cursor_service.proto | 13 +++ proto/model/cursor.proto | 32 +++++++ proto/model/files.proto | 0 proto/{ => model}/user.proto | 6 +- proto/model/workspace.proto | 0 proto/workspace.proto | 158 ---------------------------------- proto/workspace_service.proto | 135 +++++++++++++++++++++++++++++ 10 files changed, 198 insertions(+), 280 deletions(-) delete mode 100644 proto/buffer.proto create mode 100644 proto/buffer_service.proto delete mode 100644 proto/cursor.proto create mode 100644 proto/cursor_service.proto create mode 100644 proto/model/cursor.proto create mode 100644 proto/model/files.proto rename proto/{ => model}/user.proto (57%) create mode 100644 proto/model/workspace.proto delete mode 100644 proto/workspace.proto create mode 100644 proto/workspace_service.proto diff --git a/proto/buffer.proto b/proto/buffer.proto deleted file mode 100644 index 5e5e719..0000000 --- a/proto/buffer.proto +++ /dev/null @@ -1,75 +0,0 @@ -syntax = "proto3"; - -package buffer; -import "proto/user.proto"; - -// handle buffer changes, keep in sync users -service Buffer { - // attach to a buffer and receive operations - rpc Attach (BufferPayload) returns (stream RawOp); - // send an operation for a specific buffer - rpc Edit (OperationRequest) returns (BufferEditResponse); - // create a new buffer - rpc Create (BufferPayload) returns (BufferCreateResponse); - // get contents of buffer - rpc Sync (BufferPayload) returns (BufferResponse); -} - -message BufferView { - int32 id = 1; - - BufferMetadata metadata = 2; -} - -message BufferMetadata { - string name = 2; - string path = 3; - uint64 created = 4; // unix timestamp - repeated user.UserIdentity attached_users = 5; -} - - -// empty request -message BufferCreateResponse {} - -// empty request -message BufferEditResponse {} - -// raw wire operation sequence event -message RawOp { - // operation seq serialized to json - string opseq = 1; - - // user id that has executed the operation - string user = 2; -} - -// client buffer operation request -message OperationRequest { - // buffer path to operate onto - string path = 1; - - // buffer hash of source state - string hash = 2; - - // raw operation sequence - RawOp op = 3; -} - -// generic buffer operation request -message BufferPayload { - // buffer path to operate onto - string path = 1; - - // user id that is requesting the operation - string user = 2; - - // optional buffer full content for replacing - optional string content = 3; -} - -// response from server with buffer content -message BufferResponse { - // current buffer content - string content = 1; -} diff --git a/proto/buffer_service.proto b/proto/buffer_service.proto new file mode 100644 index 0000000..6b09fd0 --- /dev/null +++ b/proto/buffer_service.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; + +package codemp.buffer_service; + +// handle buffer changes, keep in sync users +service Buffer { + // attach to a buffer and receive operations + rpc Attach (stream Operation) returns (stream Operation); +} + +message Operation { + required bytes data = 1; + optional string user = 2; + optional string path = 3; +} \ No newline at end of file diff --git a/proto/cursor.proto b/proto/cursor.proto deleted file mode 100644 index ec53e17..0000000 --- a/proto/cursor.proto +++ /dev/null @@ -1,44 +0,0 @@ -syntax = "proto3"; - -package cursor; - -// handle cursor events and broadcast to all users -service Cursor { - // send cursor movement to server - rpc Moved (CursorEvent) returns (MovedResponse); - // attach to a workspace and receive cursor events - rpc Listen (UserIdentity) returns (stream CursorEvent); -} - -// empty request -message MovedResponse {} - -// a tuple indicating row and column -message RowCol { - int32 row = 1; - int32 col = 2; -} - -// cursor position object -message CursorPosition { - // path of current buffer this cursor is into - string buffer = 1; - // cursor start position - RowCol start = 2; - // cursor end position - RowCol end = 3; -} - -// cursor event, with user id and cursor position -message CursorEvent { - // user moving the cursor - string user = 1; - // new cursor position - CursorPosition position = 2; -} - -// payload identifying user for cursor attaching -message UserIdentity { - // user identifier - string id = 1; -} diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto new file mode 100644 index 0000000..b0319a8 --- /dev/null +++ b/proto/cursor_service.proto @@ -0,0 +1,13 @@ +syntax = "proto2"; + +package codemp.cursor; +import "proto/model/cursor.proto"; +import "proto/model/user.proto"; + +// handle cursor events and broadcast to all users +service Cursor { + // send cursor movement to server + rpc Moved (cursor.CursorEvent) returns (cursor.MovedResponse); + // attach to a workspace and receive cursor events + rpc Listen (codemp.model.user.UserIdentity) returns (stream cursor.CursorEvent); +} diff --git a/proto/model/cursor.proto b/proto/model/cursor.proto new file mode 100644 index 0000000..d9a4666 --- /dev/null +++ b/proto/model/cursor.proto @@ -0,0 +1,32 @@ +syntax = "proto2"; + +package codemp.model.cursor; + +import "proto/model/user.proto"; + +// empty request +message MovedResponse {} + +// a tuple indicating row and column +message RowCol { + required int32 row = 1; + required int32 col = 2; +} + +// cursor position object +message CursorPosition { + // path of current buffer this cursor is into + required string buffer = 1; + // cursor start position + required RowCol start = 2; + // cursor end position + required RowCol end = 3; +} + +// cursor event, with user id and cursor position +message CursorEvent { + // user moving the cursor + required codemp.model.user.UserIdentity user = 1; + // new cursor position + required CursorPosition position = 2; +} \ No newline at end of file diff --git a/proto/model/files.proto b/proto/model/files.proto new file mode 100644 index 0000000..e69de29 diff --git a/proto/user.proto b/proto/model/user.proto similarity index 57% rename from proto/user.proto rename to proto/model/user.proto index 1697d5d..9c37484 100644 --- a/proto/user.proto +++ b/proto/model/user.proto @@ -1,12 +1,12 @@ -syntax = "proto3"; +syntax = "proto2"; -package user; +package codemp.model.user; // payload identifying user message UserIdentity { // user identifier - string id = 1; + required string id = 1; } service User{ diff --git a/proto/model/workspace.proto b/proto/model/workspace.proto new file mode 100644 index 0000000..e69de29 diff --git a/proto/workspace.proto b/proto/workspace.proto deleted file mode 100644 index 3f0dd42..0000000 --- a/proto/workspace.proto +++ /dev/null @@ -1,158 +0,0 @@ -// Workspace effimero: sta in /tmp o proprio in memoria -// Workspace e` autenticato: come si decide mentre si rifa il server -// Workspace ha id univoco (stringa), usato per connettercisi -// Workspace implementera` access control: -// * accedere al workspace -// * i singoli buffer -// - i metadati maybe???? -// Workspace offre le seguenti features: -// * listare i buffer DONE -// * listare gli user connessi DONE -// * creare buffers DONE REPLACE THE ONE ON buffer.proto -// * NO ATTACH: responsabilita` del buffer service -// * contiene metadata dei buffers: -// * path -// * data creazione -// Buffer id NON E` il path DONE -// BufferService NON ha metadata: -// Workspace tiene traccia di utenti attached (nel futuro) DONE - - - - - - -syntax = "proto3"; - -package workspace; -import "proto/user.proto"; -import "proto/buffer.proto"; - -message Empty {} - -message WorkspaceEvent { - oneof event { - CursorEvent cursor = 1; - FileEvent file = 2; - UserEvent user = 3; - } -} - -message WorkspaceFileTree { - // list of strings may be more efficient but it's a lot more hassle - string payload = 1; // spappolata di json -} - -message WorkspaceUserList { - repeated user.UserIdentity user = 1; -} - -message WorkspaceMessage { - int32 id = 1; -} - - -message TreeRequest {} // empty -message UserRequest {} -message CursorResponse{} -message UserListRequest{} - -service Workspace { - - // - rpc Create (BufferPayload) returns (BufferCreateResponse); - - rpc ListBuffers (BufferListRequest) returns (BufferList); - - rpc ListUsers (UserListRequest) returns (UserList); - - // - rpc Join (user.UserIdentity) returns (stream WorkspaceEvent); - - // - rpc Tree (TreeRequest) returns (WorkspaceFileTree); - - // - rpc Users (UserRequest) returns (WorkspaceUserList); // TODO could be handled by cursor service - - // send cursor movement to server - rpc Cursor (CursorEvent) returns (CursorResponse); -} - -// a tuple indicating row and column -message RowCol { - int32 row = 1; - int32 col = 2; -} - -// cursor position object -message CursorPosition { - // cursor start position - RowCol start = 1; - // cursor end position - RowCol end = 2; -} - -// cursor event, with user id and cursor position -message CursorEvent { - // user moving the cursor - string user = 1; - // path of current buffer this cursor is into - string buffer = 2; - // new cursor position - repeated CursorPosition position = 3; -} - -enum FileEventType { - CREATE = 0; - DELETE = 1; - RENAME = 2; -} - -message FileEvent { - string buffer = 1; - - FileEventType type = 2; -} - -enum UserEventType { - JOIN = 0; - LEAVE = 1; -} - -message UserEvent { - user.UserIdentity user = 1; - - UserEventType type = 2; -} - - - -message BufferPayload { - // buffer path to operate onto - string path = 1; - - // user id that is requesting the operation - user.UserIdentity user = 2; - - // optional buffer full content for replacing - optional string content = 3; -} - -message BufferCreateResponse { - - string status = 1; -} - - -message BufferListRequest{ - -} - -message BufferList{ - repeated buffer.BufferView buffers = 1; -} - -message UserList{ - repeated user.UserIdentity users = 1; -} \ No newline at end of file diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto new file mode 100644 index 0000000..f06a5fa --- /dev/null +++ b/proto/workspace_service.proto @@ -0,0 +1,135 @@ +// Workspace effimero: sta in /tmp o proprio in memoria +// Workspace e` autenticato: come si decide mentre si rifa il server +// Workspace ha id univoco (stringa), usato per connettercisi +// Workspace implementera` access control: +// * accedere al workspace +// * i singoli buffer +// - i metadati maybe???? +// Workspace offre le seguenti features: +// * listare i buffer DONE +// * listare gli user connessi DONE +// * creare buffers DONE REPLACE THE ONE ON buffer.proto +// * NO ATTACH: responsabilita` del buffer service +// * contiene metadata dei buffers: +// * path +// * data creazione +// Buffer id NON E` il path DONE +// BufferService NON ha metadata: +// Workspace tiene traccia di utenti attached (nel futuro) DONE + + + + + + +syntax = "proto2"; + +package codemp.workspace_service; +import "proto/model/cursor.proto"; +import "proto/model/user.proto"; + + +message Empty {} + + +message WorkspaceFileTree { + // list of strings may be more efficient but it's a lot more hassle + required string payload = 1; // spappolata di json +} + +message WorkspaceUserList { + repeated codemp.model.user.UserIdentity user = 1; +} + +message WorkspaceMessage { + required int32 id = 1; +} + + +message TreeRequest {} // empty +message UserRequest {} +message CursorResponse{} +message UserListRequest{} + +service Workspace { + + // + rpc Create (BufferPayload) returns (Empty); + + rpc Attach (AttachRequest) returns (Token); + + rpc ListBuffers (BufferListRequest) returns (Empty); + + rpc ListUsers (UserListRequest) returns (UserList); + + rpc Join (JoinRequest) returns (Token); + +} + + +message JoinRequest{ + required string username=1; + required string password=2; +} + +message AttachRequest{ + required string bufferAttach = 1; +} + + + + +message Token{ + required string token = 1; +} + +enum FileEventType { + CREATE = 0; + DELETE = 1; + RENAME = 2; +} + +message FileEvent { + required string buffer = 1; + + required FileEventType type = 2; +} + +enum UserEventType { + JOIN = 0; + LEAVE = 1; +} + +message UserEvent { + required codemp.model.user.UserIdentity user = 1; + + required UserEventType type = 2; +} + + + +message BufferPayload { + // buffer path to operate onto + required string path = 1; + + // user id that is requesting the operation + required codemp.model.user.UserIdentity user = 2; + +} + + +message BufferListRequest{ + +} + +message BufferNode{ + required string path = 1; +} + +message BufferTree{ + repeated BufferNode buffers = 1; +} + +message UserList{ + repeated codemp.model.user.UserIdentity users = 1; +} \ No newline at end of file From 3f49730e7ed56d00da744bacce06cb3207ef2514 Mon Sep 17 00:00:00 2001 From: frelodev Date: Thu, 28 Dec 2023 00:20:45 +0100 Subject: [PATCH 06/42] list of proto files to build --- build.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.rs b/build.rs index 8c727c0..e044c83 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,8 @@ fn main() -> Result<(), Box> { - tonic_build::compile_protos("proto/buffer.proto")?; - tonic_build::compile_protos("proto/cursor.proto")?; + tonic_build::compile_protos("proto/model/cursor.proto")?; + tonic_build::compile_protos("proto/model/user.proto")?; + tonic_build::compile_protos("proto/buffer_service.proto")?; + tonic_build::compile_protos("proto/cursor_service.proto")?; + tonic_build::compile_protos("proto/workspace_service.proto")?; Ok(()) } From b78775239f2cc4d4b867a24ebe73fe6ab6537c2b Mon Sep 17 00:00:00 2001 From: frelodev Date: Thu, 28 Dec 2023 00:21:41 +0100 Subject: [PATCH 07/42] cursor and workspace services --- proto/cursor_service.proto | 6 +++--- proto/workspace_service.proto | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto index b0319a8..a9fa0bc 100644 --- a/proto/cursor_service.proto +++ b/proto/cursor_service.proto @@ -1,13 +1,13 @@ syntax = "proto2"; -package codemp.cursor; +package codemp.cursor_service; import "proto/model/cursor.proto"; import "proto/model/user.proto"; // handle cursor events and broadcast to all users service Cursor { // send cursor movement to server - rpc Moved (cursor.CursorEvent) returns (cursor.MovedResponse); + rpc Moved (codemp.model.cursor.CursorEvent) returns (codemp.model.cursor.MovedResponse); // attach to a workspace and receive cursor events - rpc Listen (codemp.model.user.UserIdentity) returns (stream cursor.CursorEvent); + rpc Listen (codemp.model.user.UserIdentity) returns (stream codemp.model.cursor.CursorEvent); } diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index f06a5fa..9d4a7a1 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -25,7 +25,6 @@ syntax = "proto2"; package codemp.workspace_service; -import "proto/model/cursor.proto"; import "proto/model/user.proto"; @@ -54,14 +53,16 @@ message UserListRequest{} service Workspace { // - rpc Create (BufferPayload) returns (Empty); + rpc Create (BufferPayload) returns (Empty); rpc Attach (AttachRequest) returns (Token); - rpc ListBuffers (BufferListRequest) returns (Empty); + rpc ListBuffers (BufferListRequest) returns (BufferTree); rpc ListUsers (UserListRequest) returns (UserList); + rpc ListBufferUsers (Empty) returns (Empty); + rpc Join (JoinRequest) returns (Token); } From ab982f4882bf23461b019ddd903308fe6045d068 Mon Sep 17 00:00:00 2001 From: frelodev Date: Thu, 28 Dec 2023 22:04:40 +0100 Subject: [PATCH 08/42] changed build method for proto and lib proto includes --- build.rs | 19 +++++++++++++------ proto/cursor_service.proto | 4 ++-- proto/model/cursor.proto | 2 +- proto/model/user.proto | 6 +----- proto/workspace_service.proto | 2 +- src/lib.rs | 7 +++++-- 6 files changed, 23 insertions(+), 17 deletions(-) diff --git a/build.rs b/build.rs index e044c83..a33719e 100644 --- a/build.rs +++ b/build.rs @@ -1,8 +1,15 @@ fn main() -> Result<(), Box> { - tonic_build::compile_protos("proto/model/cursor.proto")?; - tonic_build::compile_protos("proto/model/user.proto")?; - tonic_build::compile_protos("proto/buffer_service.proto")?; - tonic_build::compile_protos("proto/cursor_service.proto")?; - tonic_build::compile_protos("proto/workspace_service.proto")?; + tonic_build::configure() + .build_server(false) + .compile( + &[ + "proto/model/cursor.proto", + "proto/model/user.proto", + "proto/buffer_service.proto", + "proto/cursor_service.proto", + "proto/workspace_service.proto" + ], + &["proto", "proto", "proto","proto", "proto"] + )?; Ok(()) -} + } \ No newline at end of file diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto index a9fa0bc..cdeb88d 100644 --- a/proto/cursor_service.proto +++ b/proto/cursor_service.proto @@ -1,8 +1,8 @@ syntax = "proto2"; package codemp.cursor_service; -import "proto/model/cursor.proto"; -import "proto/model/user.proto"; +import "model/cursor.proto"; +import "model/user.proto"; // handle cursor events and broadcast to all users service Cursor { diff --git a/proto/model/cursor.proto b/proto/model/cursor.proto index d9a4666..1f2ddb1 100644 --- a/proto/model/cursor.proto +++ b/proto/model/cursor.proto @@ -2,7 +2,7 @@ syntax = "proto2"; package codemp.model.cursor; -import "proto/model/user.proto"; +import "model/user.proto"; // empty request message MovedResponse {} diff --git a/proto/model/user.proto b/proto/model/user.proto index 9c37484..4cb0f78 100644 --- a/proto/model/user.proto +++ b/proto/model/user.proto @@ -4,11 +4,7 @@ package codemp.model.user; // payload identifying user -message UserIdentity { +message UserIdentity{ // user identifier required string id = 1; } - -service User{ - -} \ No newline at end of file diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 9d4a7a1..feffa91 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -25,7 +25,7 @@ syntax = "proto2"; package codemp.workspace_service; -import "proto/model/user.proto"; +import "model/user.proto"; message Empty {} diff --git a/src/lib.rs b/src/lib.rs index 3345a0e..f45287a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -165,8 +165,11 @@ pub use woot; #[cfg(feature = "proto")] #[allow(non_snake_case)] pub mod proto { - tonic::include_proto!("codemp.buffer"); - tonic::include_proto!("codemp.cursor"); + tonic::include_proto!("codemp.model.cursor"); + tonic::include_proto!("codemp.model.user"); + tonic::include_proto!("codemp.buffer_service"); + tonic::include_proto!("codemp.cursor_service"); + tonic::include_proto!("codemp.workspace_service"); } From 3b1be930d8660bebacc038f7e2fa4eab0bd5dffb Mon Sep 17 00:00:00 2001 From: frelodev Date: Fri, 29 Dec 2023 01:03:08 +0100 Subject: [PATCH 09/42] new features --- Cargo.toml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 78b6b23..6039141 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,15 +25,17 @@ serde_json = { version = "1", optional = true } tokio-stream = { version = "0.1", optional = true } # global lazy_static = { version = "1.4", optional = true } +serde = { version = "1.0.193", features = ["derive"] } [build-dependencies] tonic-build = "0.9" [features] -default = ["client"] -api = ["woot", "dep:similar", "dep:tokio", "dep:async-trait"] -woot = ["dep:codemp-woot"] -proto = ["dep:prost", "dep:tonic"] -client = ["proto", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json"] -global = ["client", "dep:lazy_static"] -sync = ["client"] +default = ["transport", "dep:serde_json"] +api = ["woot", "dep:similar", "dep:tokio", "dep:async-trait"] +woot = ["dep:codemp-woot"] +transport = ["dep:prost", "dep:tonic"] +client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json"] +server = ["transport"] +global = ["client", "dep:lazy_static"] +sync = ["client"] From 62303710205978dc30c8af187eb754e62828d7d3 Mon Sep 17 00:00:00 2001 From: frelodev Date: Fri, 29 Dec 2023 01:06:28 +0100 Subject: [PATCH 10/42] feat: fixed proto and tonic includes Co-authored-by: zaaarf Co-authored-by: alemi --- build.rs | 12 +++++++----- proto/buffer_service.proto | 2 +- proto/{model => }/cursor.proto | 6 +++--- proto/cursor_service.proto | 10 +++++----- proto/{model => }/files.proto | 0 proto/model/workspace.proto | 0 proto/{model => }/user.proto | 2 +- proto/workspace_service.proto | 14 ++++++++------ src/lib.rs | 12 ++++++------ 9 files changed, 31 insertions(+), 27 deletions(-) rename proto/{model => }/cursor.proto (84%) rename proto/{model => }/files.proto (100%) delete mode 100644 proto/model/workspace.proto rename proto/{model => }/user.proto (81%) diff --git a/build.rs b/build.rs index a33719e..ae01fed 100644 --- a/build.rs +++ b/build.rs @@ -1,15 +1,17 @@ fn main() -> Result<(), Box> { tonic_build::configure() - .build_server(false) + // .build_client(cfg!(feature = "client")) + //.build_server(cfg!(feature = "server")) // FIXME if false, build fails???? + // .build_transport(cfg!(feature = "transport")) .compile( &[ - "proto/model/cursor.proto", - "proto/model/user.proto", + "proto/user.proto", + "proto/cursor.proto", "proto/buffer_service.proto", "proto/cursor_service.proto", "proto/workspace_service.proto" ], - &["proto", "proto", "proto","proto", "proto"] - )?; + &["proto"] + )?; Ok(()) } \ No newline at end of file diff --git a/proto/buffer_service.proto b/proto/buffer_service.proto index 6b09fd0..96ea355 100644 --- a/proto/buffer_service.proto +++ b/proto/buffer_service.proto @@ -1,6 +1,6 @@ syntax = "proto2"; -package codemp.buffer_service; +package buffer_service; // handle buffer changes, keep in sync users service Buffer { diff --git a/proto/model/cursor.proto b/proto/cursor.proto similarity index 84% rename from proto/model/cursor.proto rename to proto/cursor.proto index 1f2ddb1..2c4ea18 100644 --- a/proto/model/cursor.proto +++ b/proto/cursor.proto @@ -1,8 +1,8 @@ syntax = "proto2"; -package codemp.model.cursor; +package cursor; -import "model/user.proto"; +import "user.proto"; // empty request message MovedResponse {} @@ -26,7 +26,7 @@ message CursorPosition { // cursor event, with user id and cursor position message CursorEvent { // user moving the cursor - required codemp.model.user.UserIdentity user = 1; + required user.UserIdentity user = 1; // new cursor position required CursorPosition position = 2; } \ No newline at end of file diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto index cdeb88d..088add5 100644 --- a/proto/cursor_service.proto +++ b/proto/cursor_service.proto @@ -1,13 +1,13 @@ syntax = "proto2"; -package codemp.cursor_service; -import "model/cursor.proto"; -import "model/user.proto"; +package cursor_service; +import "cursor.proto"; +import "user.proto"; // handle cursor events and broadcast to all users service Cursor { // send cursor movement to server - rpc Moved (codemp.model.cursor.CursorEvent) returns (codemp.model.cursor.MovedResponse); + rpc Moved (cursor.CursorEvent) returns (cursor.MovedResponse); // attach to a workspace and receive cursor events - rpc Listen (codemp.model.user.UserIdentity) returns (stream codemp.model.cursor.CursorEvent); + rpc Listen (user.UserIdentity) returns (stream cursor.CursorEvent); } diff --git a/proto/model/files.proto b/proto/files.proto similarity index 100% rename from proto/model/files.proto rename to proto/files.proto diff --git a/proto/model/workspace.proto b/proto/model/workspace.proto deleted file mode 100644 index e69de29..0000000 diff --git a/proto/model/user.proto b/proto/user.proto similarity index 81% rename from proto/model/user.proto rename to proto/user.proto index 4cb0f78..f639f8c 100644 --- a/proto/model/user.proto +++ b/proto/user.proto @@ -1,6 +1,6 @@ syntax = "proto2"; -package codemp.model.user; +package user; // payload identifying user diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index feffa91..6ed7c93 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -24,8 +24,8 @@ syntax = "proto2"; -package codemp.workspace_service; -import "model/user.proto"; +package workspace_service; +import "user.proto"; message Empty {} @@ -37,7 +37,7 @@ message WorkspaceFileTree { } message WorkspaceUserList { - repeated codemp.model.user.UserIdentity user = 1; + repeated user.UserIdentity user = 1; } message WorkspaceMessage { @@ -65,6 +65,8 @@ service Workspace { rpc Join (JoinRequest) returns (Token); + rpc Delete (BufferPayload) returns (Empty); //deletes buffer + } @@ -102,7 +104,7 @@ enum UserEventType { } message UserEvent { - required codemp.model.user.UserIdentity user = 1; + required user.UserIdentity user = 1; required UserEventType type = 2; } @@ -114,7 +116,7 @@ message BufferPayload { required string path = 1; // user id that is requesting the operation - required codemp.model.user.UserIdentity user = 2; + required user.UserIdentity user = 2; } @@ -132,5 +134,5 @@ message BufferTree{ } message UserList{ - repeated codemp.model.user.UserIdentity users = 1; + repeated user.UserIdentity users = 1; } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index f45287a..5f27327 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -162,14 +162,14 @@ pub mod prelude; pub use woot; /// protocol types and services auto-generated by grpc -#[cfg(feature = "proto")] +#[cfg(feature = "transport")] #[allow(non_snake_case)] pub mod proto { - tonic::include_proto!("codemp.model.cursor"); - tonic::include_proto!("codemp.model.user"); - tonic::include_proto!("codemp.buffer_service"); - tonic::include_proto!("codemp.cursor_service"); - tonic::include_proto!("codemp.workspace_service"); + pub mod user { tonic::include_proto!("user"); } + pub mod cursor { tonic::include_proto!("cursor"); } + pub mod buffer_service { tonic::include_proto!("buffer_service"); } + pub mod cursor_service { tonic::include_proto!("cursor_service"); } + pub mod workspace_service { tonic::include_proto!("workspace_service"); } } From 85add1ca0da5093430ebb231c99dec42ca151aa3 Mon Sep 17 00:00:00 2001 From: frelodev Date: Mon, 1 Jan 2024 23:29:35 +0100 Subject: [PATCH 11/42] moved some messages from workspace_service to files --- build.rs | 1 + proto/files.proto | 17 +++++++++++++++++ proto/workspace.proto | 0 proto/workspace_service.proto | 14 +++----------- src/lib.rs | 1 + 5 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 proto/workspace.proto diff --git a/build.rs b/build.rs index ae01fed..3865432 100644 --- a/build.rs +++ b/build.rs @@ -7,6 +7,7 @@ fn main() -> Result<(), Box> { &[ "proto/user.proto", "proto/cursor.proto", + "proto/files.proto", "proto/buffer_service.proto", "proto/cursor_service.proto", "proto/workspace_service.proto" diff --git a/proto/files.proto b/proto/files.proto index e69de29..152a00b 100644 --- a/proto/files.proto +++ b/proto/files.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; + +package files; + + +message BufferNode{ + required string path = 1; +} + +message BufferTree{ + repeated BufferNode buffers = 1; +} + +message WorkspaceFileTree { + // list of strings may be more efficient but it's a lot more hassle + required string payload = 1; // spappolata di json +} //Alla fine non si usa questo per ora ma BufferTree \ No newline at end of file diff --git a/proto/workspace.proto b/proto/workspace.proto new file mode 100644 index 0000000..e69de29 diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 6ed7c93..470fcd9 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -26,15 +26,13 @@ syntax = "proto2"; package workspace_service; import "user.proto"; +import "files.proto"; message Empty {} -message WorkspaceFileTree { - // list of strings may be more efficient but it's a lot more hassle - required string payload = 1; // spappolata di json -} + message WorkspaceUserList { repeated user.UserIdentity user = 1; @@ -57,7 +55,7 @@ service Workspace { rpc Attach (AttachRequest) returns (Token); - rpc ListBuffers (BufferListRequest) returns (BufferTree); + rpc ListBuffers (BufferListRequest) returns (files.BufferTree); rpc ListUsers (UserListRequest) returns (UserList); @@ -125,13 +123,7 @@ message BufferListRequest{ } -message BufferNode{ - required string path = 1; -} -message BufferTree{ - repeated BufferNode buffers = 1; -} message UserList{ repeated user.UserIdentity users = 1; diff --git a/src/lib.rs b/src/lib.rs index 5f27327..15042a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,6 +167,7 @@ pub use woot; pub mod proto { pub mod user { tonic::include_proto!("user"); } pub mod cursor { tonic::include_proto!("cursor"); } + pub mod files { tonic::include_proto!("files"); } pub mod buffer_service { tonic::include_proto!("buffer_service"); } pub mod cursor_service { tonic::include_proto!("cursor_service"); } pub mod workspace_service { tonic::include_proto!("workspace_service"); } From 16575213566900b6d0e44caaa87ad66d48b5fb06 Mon Sep 17 00:00:00 2001 From: frelodev Date: Mon, 1 Jan 2024 23:34:59 +0100 Subject: [PATCH 12/42] workspace_service cleanup --- build.rs | 1 + proto/workspace.proto | 83 +++++++++++++++++++++++++++++++ proto/workspace_service.proto | 94 +++-------------------------------- src/lib.rs | 1 + 4 files changed, 93 insertions(+), 86 deletions(-) diff --git a/build.rs b/build.rs index 3865432..b9f9a1c 100644 --- a/build.rs +++ b/build.rs @@ -8,6 +8,7 @@ fn main() -> Result<(), Box> { "proto/user.proto", "proto/cursor.proto", "proto/files.proto", + "proto/workspace.proto", "proto/buffer_service.proto", "proto/cursor_service.proto", "proto/workspace_service.proto" diff --git a/proto/workspace.proto b/proto/workspace.proto index e69de29..f0f2427 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -0,0 +1,83 @@ +syntax = "proto2"; + +package workspace; +import "user.proto"; +import "files.proto"; + + +message Empty {} + + +message TreeRequest {} // empty +message UserRequest {} +message CursorResponse{} +message UserListRequest{} + +message WorkspaceUserList { + repeated user.UserIdentity user = 1; +} + +message WorkspaceMessage { + required int32 id = 1; +} + +message JoinRequest{ + required string username=1; + required string password=2; +} + +message AttachRequest{ + required string bufferAttach = 1; +} + + + + +message Token{ + required string token = 1; +} + +enum FileEventType { + CREATE = 0; + DELETE = 1; + RENAME = 2; +} + +message FileEvent { + required string buffer = 1; + + required FileEventType type = 2; +} + +enum UserEventType { + JOIN = 0; + LEAVE = 1; +} + +message UserEvent { + required user.UserIdentity user = 1; + + required UserEventType type = 2; +} + + + +message BufferPayload { + // buffer path to operate onto + required string path = 1; + + // user id that is requesting the operation + required user.UserIdentity user = 2; + +} + + +message BufferListRequest{ + +} + + + +message UserList{ + repeated user.UserIdentity users = 1; +} \ No newline at end of file diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 470fcd9..55857a6 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -27,104 +27,26 @@ syntax = "proto2"; package workspace_service; import "user.proto"; import "files.proto"; +import "workspace.proto"; -message Empty {} - - - - -message WorkspaceUserList { - repeated user.UserIdentity user = 1; -} - -message WorkspaceMessage { - required int32 id = 1; -} - - -message TreeRequest {} // empty -message UserRequest {} -message CursorResponse{} -message UserListRequest{} service Workspace { - // - rpc Create (BufferPayload) returns (Empty); + rpc Create (workspace.BufferPayload) returns (workspace.Empty); - rpc Attach (AttachRequest) returns (Token); + rpc Attach (workspace.AttachRequest) returns (workspace.Token); - rpc ListBuffers (BufferListRequest) returns (files.BufferTree); + rpc ListBuffers (workspace.BufferListRequest) returns (files.BufferTree); - rpc ListUsers (UserListRequest) returns (UserList); + rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); - rpc ListBufferUsers (Empty) returns (Empty); + rpc ListBufferUsers (workspace.Empty) returns (workspace.Empty); - rpc Join (JoinRequest) returns (Token); + rpc Join (workspace.JoinRequest) returns (workspace.Token); - rpc Delete (BufferPayload) returns (Empty); //deletes buffer + rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer } -message JoinRequest{ - required string username=1; - required string password=2; -} - -message AttachRequest{ - required string bufferAttach = 1; -} - - - - -message Token{ - required string token = 1; -} - -enum FileEventType { - CREATE = 0; - DELETE = 1; - RENAME = 2; -} - -message FileEvent { - required string buffer = 1; - - required FileEventType type = 2; -} - -enum UserEventType { - JOIN = 0; - LEAVE = 1; -} - -message UserEvent { - required user.UserIdentity user = 1; - - required UserEventType type = 2; -} - - - -message BufferPayload { - // buffer path to operate onto - required string path = 1; - - // user id that is requesting the operation - required user.UserIdentity user = 2; - -} - - -message BufferListRequest{ - -} - - - -message UserList{ - repeated user.UserIdentity users = 1; -} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 15042a7..1acd87c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -168,6 +168,7 @@ pub mod proto { pub mod user { tonic::include_proto!("user"); } pub mod cursor { tonic::include_proto!("cursor"); } pub mod files { tonic::include_proto!("files"); } + pub mod workspace { tonic::include_proto!("workspace"); } pub mod buffer_service { tonic::include_proto!("buffer_service"); } pub mod cursor_service { tonic::include_proto!("cursor_service"); } pub mod workspace_service { tonic::include_proto!("workspace_service"); } From 94a7786812ae0b929bc5029657131e6c46cf3804 Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 25 Jan 2024 02:13:45 +0100 Subject: [PATCH 13/42] feat: workspaces and new library structure Co-authored-by: alemi Co-authored-by: frelodev --- Cargo.toml | 6 +- proto/cursor_service.proto | 6 +- proto/files.proto | 4 +- proto/user.proto | 2 +- proto/workspace.proto | 31 ++---- src/api/change.rs | 2 + src/buffer/worker.rs | 49 ++++----- src/client.rs | 199 +++++++++++++++------------------- src/cursor/controller.rs | 21 ++-- src/cursor/mod.rs | 6 +- src/cursor/worker.rs | 26 ++--- src/errors.rs | 7 ++ src/lib.rs | 6 +- src/prelude.rs | 4 +- src/workspace.rs | 215 +++++++++++++++++++++++++++++++++++++ 15 files changed, 380 insertions(+), 204 deletions(-) create mode 100644 src/workspace.rs diff --git a/Cargo.toml b/Cargo.toml index 6039141..575d071 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ name = "codemp" # core tracing = "0.1" # woot -codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", tag = "v0.1.0", optional = true } +codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.0", optional = true } # proto tonic = { version = "0.9", features = ["tls", "tls-roots"], optional = true } prost = { version = "0.11.8", optional = true } @@ -26,12 +26,13 @@ tokio-stream = { version = "0.1", optional = true } # global lazy_static = { version = "1.4", optional = true } serde = { version = "1.0.193", features = ["derive"] } +postcard = "1.0.8" [build-dependencies] tonic-build = "0.9" [features] -default = ["transport", "dep:serde_json"] +default = ["client"] api = ["woot", "dep:similar", "dep:tokio", "dep:async-trait"] woot = ["dep:codemp-woot"] transport = ["dep:prost", "dep:tonic"] @@ -39,3 +40,4 @@ client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "d server = ["transport"] global = ["client", "dep:lazy_static"] sync = ["client"] +backport = [] # TODO remove! diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto index 088add5..4044f3e 100644 --- a/proto/cursor_service.proto +++ b/proto/cursor_service.proto @@ -6,8 +6,6 @@ import "user.proto"; // handle cursor events and broadcast to all users service Cursor { - // send cursor movement to server - rpc Moved (cursor.CursorEvent) returns (cursor.MovedResponse); - // attach to a workspace and receive cursor events - rpc Listen (user.UserIdentity) returns (stream cursor.CursorEvent); + // subscribe to a workspace's cursor events + rpc Attach (stream cursor.CursorEvent) returns (stream cursor.CursorEvent); } diff --git a/proto/files.proto b/proto/files.proto index 152a00b..e57cdc2 100644 --- a/proto/files.proto +++ b/proto/files.proto @@ -3,11 +3,11 @@ syntax = "proto2"; package files; -message BufferNode{ +message BufferNode { required string path = 1; } -message BufferTree{ +message BufferTree { repeated BufferNode buffers = 1; } diff --git a/proto/user.proto b/proto/user.proto index f639f8c..322a935 100644 --- a/proto/user.proto +++ b/proto/user.proto @@ -4,7 +4,7 @@ package user; // payload identifying user -message UserIdentity{ +message UserIdentity { // user identifier required string id = 1; } diff --git a/proto/workspace.proto b/proto/workspace.proto index f0f2427..b076cf3 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -4,14 +4,12 @@ package workspace; import "user.proto"; import "files.proto"; - message Empty {} - message TreeRequest {} // empty message UserRequest {} -message CursorResponse{} -message UserListRequest{} +message CursorResponse {} +message UserListRequest {} message WorkspaceUserList { repeated user.UserIdentity user = 1; @@ -21,19 +19,16 @@ message WorkspaceMessage { required int32 id = 1; } -message JoinRequest{ +message JoinRequest { required string username=1; required string password=2; } -message AttachRequest{ - required string bufferAttach = 1; +message AttachRequest { + required string id = 1; } - - - -message Token{ +message Token { required string token = 1; } @@ -44,7 +39,7 @@ enum FileEventType { } message FileEvent { - required string buffer = 1; + required string bufferbuffertree = 1; required FileEventType type = 2; } @@ -56,28 +51,18 @@ enum UserEventType { message UserEvent { required user.UserIdentity user = 1; - required UserEventType type = 2; } - - message BufferPayload { // buffer path to operate onto required string path = 1; - - // user id that is requesting the operation - required user.UserIdentity user = 2; - } - message BufferListRequest{ } - - -message UserList{ +message UserList { repeated user.UserIdentity users = 1; } \ No newline at end of file diff --git a/src/api/change.rs b/src/api/change.rs index 1894c18..e9e6792 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -3,6 +3,8 @@ //! an editor-friendly representation of a text change in a buffer //! to easily interface with codemp from various editors +use crate::proto::cursor::RowCol; + /// an editor-friendly representation of a text change in a buffer /// /// this represent a range in the previous state of the string and a new content which should be diff --git a/src/buffer/worker.rs b/src/buffer/worker.rs index 5804a0d..c2e77bc 100644 --- a/src/buffer/worker.rs +++ b/src/buffer/worker.rs @@ -3,22 +3,20 @@ use std::hash::{Hash, Hasher}; use similar::{TextDiff, ChangeTag}; use tokio::sync::{watch, mpsc, oneshot}; -use tonic::transport::Channel; use tonic::{async_trait, Streaming}; +use uuid::Uuid; use woot::crdt::{Op, CRDT, TextEditor}; use woot::woot::Woot; use crate::errors::IgnorableError; -use crate::proto::{OperationRequest, RawOp}; -use crate::proto::buffer_client::BufferClient; use crate::api::controller::ControllerWorker; use crate::api::TextChange; +use crate::proto::buffer_service::Operation; use super::controller::BufferController; - -pub(crate) struct BufferControllerWorker { - uid: String, +pub(crate) struct BufferWorker { + _user_id: Uuid, name: String, buffer: Woot, content: watch::Sender, @@ -36,17 +34,17 @@ struct ClonableHandlesForController { content: watch::Receiver, } -impl BufferControllerWorker { - pub fn new(uid: String, path: &str) -> Self { +impl BufferWorker { + pub fn new(user_id: Uuid, path: &str) -> Self { let (txt_tx, txt_rx) = watch::channel("".to_string()); let (op_tx, op_rx) = mpsc::unbounded_channel(); let (end_tx, end_rx) = mpsc::unbounded_channel(); let (poller_tx, poller_rx) = mpsc::unbounded_channel(); let mut hasher = DefaultHasher::new(); - uid.hash(&mut hasher); + user_id.hash(&mut hasher); let site_id = hasher.finish() as usize; - BufferControllerWorker { - uid, + BufferWorker { + _user_id: user_id, name: path.to_string(), buffer: Woot::new(site_id % (2<<10), ""), // TODO remove the modulo, only for debugging! content: txt_tx, @@ -62,26 +60,13 @@ impl BufferControllerWorker { stop: end_rx, } } - - async fn send_op(&self, tx: &mut BufferClient, outbound: &Op) -> crate::Result<()> { - let opseq = serde_json::to_string(outbound).expect("could not serialize opseq"); - let req = OperationRequest { - path: self.name.clone(), - hash: format!("{:x}", md5::compute(self.buffer.view())), - op: Some(RawOp { - opseq, user: self.uid.clone(), - }), - }; - let _ = tx.edit(req).await?; - Ok(()) - } } #[async_trait] impl ControllerWorker for BufferControllerWorker { type Controller = BufferController; - type Tx = BufferClient; - type Rx = Streaming; + type Tx = mpsc::Sender; + type Rx = Streaming; fn subscribe(&self) -> BufferController { BufferController::new( @@ -93,7 +78,7 @@ impl ControllerWorker for BufferControllerWorker { ) } - async fn work(mut self, mut tx: Self::Tx, mut rx: Self::Rx) { + async fn work(mut self, tx: Self::Tx, mut rx: Self::Rx) { loop { // block until one of these is ready tokio::select! { @@ -143,7 +128,13 @@ impl ControllerWorker for BufferControllerWorker { } for op in ops { - match self.send_op(&mut tx, &op).await { + let operation = Operation { + data: postcard::to_extend(&op, Vec::new()).unwrap(), + user: None, + path: Some(self.name.clone()) + }; + + match tx.send(operation).await { Err(e) => tracing::error!("server refused to broadcast {}: {}", op, e), Ok(()) => { self.content.send(self.buffer.view()).unwrap_or_warn("could not send buffer update"); @@ -160,7 +151,7 @@ impl ControllerWorker for BufferControllerWorker { res = rx.message() => match res { Err(_e) => break, Ok(None) => break, - Ok(Some(change)) => match serde_json::from_str::(&change.opseq) { + Ok(Some(change)) => match postcard::from_bytes::(&change.data) { Ok(op) => { self.buffer.merge(op); self.content.send(self.buffer.view()).unwrap_or_warn("could not send buffer update"); diff --git a/src/client.rs b/src/client.rs index 5b7bf4a..fbfaf51 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,20 +1,21 @@ //! ### client -//! +//! //! codemp client manager, containing grpc services -use std::{sync::Arc, collections::BTreeMap}; - -use tonic::transport::Channel; - -use crate::{ - cursor::{worker::CursorControllerWorker, controller::CursorController}, - proto::{ - buffer_client::BufferClient, cursor_client::CursorClient, UserIdentity, BufferPayload, - }, - Error, api::controller::ControllerWorker, - buffer::{controller::BufferController, worker::BufferControllerWorker}, -}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tonic::service::interceptor::InterceptedService; +use tonic::service::Interceptor; +use tonic::transport::{Channel, Endpoint}; +use uuid::Uuid; +use crate::api::controller::ControllerWorker; +use crate::cursor::worker::CursorWorker; +use crate::proto::buffer_service::buffer_client::BufferClient; +use crate::proto::cursor_service::cursor_client::CursorClient; +use crate::proto::workspace::{JoinRequest, Token}; +use crate::proto::workspace_service::workspace_client::WorkspaceClient; +use crate::workspace::Workspace; /// codemp client manager /// @@ -22,130 +23,102 @@ use crate::{ /// will disconnect when dropped /// can be used to interact with server pub struct Client { - id: String, - client: Services, + user_id: Uuid, + token_tx: Arc>, workspace: Option, + services: Arc } -struct Services { - buffer: BufferClient, - cursor: CursorClient, +#[derive(Clone)] +pub(crate) struct ClientInterceptor { + token: tokio::sync::watch::Receiver } -struct Workspace { - cursor: Arc, - buffers: BTreeMap>, +impl Interceptor for ClientInterceptor { + fn call(&mut self, mut request: tonic::Request<()>) -> Result, tonic::Status> { + if let Ok(token) = self.token.borrow().token.parse() { + request.metadata_mut().insert("auth", token); + } + + Ok(request) + } } +#[derive(Debug, Clone)] +pub(crate) struct Services { + pub(crate) workspace: crate::proto::workspace_service::workspace_client::WorkspaceClient>, + pub(crate) buffer: crate::proto::buffer_service::buffer_client::BufferClient>, + pub(crate) cursor: crate::proto::cursor_service::cursor_client::CursorClient>, +} + +// TODO meno losco +fn parse_codemp_connection_string<'a>(string: &'a str) -> (String, String) { + let url = string.replace("codemp://", ""); + let (host, workspace) = url.split_once('/').unwrap(); + (format!("http://{}", host), workspace.to_string()) +} + impl Client { /// instantiate and connect a new client - pub async fn new(dst: &str) -> Result { - let buffer = BufferClient::connect(dst.to_string()).await?; - let cursor = CursorClient::connect(dst.to_string()).await?; - let id = uuid::Uuid::new_v4().to_string(); - - Ok(Client { id, client: Services { buffer, cursor}, workspace: None }) - } + pub async fn new(dest: &str) -> crate::Result { //TODO interceptor + let (_host, _workspace_id) = parse_codemp_connection_string(dest); - /// return a reference to current cursor controller, if currently in a workspace - pub fn get_cursor(&self) -> Option> { - Some(self.workspace.as_ref()?.cursor.clone()) - } + let channel = Endpoint::from_shared(dest.to_string())? + .connect() + .await?; - /// leave current workspace if in one, disconnecting buffer and cursor controllers - pub fn leave_workspace(&mut self) { - // TODO need to stop tasks? - self.workspace = None - } + let (token_tx, token_rx) = tokio::sync::watch::channel( + Token { token: "".to_string() } + ); - /// disconnect from a specific buffer - pub fn disconnect_buffer(&mut self, path: &str) -> bool { - match &mut self.workspace { - Some(w) => w.buffers.remove(path).is_some(), - None => false, - } - } + let inter = ClientInterceptor { token: token_rx }; - /// get a new reference to a buffer controller, if any is active to given path - pub fn get_buffer(&self, path: &str) -> Option> { - self.workspace.as_ref()?.buffers.get(path).cloned() + let buffer = BufferClient::with_interceptor(channel.clone(), inter.clone()); + let cursor = CursorClient::with_interceptor(channel.clone(), inter.clone()); + let workspace = WorkspaceClient::with_interceptor(channel.clone(), inter.clone()); + + let user_id = uuid::Uuid::new_v4(); + + Ok(Client { + user_id, + token_tx: Arc::new(token_tx), + workspace: None, + services: Arc::new(Services { workspace, buffer, cursor }) + }) } /// join a workspace, starting a cursorcontroller and returning a new reference to it - /// + /// /// to interact with such workspace [crate::api::Controller::send] cursor events or /// [crate::api::Controller::recv] for events on the associated [crate::cursor::Controller]. - pub async fn join(&mut self, _session: &str) -> crate::Result> { - // TODO there is no real workspace handling in codemp server so it behaves like one big global - // session. I'm still creating this to start laying out the proper use flow - let stream = self.client.cursor.listen(UserIdentity { id: "".into() }).await?.into_inner(); + pub async fn join(&mut self, workspace_id: &str) -> crate::Result<()> { + self.token_tx.send(self.services.workspace.clone().join( + tonic::Request::new(JoinRequest { username: "".to_string(), password: "".to_string() }) //TODO + ).await?.into_inner())?; - let controller = CursorControllerWorker::new(self.id.clone()); - let client = self.client.cursor.clone(); - - let handle = Arc::new(controller.subscribe()); + let (tx, rx) = mpsc::channel(10); + let stream = self.services.cursor.clone() + .attach(tokio_stream::wrappers::ReceiverStream::new(rx)) + .await? + .into_inner(); + let worker = CursorWorker::new(self.user_id.clone()); + let controller = Arc::new(worker.subscribe()); tokio::spawn(async move { - tracing::debug!("cursor worker started"); - controller.work(client, stream).await; - tracing::debug!("cursor worker stopped"); + tracing::debug!("controller worker started"); + worker.work(tx, stream).await; + tracing::debug!("controller worker stopped"); }); - self.workspace = Some( - Workspace { - cursor: handle.clone(), - buffers: BTreeMap::new() - } - ); + self.workspace = Some(Workspace::new( + workspace_id.to_string(), + self.user_id, + self.token_tx.clone(), + controller, + self.services.clone() + ).await?); - Ok(handle) - } - - /// create a new buffer in current workspace, with optional given content - pub async fn create(&mut self, path: &str, content: Option<&str>) -> crate::Result<()> { - if let Some(_workspace) = &self.workspace { - self.client.buffer - .create(BufferPayload { - user: self.id.clone(), - path: path.to_string(), - content: content.map(|x| x.to_string()), - }).await?; - - Ok(()) - } else { - Err(Error::InvalidState { msg: "join a workspace first".into() }) - } - } - - /// attach to a buffer, starting a buffer controller and returning a new reference to it - /// - /// to interact with such buffer use [crate::api::Controller::send] or - /// [crate::api::Controller::recv] to exchange [crate::api::TextChange] - pub async fn attach(&mut self, path: &str) -> crate::Result> { - if let Some(workspace) = &mut self.workspace { - let mut client = self.client.buffer.clone(); - let req = BufferPayload { - path: path.to_string(), user: self.id.clone(), content: None - }; - - let stream = client.attach(req).await?.into_inner(); - - let controller = BufferControllerWorker::new(self.id.clone(), path); - let handler = Arc::new(controller.subscribe()); - - let _path = path.to_string(); - tokio::spawn(async move { - tracing::debug!("buffer[{}] worker started", _path); - controller.work(client, stream).await; - tracing::debug!("buffer[{}] worker stopped", _path); - }); - - workspace.buffers.insert(path.to_string(), handler.clone()); - - Ok(handler) - } else { - Err(Error::InvalidState { msg: "join a workspace first".into() }) - } + Ok(()) } } diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index c77784e..3e793ac 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -4,8 +4,9 @@ use tokio::sync::{mpsc, broadcast::{self, error::{TryRecvError, RecvError}}, Mutex, watch}; use tonic::async_trait; +use uuid::Uuid; -use crate::{proto::{CursorPosition, CursorEvent}, Error, api::Controller, errors::IgnorableError}; +use crate::{api::Controller, errors::IgnorableError, proto::{cursor::{CursorEvent, CursorPosition}, user::UserIdentity}}; /// the cursor controller implementation /// @@ -20,7 +21,7 @@ use crate::{proto::{CursorPosition, CursorEvent}, Error, api::Controller, errors /// upon dropping this handle will stop the associated worker #[derive(Debug)] pub struct CursorController { - uid: String, + user_id: Uuid, op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, @@ -35,13 +36,13 @@ impl Drop for CursorController { impl CursorController { pub(crate) fn new( - uid: String, + user_id: Uuid, op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, stop: mpsc::UnboundedSender<()>, ) -> Self { - CursorController { uid, op, last_op, stream, stop } + CursorController { user_id, op, last_op, stream, stop } } } @@ -51,13 +52,13 @@ impl Controller for CursorController { /// enqueue a cursor event to be broadcast to current workspace /// will automatically invert cursor start/end if they are inverted - fn send(&self, mut cursor: CursorPosition) -> Result<(), Error> { + fn send(&self, mut cursor: CursorPosition) -> crate::Result<()> { if cursor.start() > cursor.end() { std::mem::swap(&mut cursor.start, &mut cursor.end); } Ok(self.op.send(CursorEvent { - user: self.uid.clone(), - position: Some(cursor), + user: UserIdentity { id: self.user_id.to_string() }, + position: cursor, })?) } @@ -67,7 +68,7 @@ impl Controller for CursorController { match stream.try_recv() { Ok(x) => Ok(Some(x)), Err(TryRecvError::Empty) => Ok(None), - Err(TryRecvError::Closed) => Err(Error::Channel { send: false }), + Err(TryRecvError::Closed) => Err(crate::Error::Channel { send: false }), Err(TryRecvError::Lagged(n)) => { tracing::warn!("cursor channel lagged, skipping {} events", n); Ok(stream.try_recv().ok()) @@ -78,11 +79,11 @@ impl Controller for CursorController { // TODO is this cancelable? so it can be used in tokio::select! // TODO is the result type overkill? should be an option? /// get next cursor event from current workspace, or block until one is available - async fn recv(&self) -> Result { + async fn recv(&self) -> crate::Result { let mut stream = self.stream.lock().await; match stream.recv().await { Ok(x) => Ok(x), - Err(RecvError::Closed) => Err(Error::Channel { send: false }), + Err(RecvError::Closed) => Err(crate::Error::Channel { send: false }), Err(RecvError::Lagged(n)) => { tracing::error!("cursor channel lagged behind, skipping {} events", n); Ok(stream.recv().await.expect("could not receive after lagging")) diff --git a/src/cursor/mod.rs b/src/cursor/mod.rs index 0e34aa7..5b5f5ff 100644 --- a/src/cursor/mod.rs +++ b/src/cursor/mod.rs @@ -12,7 +12,7 @@ pub mod controller; pub use controller::CursorController as Controller; -use crate::proto::{RowCol, CursorPosition}; +use crate::proto::cursor::{RowCol, CursorPosition}; impl From:: for (i32, i32) { fn from(pos: RowCol) -> (i32, i32) { @@ -36,12 +36,12 @@ impl RowCol { impl CursorPosition { /// extract start position, defaulting to (0,0), to help build protocol packets pub fn start(&self) -> RowCol { - self.start.clone().unwrap_or((0, 0).into()) + self.start.clone() } /// extract end position, defaulting to (0,0), to help build protocol packets pub fn end(&self) -> RowCol { - self.end.clone().unwrap_or((0, 0).into()) + self.end.clone() } } diff --git a/src/cursor/worker.rs b/src/cursor/worker.rs index e0196bd..08045be 100644 --- a/src/cursor/worker.rs +++ b/src/cursor/worker.rs @@ -1,14 +1,15 @@ use std::sync::Arc; use tokio::sync::{mpsc, broadcast::{self}, Mutex, watch}; -use tonic::{Streaming, transport::Channel, async_trait}; +use tonic::{Streaming, async_trait}; +use uuid::Uuid; -use crate::{proto::{cursor_client::CursorClient, CursorEvent}, errors::IgnorableError, api::controller::ControllerWorker}; +use crate::{api::controller::ControllerWorker, errors::IgnorableError, proto::cursor::CursorEvent}; use super::controller::CursorController; -pub(crate) struct CursorControllerWorker { - uid: String, +pub(crate) struct CursorWorker { + user_id: Uuid, producer: mpsc::UnboundedSender, op: mpsc::UnboundedReceiver, changed: watch::Sender, @@ -18,14 +19,14 @@ pub(crate) struct CursorControllerWorker { stop_control: mpsc::UnboundedSender<()>, } -impl CursorControllerWorker { - pub(crate) fn new(uid: String) -> Self { +impl CursorWorker { + pub(crate) fn new(user_id: Uuid) -> Self { let (op_tx, op_rx) = mpsc::unbounded_channel(); let (cur_tx, _cur_rx) = broadcast::channel(64); let (end_tx, end_rx) = mpsc::unbounded_channel(); let (change_tx, change_rx) = watch::channel(CursorEvent::default()); Self { - uid, + user_id, producer: op_tx, op: op_rx, changed: change_tx, @@ -40,12 +41,12 @@ impl CursorControllerWorker { #[async_trait] impl ControllerWorker for CursorControllerWorker { type Controller = CursorController; - type Tx = CursorClient; + type Tx = mpsc::Sender; type Rx = Streaming; fn subscribe(&self) -> CursorController { CursorController::new( - self.uid.clone(), + self.user_id.clone(), self.producer.clone(), Mutex::new(self.last_op.clone()), Mutex::new(self.channel.subscribe()), @@ -53,19 +54,18 @@ impl ControllerWorker for CursorControllerWorker { ) } - async fn work(mut self, mut tx: Self::Tx, mut rx: Self::Rx) { + async fn work(mut self, tx: Self::Tx, mut rx: Self::Rx) { loop { tokio::select!{ Ok(Some(cur)) = rx.message() => { - if cur.user == self.uid { continue } + if cur.user.id == self.user_id.to_string() { continue } self.channel.send(cur.clone()).unwrap_or_warn("could not broadcast event"); self.changed.send(cur).unwrap_or_warn("could not update last event"); }, - Some(op) = self.op.recv() => { tx.moved(op).await.unwrap_or_warn("could not update cursor"); }, + Some(op) = self.op.recv() => { tx.send(op).await.unwrap_or_warn("could not update cursor"); }, Some(()) = self.stop.recv() => { break; }, else => break, } } } } - diff --git a/src/errors.rs b/src/errors.rs index 5409ca2..ac4d616 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -109,6 +109,13 @@ impl From> for Error { } } +#[cfg(feature = "client")] +impl From> for Error { + fn from(_value: tokio::sync::watch::error::SendError) -> Self { + Error::Channel { send: true } + } +} + #[cfg(feature = "client")] impl From for Error { fn from(_value: tokio::sync::broadcast::error::RecvError) -> Self { diff --git a/src/lib.rs b/src/lib.rs index 1acd87c..74373c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -151,9 +151,11 @@ pub mod client; pub mod tools; /// client wrapper to handle memory persistence -#[cfg(feature = "client")] +#[cfg(feature = "backport")] pub mod instance; +pub mod workspace; + /// all-in-one imports : `use codemp::prelude::*;` pub mod prelude; @@ -181,6 +183,6 @@ pub use errors::Result; #[cfg(all(feature = "client", feature = "sync"))] pub use instance::sync::Instance; -#[cfg(all(feature = "client", not(feature = "sync")))] +#[cfg(all(feature = "backport", not(feature = "sync")))] pub use instance::a_sync::Instance; diff --git a/src/prelude.rs b/src/prelude.rs index 44170b9..836ace7 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -18,10 +18,10 @@ pub use crate::api::{ #[cfg(feature = "client")] pub use crate::{ - Instance as CodempInstance, + // Instance as CodempInstance, client::Client as CodempClient, cursor::Controller as CodempCursorController, - buffer::Controller as CodempBufferController, + // buffer::Controller as CodempBufferController, }; #[cfg(feature = "proto")] diff --git a/src/workspace.rs b/src/workspace.rs new file mode 100644 index 0000000..6a20afa --- /dev/null +++ b/src/workspace.rs @@ -0,0 +1,215 @@ +use std::{collections::{BTreeMap, BTreeSet}, str::FromStr, sync::Arc}; +use tokio::sync::mpsc; +use uuid::Uuid; +use crate::{ + proto::{user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, + api::controller::ControllerWorker, + buffer::{self, worker::BufferWorker}, + client::Services, + cursor +}; + +//TODO may contain more info in the future +#[derive(Debug, Clone)] +pub struct UserInfo { + pub uuid: Uuid +} + +impl From for UserInfo { + fn from(uuid: Uuid) -> Self { + UserInfo { + uuid + } + } +} + +impl From for Uuid { + fn from(uid: UserIdentity) -> Uuid { + Uuid::from_str(&uid.id).expect("expected an uuid") + } +} + +/// list_users -> A() , B() +/// get_user_info(B) -> B(cacca, pipu@piu) + +pub struct Workspace { + id: String, + user_id: Uuid, + token: Arc>, + cursor: Arc, + buffers: BTreeMap>, + filetree: BTreeSet, + users: BTreeMap, + services: Arc +} + +impl Workspace { + pub(crate) async fn new( + id: String, + user_id: Uuid, + token: Arc>, + cursor: Arc, + services: Arc + ) -> crate::Result { + let mut ws = Workspace { + id, + user_id, + token, + cursor, + buffers: BTreeMap::new(), + filetree: BTreeSet::new(), + users: BTreeMap::new(), + services + }; + + ws.fetch_buffers().await?; + ws.fetch_users().await?; + + Ok(ws) + } + + /// create a new buffer in current workspace, with optional given content + pub async fn create(&mut self, path: &str) -> crate::Result<()> { + let mut workspace_client = self.services.workspace.clone(); + workspace_client.create( + tonic::Request::new(BufferPayload { path: path.to_string() }) + ).await?; + + //add to filetree + self.filetree.insert(path.to_string()); + + Ok(()) + } + + /// attach to a buffer, starting a buffer controller and returning a new reference to it + /// + /// to interact with such buffer use [crate::api::Controller::send] or + /// [crate::api::Controller::recv] to exchange [crate::api::TextChange] + pub async fn attach(&mut self, path: &str) -> crate::Result> { + let mut worskspace_client = self.services.workspace.clone(); + self.token.send(worskspace_client.attach( + tonic::Request::new(AttachRequest { id: path.to_string() }) + ).await?.into_inner())?; + + let (tx, rx) = mpsc::channel(10); + let stream = self.services.buffer.clone() + .attach(tokio_stream::wrappers::ReceiverStream::new(rx)) + .await? + .into_inner(); + + let worker = BufferWorker::new(self.user_id, path); + let controller = Arc::new(worker.subscribe()); + tokio::spawn(async move { + tracing::debug!("controller worker started"); + worker.work(tx, stream).await; + tracing::debug!("controller worker stopped"); + }); + + self.buffers.insert(path.to_string(), controller.clone()); + + Ok(controller) + } + + pub async fn fetch_buffers(&mut self) -> crate::Result<()> { + let mut workspace_client = self.services.workspace.clone(); + let buffers = workspace_client.list_buffers( + tonic::Request::new(BufferListRequest {}) + ).await?.into_inner().buffers; + + self.filetree.clear(); + for b in buffers { + self.filetree.insert(b.path); + } + + Ok(()) + } + + pub async fn fetch_users(&mut self) -> crate::Result<()> { + let mut workspace_client = self.services.workspace.clone(); + let users = BTreeSet::from_iter(workspace_client.list_users( + tonic::Request::new(UserListRequest {}) + ).await?.into_inner().users.into_iter().map(Uuid::from)); + + // only keep userinfo for users that still exist + self.users.retain(|k, _v| users.contains(k)); + + let _users = self.users.clone(); // damnnn rust + users.iter() + .filter(|u| _users.contains_key(u)) + .for_each(|u| { self.users.insert(*u, UserInfo::from(*u)); }); + + Ok(()) + } + + pub async fn list_buffer_users() { + todo!(); //TODO what is this + } + + pub async fn delete(&mut self, path: &str) -> crate::Result<()> { + let mut workspace_client = self.services.workspace.clone(); + workspace_client.delete( + tonic::Request::new(BufferPayload { path: path.to_string() }) + ).await?; + + self.filetree.remove(path); + + Ok(()) + } + + /// leave current workspace if in one, disconnecting buffer and cursor controllers + pub fn leave_workspace(&self) { + todo!(); //TODO need proto + } + + /// disconnect from a specific buffer + pub fn disconnect_buffer(&mut self, path: &str) -> bool { + match &mut self.buffers.remove(path) { + None => false, + Some(_) => true + } + } + + pub fn id(&self) -> String { self.id.clone() } + + /// get a new reference to a buffer controller, if any is active to given path + pub fn buffer_by_name(&self, path: &str) -> Option> { + self.buffers.get(path).cloned() + } + + /// return a reference to current cursor controller, if currently in a workspace + pub fn cursor(&self) -> Arc { self.cursor.clone() } + +} + +/* +impl Interceptor for Workspace { //TODO + fn call(&mut self, mut request: tonic::Request<()>) -> Result, tonic::Status> { + request.metadata_mut().insert("auth", self.token.token.parse().unwrap()); + Ok(request) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum FSNode { + File(String), + Directory(String, Vec), +} + fn file_tree_rec(path: &str, root: &mut Vec) { + if let Some(idx) = path.find("/") { + let dir = path[..idx].to_string(); + let mut dir_node = vec![]; + Self::file_tree_rec(&path[idx..], &mut dir_node); + root.push(FSNode::Directory(dir, dir_node)); + } else { + root.push(FSNode::File(path.to_string())); + } + } + + fn file_tree(&self) -> Vec { + let mut root = vec![]; + for path in &self.filetree { + Self::file_tree_rec(&path, &mut root); + } + root + } +*/ \ No newline at end of file From 4c738e726cdbf315bd74599f404d1782644e36af Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 25 Jan 2024 02:31:02 +0100 Subject: [PATCH 14/42] chore: add .cargo to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6ab98ef..6b2fb8c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ Cargo.lock /client/vscode/*.vsix /client/vscode/codemp.node +.cargo \ No newline at end of file From f7bd5849be4505c4ebd6576c2c29dc6c85df6c9f Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 25 Jan 2024 03:25:45 +0100 Subject: [PATCH 15/42] fix: removed instance module, fixed prelude --- Cargo.toml | 1 - src/client.rs | 11 ++++++++--- src/cursor/controller.rs | 2 +- src/cursor/mod.rs | 21 +-------------------- src/lib.rs | 17 ++++------------- src/prelude.rs | 17 ++++++++--------- 6 files changed, 22 insertions(+), 47 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 575d071..d3abf75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,4 +40,3 @@ client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "d server = ["transport"] global = ["client", "dep:lazy_static"] sync = ["client"] -backport = [] # TODO remove! diff --git a/src/client.rs b/src/client.rs index fbfaf51..41b5c66 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,6 +2,7 @@ //! //! codemp client manager, containing grpc services +use std::collections::BTreeMap; use std::sync::Arc; use tokio::sync::mpsc; use tonic::service::interceptor::InterceptedService; @@ -25,7 +26,7 @@ use crate::workspace::Workspace; pub struct Client { user_id: Uuid, token_tx: Arc>, - workspace: Option, + pub workspaces: BTreeMap, services: Arc } @@ -83,7 +84,7 @@ impl Client { Ok(Client { user_id, token_tx: Arc::new(token_tx), - workspace: None, + workspaces: BTreeMap::new(), services: Arc::new(Services { workspace, buffer, cursor }) }) } @@ -111,7 +112,7 @@ impl Client { tracing::debug!("controller worker stopped"); }); - self.workspace = Some(Workspace::new( + self.workspaces.insert(workspace_id.to_string(), Workspace::new( workspace_id.to_string(), self.user_id, self.token_tx.clone(), @@ -121,4 +122,8 @@ impl Client { Ok(()) } + + pub fn user_id(&self) -> Uuid { + self.user_id.clone() + } } diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index 3e793ac..af1f783 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -53,7 +53,7 @@ impl Controller for CursorController { /// enqueue a cursor event to be broadcast to current workspace /// will automatically invert cursor start/end if they are inverted fn send(&self, mut cursor: CursorPosition) -> crate::Result<()> { - if cursor.start() > cursor.end() { + if cursor.start > cursor.end { std::mem::swap(&mut cursor.start, &mut cursor.end); } Ok(self.op.send(CursorEvent { diff --git a/src/cursor/mod.rs b/src/cursor/mod.rs index 5b5f5ff..59d68f3 100644 --- a/src/cursor/mod.rs +++ b/src/cursor/mod.rs @@ -12,7 +12,7 @@ pub mod controller; pub use controller::CursorController as Controller; -use crate::proto::cursor::{RowCol, CursorPosition}; +use crate::proto::cursor::RowCol; impl From:: for (i32, i32) { fn from(pos: RowCol) -> (i32, i32) { @@ -26,25 +26,6 @@ impl From::<(i32, i32)> for RowCol { } } -impl RowCol { - /// create a RowCol and wrap into an Option, to help build protocol packets - pub fn wrap(row: i32, col: i32) -> Option { - Some(RowCol { row, col }) - } -} - -impl CursorPosition { - /// extract start position, defaulting to (0,0), to help build protocol packets - pub fn start(&self) -> RowCol { - self.start.clone() - } - - /// extract end position, defaulting to (0,0), to help build protocol packets - pub fn end(&self) -> RowCol { - self.end.clone() - } -} - impl PartialOrd for RowCol { fn partial_cmp(&self, other: &Self) -> Option { match self.row.partial_cmp(&other.row) { diff --git a/src/lib.rs b/src/lib.rs index 74373c1..c305f03 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -148,12 +148,11 @@ pub mod errors; #[cfg(feature = "client")] pub mod client; +/// assorted helpers pub mod tools; -/// client wrapper to handle memory persistence -#[cfg(feature = "backport")] -pub mod instance; - +/// workspace operations +#[cfg(feature = "client")] pub mod workspace; /// all-in-one imports : `use codemp::prelude::*;` @@ -176,13 +175,5 @@ pub mod proto { pub mod workspace_service { tonic::include_proto!("workspace_service"); } } - pub use errors::Error; -pub use errors::Result; - -#[cfg(all(feature = "client", feature = "sync"))] -pub use instance::sync::Instance; - -#[cfg(all(feature = "backport", not(feature = "sync")))] -pub use instance::a_sync::Instance; - +pub use errors::Result; \ No newline at end of file diff --git a/src/prelude.rs b/src/prelude.rs index 836ace7..3a2934b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -20,16 +20,15 @@ pub use crate::api::{ pub use crate::{ // Instance as CodempInstance, client::Client as CodempClient, + workspace::Workspace as CodempWorkspace, + workspace::UserInfo as CodempUserInfo, cursor::Controller as CodempCursorController, - // buffer::Controller as CodempBufferController, + buffer::Controller as CodempBufferController, }; -#[cfg(feature = "proto")] +#[cfg(feature = "transport")] pub use crate::{ - proto::CursorPosition as CodempCursorPosition, - proto::CursorEvent as CodempCursorEvent, - proto::RowCol as CodempRowCol, -}; - -#[cfg(feature = "global")] -pub use crate::instance::global::INSTANCE as CODEMP_INSTANCE; + proto::cursor::CursorPosition as CodempCursorPosition, + proto::cursor::CursorEvent as CodempCursorEvent, + proto::cursor::RowCol as CodempRowCol, +}; \ No newline at end of file From 6a061ca4324cfd22af1702723826044c1a182eca Mon Sep 17 00:00:00 2001 From: frelodev Date: Thu, 25 Jan 2024 16:08:35 +0100 Subject: [PATCH 16/42] feat: LeaveWorkspace, CreateWorkspace, ListBufferUsers --- proto/workspace.proto | 6 +++++- proto/workspace_service.proto | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/proto/workspace.proto b/proto/workspace.proto index b076cf3..192b813 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -16,7 +16,7 @@ message WorkspaceUserList { } message WorkspaceMessage { - required int32 id = 1; + required int32 id = 1; //unused??? } message JoinRequest { @@ -65,4 +65,8 @@ message BufferListRequest{ message UserList { repeated user.UserIdentity users = 1; +} + +message WorkspaceDetails{ + required int32 id=1; } \ No newline at end of file diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 55857a6..108f5c1 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -33,20 +33,26 @@ import "workspace.proto"; service Workspace { - rpc Create (workspace.BufferPayload) returns (workspace.Empty); rpc Attach (workspace.AttachRequest) returns (workspace.Token); + rpc LeaveWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); + + rpc CreateWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); + + rpc CreateBuffer (workspace.BufferPayload) returns (workspace.Empty); + rpc ListBuffers (workspace.BufferListRequest) returns (files.BufferTree); rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); - rpc ListBufferUsers (workspace.Empty) returns (workspace.Empty); + rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.Empty); rpc Join (workspace.JoinRequest) returns (workspace.Token); rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer + } From 741a07446414ea172cc82f7afe78c21bc1e89d8c Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 25 Jan 2024 16:31:38 +0100 Subject: [PATCH 17/42] feat: implemented leave workspace and list buffer users, various fixes --- Cargo.toml | 2 +- proto/files.proto | 2 +- proto/user.proto | 2 +- proto/workspace.proto | 8 +-- proto/workspace_service.proto | 34 +------------ src/api/change.rs | 4 +- src/buffer/mod.rs | 2 +- src/buffer/worker.rs | 2 +- src/client.rs | 55 ++++++++++++++------ src/cursor/controller.rs | 2 +- src/cursor/worker.rs | 4 +- src/tools.rs | 2 +- src/workspace.rs | 94 ++++++++++++----------------------- 13 files changed, 88 insertions(+), 125 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d3abf75..7cca7ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ name = "codemp" # core tracing = "0.1" # woot -codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.0", optional = true } +codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.1", optional = true } # proto tonic = { version = "0.9", features = ["tls", "tls-roots"], optional = true } prost = { version = "0.11.8", optional = true } diff --git a/proto/files.proto b/proto/files.proto index e57cdc2..a7ab7d2 100644 --- a/proto/files.proto +++ b/proto/files.proto @@ -14,4 +14,4 @@ message BufferTree { message WorkspaceFileTree { // list of strings may be more efficient but it's a lot more hassle required string payload = 1; // spappolata di json -} //Alla fine non si usa questo per ora ma BufferTree \ No newline at end of file +} \ No newline at end of file diff --git a/proto/user.proto b/proto/user.proto index 322a935..2fa52ac 100644 --- a/proto/user.proto +++ b/proto/user.proto @@ -6,5 +6,5 @@ package user; // payload identifying user message UserIdentity { // user identifier - required string id = 1; + required bytes id = 1; //since uuid is 8 bytes we prefer to just send the raw bytes instead of string } diff --git a/proto/workspace.proto b/proto/workspace.proto index 192b813..d4c1a01 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -20,8 +20,8 @@ message WorkspaceMessage { } message JoinRequest { - required string username=1; - required string password=2; + required string username = 1; + required string password = 2; } message AttachRequest { @@ -67,6 +67,6 @@ message UserList { repeated user.UserIdentity users = 1; } -message WorkspaceDetails{ - required int32 id=1; +message WorkspaceDetails { + required string id = 1; } \ No newline at end of file diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 108f5c1..3fe7c2e 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -1,27 +1,3 @@ -// Workspace effimero: sta in /tmp o proprio in memoria -// Workspace e` autenticato: come si decide mentre si rifa il server -// Workspace ha id univoco (stringa), usato per connettercisi -// Workspace implementera` access control: -// * accedere al workspace -// * i singoli buffer -// - i metadati maybe???? -// Workspace offre le seguenti features: -// * listare i buffer DONE -// * listare gli user connessi DONE -// * creare buffers DONE REPLACE THE ONE ON buffer.proto -// * NO ATTACH: responsabilita` del buffer service -// * contiene metadata dei buffers: -// * path -// * data creazione -// Buffer id NON E` il path DONE -// BufferService NON ha metadata: -// Workspace tiene traccia di utenti attached (nel futuro) DONE - - - - - - syntax = "proto2"; package workspace_service; @@ -29,11 +5,8 @@ import "user.proto"; import "files.proto"; import "workspace.proto"; - - service Workspace { - rpc Attach (workspace.AttachRequest) returns (workspace.Token); rpc LeaveWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); @@ -46,13 +19,10 @@ service Workspace { rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); - rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.Empty); + rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.UserList); //TODO discuss rpc Join (workspace.JoinRequest) returns (workspace.Token); rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer - -} - - +} \ No newline at end of file diff --git a/src/api/change.rs b/src/api/change.rs index e9e6792..041e628 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -84,11 +84,11 @@ impl TextChange { /// convert from byte index to row and column /// txt must be the whole content of the buffer, in order to count lines - pub fn index_to_rowcol(txt: &str, index: usize) -> crate::proto::RowCol { + pub fn index_to_rowcol(txt: &str, index: usize) -> RowCol { // FIXME might panic, use .get() let row = txt[..index].matches('\n').count() as i32; let col = txt[..index].split('\n').last().unwrap_or("").len() as i32; - crate::proto::RowCol { row, col } + RowCol { row, col } } } diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 1610400..7722e9b 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -11,4 +11,4 @@ pub mod controller; pub(crate) mod worker; -pub use controller::BufferController as Controller; +pub use controller::BufferController as Controller; \ No newline at end of file diff --git a/src/buffer/worker.rs b/src/buffer/worker.rs index c2e77bc..031fc57 100644 --- a/src/buffer/worker.rs +++ b/src/buffer/worker.rs @@ -63,7 +63,7 @@ impl BufferWorker { } #[async_trait] -impl ControllerWorker for BufferControllerWorker { +impl ControllerWorker for BufferWorker { type Controller = BufferController; type Tx = mpsc::Sender; type Rx = Streaming; diff --git a/src/client.rs b/src/client.rs index 41b5c66..35b3faa 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,7 +4,8 @@ use std::collections::BTreeMap; use std::sync::Arc; -use tokio::sync::mpsc; + +use tokio::sync::{mpsc, RwLock}; use tonic::service::interceptor::InterceptedService; use tonic::service::Interceptor; use tonic::transport::{Channel, Endpoint}; @@ -14,7 +15,7 @@ use crate::api::controller::ControllerWorker; use crate::cursor::worker::CursorWorker; use crate::proto::buffer_service::buffer_client::BufferClient; use crate::proto::cursor_service::cursor_client::CursorClient; -use crate::proto::workspace::{JoinRequest, Token}; +use crate::proto::workspace::{JoinRequest, Token, WorkspaceDetails}; use crate::proto::workspace_service::workspace_client::WorkspaceClient; use crate::workspace::Workspace; @@ -26,7 +27,7 @@ use crate::workspace::Workspace; pub struct Client { user_id: Uuid, token_tx: Arc>, - pub workspaces: BTreeMap, + pub workspaces: BTreeMap>>, services: Arc } @@ -62,7 +63,7 @@ fn parse_codemp_connection_string<'a>(string: &'a str) -> (String, String) { impl Client { /// instantiate and connect a new client - pub async fn new(dest: &str) -> crate::Result { //TODO interceptor + pub async fn new(dest: &str) -> crate::Result { let (_host, _workspace_id) = parse_codemp_connection_string(dest); let channel = Endpoint::from_shared(dest.to_string())? @@ -89,11 +90,18 @@ impl Client { }) } - /// join a workspace, starting a cursorcontroller and returning a new reference to it - /// - /// to interact with such workspace [crate::api::Controller::send] cursor events or - /// [crate::api::Controller::recv] for events on the associated [crate::cursor::Controller]. - pub async fn join(&mut self, workspace_id: &str) -> crate::Result<()> { + /// creates a new workspace (and joins it implicitly), returns an [tokio::sync::RwLock] to interact with it + pub async fn create_workspace(&mut self, workspace_id: &str) -> crate::Result>> { + let mut workspace_client = self.services.workspace.clone(); + workspace_client.create_workspace( + tonic::Request::new(WorkspaceDetails { id: workspace_id.to_string() }) + ).await?; + + self.join_workspace(workspace_id).await + } + + /// join a workspace, returns an [tokio::sync::RwLock] to interact with it + pub async fn join_workspace(&mut self, workspace_id: &str) -> crate::Result>> { self.token_tx.send(self.services.workspace.clone().join( tonic::Request::new(JoinRequest { username: "".to_string(), password: "".to_string() }) //TODO ).await?.into_inner())?; @@ -112,17 +120,32 @@ impl Client { tracing::debug!("controller worker stopped"); }); - self.workspaces.insert(workspace_id.to_string(), Workspace::new( - workspace_id.to_string(), - self.user_id, - self.token_tx.clone(), - controller, - self.services.clone() - ).await?); + let lock = Arc::new(RwLock::new( + Workspace::new( + workspace_id.to_string(), + self.user_id, + self.token_tx.clone(), + controller, + self.services.clone() + ).await? + )); + self.workspaces.insert(workspace_id.to_string(), lock.clone()); + + Ok(lock) + } + + /// leave given workspace, disconnecting buffer and cursor controllers + pub async fn leave_workspace(&self, workspace_id: &str) -> crate::Result<()> { + let mut workspace_client = self.services.workspace.clone(); + workspace_client.leave_workspace( + tonic::Request::new(WorkspaceDetails { id: workspace_id.to_string() }) + ).await?; + Ok(()) } + /// accessor for user id pub fn user_id(&self) -> Uuid { self.user_id.clone() } diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index af1f783..2dd5cd2 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -57,7 +57,7 @@ impl Controller for CursorController { std::mem::swap(&mut cursor.start, &mut cursor.end); } Ok(self.op.send(CursorEvent { - user: UserIdentity { id: self.user_id.to_string() }, + user: UserIdentity { id: self.user_id.as_bytes().to_vec() }, position: cursor, })?) } diff --git a/src/cursor/worker.rs b/src/cursor/worker.rs index 08045be..1d27c6b 100644 --- a/src/cursor/worker.rs +++ b/src/cursor/worker.rs @@ -39,7 +39,7 @@ impl CursorWorker { } #[async_trait] -impl ControllerWorker for CursorControllerWorker { +impl ControllerWorker for CursorWorker { type Controller = CursorController; type Tx = mpsc::Sender; type Rx = Streaming; @@ -58,7 +58,7 @@ impl ControllerWorker for CursorControllerWorker { loop { tokio::select!{ Ok(Some(cur)) = rx.message() => { - if cur.user.id == self.user_id.to_string() { continue } + if Uuid::from(cur.user.clone()) == self.user_id { continue } self.channel.send(cur.clone()).unwrap_or_warn("could not broadcast event"); self.changed.send(cur).unwrap_or_warn("could not update last event"); }, diff --git a/src/tools.rs b/src/tools.rs index 5e1870d..bf5d997 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -47,4 +47,4 @@ pub async fn select_buffer( }, } } -} +} \ No newline at end of file diff --git a/src/workspace.rs b/src/workspace.rs index 6a20afa..d03e43a 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -1,4 +1,4 @@ -use std::{collections::{BTreeMap, BTreeSet}, str::FromStr, sync::Arc}; +use std::{collections::{BTreeMap, BTreeSet}, sync::Arc}; use tokio::sync::mpsc; use uuid::Uuid; use crate::{ @@ -25,13 +25,11 @@ impl From for UserInfo { impl From for Uuid { fn from(uid: UserIdentity) -> Uuid { - Uuid::from_str(&uid.id).expect("expected an uuid") + let b: [u8; 16] = uid.id.try_into().expect("expected an uuid"); + Uuid::from_bytes(b) } } -/// list_users -> A() , B() -/// get_user_info(B) -> B(cacca, pipu@piu) - pub struct Workspace { id: String, user_id: Uuid, @@ -68,10 +66,10 @@ impl Workspace { Ok(ws) } - /// create a new buffer in current workspace, with optional given content + /// create a new buffer in current workspace pub async fn create(&mut self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); - workspace_client.create( + workspace_client.create_buffer( tonic::Request::new(BufferPayload { path: path.to_string() }) ).await?; @@ -110,6 +108,7 @@ impl Workspace { Ok(controller) } + /// fetch a list of all buffers in a workspace pub async fn fetch_buffers(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let buffers = workspace_client.list_buffers( @@ -124,6 +123,7 @@ impl Workspace { Ok(()) } + /// fetch a list of all users in a workspace pub async fn fetch_users(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let users = BTreeSet::from_iter(workspace_client.list_users( @@ -141,75 +141,45 @@ impl Workspace { Ok(()) } - pub async fn list_buffer_users() { - todo!(); //TODO what is this + /// get a list of the users attached to a specific buffer + /// + /// TODO: discuss implementation details + pub async fn list_buffer_users(&mut self, path: &str) -> crate::Result> { + let mut workspace_client = self.services.workspace.clone(); + let buffer_users = workspace_client.list_buffer_users( + tonic::Request::new(BufferPayload { path: path.to_string() }) + ).await?.into_inner().users; + + Ok(buffer_users) + } + + /// detach from a specific buffer, returns false if there + pub fn detach(&mut self, path: &str) -> bool { + match &mut self.buffers.remove(path) { + None => false, + Some(_) => true + } } + /// delete a buffer pub async fn delete(&mut self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); workspace_client.delete( tonic::Request::new(BufferPayload { path: path.to_string() }) ).await?; - + self.filetree.remove(path); - + Ok(()) } - - /// leave current workspace if in one, disconnecting buffer and cursor controllers - pub fn leave_workspace(&self) { - todo!(); //TODO need proto - } - - /// disconnect from a specific buffer - pub fn disconnect_buffer(&mut self, path: &str) -> bool { - match &mut self.buffers.remove(path) { - None => false, - Some(_) => true - } - } pub fn id(&self) -> String { self.id.clone() } + /// return a reference to current cursor controller, if currently in a workspace + pub fn cursor(&self) -> Arc { self.cursor.clone() } + /// get a new reference to a buffer controller, if any is active to given path pub fn buffer_by_name(&self, path: &str) -> Option> { self.buffers.get(path).cloned() } - - /// return a reference to current cursor controller, if currently in a workspace - pub fn cursor(&self) -> Arc { self.cursor.clone() } - -} - -/* -impl Interceptor for Workspace { //TODO - fn call(&mut self, mut request: tonic::Request<()>) -> Result, tonic::Status> { - request.metadata_mut().insert("auth", self.token.token.parse().unwrap()); - Ok(request) - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub enum FSNode { - File(String), - Directory(String, Vec), -} - fn file_tree_rec(path: &str, root: &mut Vec) { - if let Some(idx) = path.find("/") { - let dir = path[..idx].to_string(); - let mut dir_node = vec![]; - Self::file_tree_rec(&path[idx..], &mut dir_node); - root.push(FSNode::Directory(dir, dir_node)); - } else { - root.push(FSNode::File(path.to_string())); - } - } - - fn file_tree(&self) -> Vec { - let mut root = vec![]; - for path in &self.filetree { - Self::file_tree_rec(&path, &mut root); - } - root - } -*/ \ No newline at end of file +} \ No newline at end of file From 6fe1b213bd95bf2cda0314f22399b9951e6ed504 Mon Sep 17 00:00:00 2001 From: frelodev Date: Fri, 26 Jan 2024 21:39:21 +0100 Subject: [PATCH 18/42] feat: snapshots --- proto/buffer_service.proto | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/proto/buffer_service.proto b/proto/buffer_service.proto index 96ea355..d7d989f 100644 --- a/proto/buffer_service.proto +++ b/proto/buffer_service.proto @@ -6,10 +6,20 @@ package buffer_service; service Buffer { // attach to a buffer and receive operations rpc Attach (stream Operation) returns (stream Operation); + + rpc Snapshot(SnapshotRequest) returns (SnapshotResponse); + } message Operation { required bytes data = 1; optional string user = 2; optional string path = 3; +} + +message SnapshotRequest { + required string path = 1; +} +message SnapshotResponse { + required string content = 1; } \ No newline at end of file From f7fcf8bd22e918007b0c3034807563907223ecf7 Mon Sep 17 00:00:00 2001 From: zaaarf Date: Sat, 27 Jan 2024 11:49:05 +0100 Subject: [PATCH 19/42] feat: implemented snapshot method and some getters --- .gitignore | 4 +++- proto/buffer_service.proto | 1 + proto/workspace.proto | 4 ++-- src/workspace.rs | 21 +++++++++++++++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 6b2fb8c..2339481 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ Cargo.lock /client/vscode/*.vsix /client/vscode/codemp.node -.cargo \ No newline at end of file +.cargo + +.vscode/ \ No newline at end of file diff --git a/proto/buffer_service.proto b/proto/buffer_service.proto index d7d989f..0e6e0ca 100644 --- a/proto/buffer_service.proto +++ b/proto/buffer_service.proto @@ -20,6 +20,7 @@ message Operation { message SnapshotRequest { required string path = 1; } + message SnapshotResponse { required string content = 1; } \ No newline at end of file diff --git a/proto/workspace.proto b/proto/workspace.proto index d4c1a01..3bb467d 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -16,7 +16,7 @@ message WorkspaceUserList { } message WorkspaceMessage { - required int32 id = 1; //unused??? + required string id = 1; //unused??? } message JoinRequest { @@ -25,7 +25,7 @@ message JoinRequest { } message AttachRequest { - required string id = 1; + required string path = 1; } message Token { diff --git a/src/workspace.rs b/src/workspace.rs index d03e43a..4e84326 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -2,7 +2,7 @@ use std::{collections::{BTreeMap, BTreeSet}, sync::Arc}; use tokio::sync::mpsc; use uuid::Uuid; use crate::{ - proto::{user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, + proto::{buffer_service::SnapshotRequest, user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, api::controller::ControllerWorker, buffer::{self, worker::BufferWorker}, client::Services, @@ -42,6 +42,7 @@ pub struct Workspace { } impl Workspace { + /// create a new buffer and perform initial fetch operations pub(crate) async fn new( id: String, user_id: Uuid, @@ -86,7 +87,7 @@ impl Workspace { pub async fn attach(&mut self, path: &str) -> crate::Result> { let mut worskspace_client = self.services.workspace.clone(); self.token.send(worskspace_client.attach( - tonic::Request::new(AttachRequest { id: path.to_string() }) + tonic::Request::new(AttachRequest { path: path.to_string() }) ).await?.into_inner())?; let (tx, rx) = mpsc::channel(10); @@ -108,6 +109,16 @@ impl Workspace { Ok(controller) } + /// get a snapshot of a buffer (meaning its contents as a flat string) + pub async fn snapshot(&mut self, path: &str) -> crate::Result { + let mut buffer_client = self.services.buffer.clone(); + let contents = buffer_client.snapshot( + tonic::Request::new(SnapshotRequest { path: path.to_string() }) + ).await?.into_inner().content; + + Ok(contents) + } + /// fetch a list of all buffers in a workspace pub async fn fetch_buffers(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); @@ -173,6 +184,7 @@ impl Workspace { Ok(()) } + /// get the id of the workspace pub fn id(&self) -> String { self.id.clone() } /// return a reference to current cursor controller, if currently in a workspace @@ -182,4 +194,9 @@ impl Workspace { pub fn buffer_by_name(&self, path: &str) -> Option> { self.buffers.get(path).cloned() } + + /// get the currently cached "filetree" + pub fn filetree(&self) -> Vec { + self.filetree.iter().map(|f| f.clone()).collect() + } } \ No newline at end of file From bc3df45726697eeb16da44054197e0eb462c8b0d Mon Sep 17 00:00:00 2001 From: zaaarf Date: Wed, 31 Jan 2024 23:57:21 +0100 Subject: [PATCH 20/42] fix: fetch buffers after creating --- src/workspace.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/workspace.rs b/src/workspace.rs index 4e84326..4fe18b6 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -74,9 +74,12 @@ impl Workspace { tonic::Request::new(BufferPayload { path: path.to_string() }) ).await?; - //add to filetree + // add to filetree self.filetree.insert(path.to_string()); + // fetch buffers + self.fetch_buffers().await?; + Ok(()) } @@ -110,7 +113,7 @@ impl Workspace { } /// get a snapshot of a buffer (meaning its contents as a flat string) - pub async fn snapshot(&mut self, path: &str) -> crate::Result { + pub async fn snapshot(&self, path: &str) -> crate::Result { let mut buffer_client = self.services.buffer.clone(); let contents = buffer_client.snapshot( tonic::Request::new(SnapshotRequest { path: path.to_string() }) From 97061524e73d9810a944b7eee7031ae7fedfc53d Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 1 Feb 2024 01:58:27 +0100 Subject: [PATCH 21/42] chore: remove snapshot --- proto/buffer_service.proto | 13 +------------ proto/files.proto | 1 - proto/workspace.proto | 5 +---- proto/workspace_service.proto | 10 ---------- src/workspace.rs | 12 +----------- 5 files changed, 3 insertions(+), 38 deletions(-) diff --git a/proto/buffer_service.proto b/proto/buffer_service.proto index 0e6e0ca..a10250a 100644 --- a/proto/buffer_service.proto +++ b/proto/buffer_service.proto @@ -5,22 +5,11 @@ package buffer_service; // handle buffer changes, keep in sync users service Buffer { // attach to a buffer and receive operations - rpc Attach (stream Operation) returns (stream Operation); - - rpc Snapshot(SnapshotRequest) returns (SnapshotResponse); - + rpc Attach (stream Operation) returns (stream Operation); } message Operation { required bytes data = 1; optional string user = 2; optional string path = 3; -} - -message SnapshotRequest { - required string path = 1; -} - -message SnapshotResponse { - required string content = 1; } \ No newline at end of file diff --git a/proto/files.proto b/proto/files.proto index a7ab7d2..4423582 100644 --- a/proto/files.proto +++ b/proto/files.proto @@ -2,7 +2,6 @@ syntax = "proto2"; package files; - message BufferNode { required string path = 1; } diff --git a/proto/workspace.proto b/proto/workspace.proto index 3bb467d..b333a32 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -40,7 +40,6 @@ enum FileEventType { message FileEvent { required string bufferbuffertree = 1; - required FileEventType type = 2; } @@ -59,9 +58,7 @@ message BufferPayload { required string path = 1; } -message BufferListRequest{ - -} +message BufferListRequest {} message UserList { repeated user.UserIdentity users = 1; diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 3fe7c2e..54fe77b 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -6,23 +6,13 @@ import "files.proto"; import "workspace.proto"; service Workspace { - rpc Attach (workspace.AttachRequest) returns (workspace.Token); - rpc LeaveWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); - rpc CreateWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); - rpc CreateBuffer (workspace.BufferPayload) returns (workspace.Empty); - rpc ListBuffers (workspace.BufferListRequest) returns (files.BufferTree); - rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); - rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.UserList); //TODO discuss - rpc Join (workspace.JoinRequest) returns (workspace.Token); - rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer - } \ No newline at end of file diff --git a/src/workspace.rs b/src/workspace.rs index 4fe18b6..79da51c 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -2,7 +2,7 @@ use std::{collections::{BTreeMap, BTreeSet}, sync::Arc}; use tokio::sync::mpsc; use uuid::Uuid; use crate::{ - proto::{buffer_service::SnapshotRequest, user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, + proto::{user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, api::controller::ControllerWorker, buffer::{self, worker::BufferWorker}, client::Services, @@ -112,16 +112,6 @@ impl Workspace { Ok(controller) } - /// get a snapshot of a buffer (meaning its contents as a flat string) - pub async fn snapshot(&self, path: &str) -> crate::Result { - let mut buffer_client = self.services.buffer.clone(); - let contents = buffer_client.snapshot( - tonic::Request::new(SnapshotRequest { path: path.to_string() }) - ).await?.into_inner().content; - - Ok(contents) - } - /// fetch a list of all buffers in a workspace pub async fn fetch_buffers(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); From 164e9887b8a848821a015621975efc9e585eb311 Mon Sep 17 00:00:00 2001 From: zaaarf Date: Thu, 1 Feb 2024 03:19:27 +0100 Subject: [PATCH 22/42] fix: serialize uuid as string when sending --- proto/user.proto | 2 +- src/cursor/controller.rs | 2 +- src/workspace.rs | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/proto/user.proto b/proto/user.proto index 2fa52ac..322a935 100644 --- a/proto/user.proto +++ b/proto/user.proto @@ -6,5 +6,5 @@ package user; // payload identifying user message UserIdentity { // user identifier - required bytes id = 1; //since uuid is 8 bytes we prefer to just send the raw bytes instead of string + required string id = 1; } diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index 2dd5cd2..af1f783 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -57,7 +57,7 @@ impl Controller for CursorController { std::mem::swap(&mut cursor.start, &mut cursor.end); } Ok(self.op.send(CursorEvent { - user: UserIdentity { id: self.user_id.as_bytes().to_vec() }, + user: UserIdentity { id: self.user_id.to_string() }, position: cursor, })?) } diff --git a/src/workspace.rs b/src/workspace.rs index 79da51c..1e00f5e 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -1,4 +1,4 @@ -use std::{collections::{BTreeMap, BTreeSet}, sync::Arc}; +use std::{collections::{BTreeMap, BTreeSet}, str::FromStr, sync::Arc}; use tokio::sync::mpsc; use uuid::Uuid; use crate::{ @@ -25,8 +25,7 @@ impl From for UserInfo { impl From for Uuid { fn from(uid: UserIdentity) -> Uuid { - let b: [u8; 16] = uid.id.try_into().expect("expected an uuid"); - Uuid::from_bytes(b) + Uuid::from_str(&uid.id).expect("expected an uuid") } } @@ -89,9 +88,9 @@ impl Workspace { /// [crate::api::Controller::recv] to exchange [crate::api::TextChange] pub async fn attach(&mut self, path: &str) -> crate::Result> { let mut worskspace_client = self.services.workspace.clone(); - self.token.send(worskspace_client.attach( - tonic::Request::new(AttachRequest { path: path.to_string() }) - ).await?.into_inner())?; + let mut request = tonic::Request::new(AttachRequest { path: path.to_string() }); + request.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(path).expect("could not represent path as byte sequence")); + self.token.send(worskspace_client.attach(request).await?.into_inner())?; let (tx, rx) = mpsc::channel(10); let stream = self.services.buffer.clone() From 1ee185b5ec88149603fe53f469b2c247b07b1f05 Mon Sep 17 00:00:00 2001 From: alemi Date: Thu, 1 Feb 2024 17:54:36 +0100 Subject: [PATCH 23/42] feat: add auth service sneaked in to help with server development --- build.rs | 9 +++++---- proto/auth_service.proto | 19 +++++++++++++++++++ src/lib.rs | 3 ++- 3 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 proto/auth_service.proto diff --git a/build.rs b/build.rs index b9f9a1c..c035671 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,7 @@ fn main() -> Result<(), Box> { tonic_build::configure() // .build_client(cfg!(feature = "client")) - //.build_server(cfg!(feature = "server")) // FIXME if false, build fails???? + // .build_server(cfg!(feature = "server")) // FIXME if false, build fails???? // .build_transport(cfg!(feature = "transport")) .compile( &[ @@ -11,9 +11,10 @@ fn main() -> Result<(), Box> { "proto/workspace.proto", "proto/buffer_service.proto", "proto/cursor_service.proto", - "proto/workspace_service.proto" + "proto/workspace_service.proto", + "proto/auth_service.proto", ], - &["proto"] + &["proto"], )?; Ok(()) - } \ No newline at end of file + } diff --git a/proto/auth_service.proto b/proto/auth_service.proto new file mode 100644 index 0000000..4d5ef47 --- /dev/null +++ b/proto/auth_service.proto @@ -0,0 +1,19 @@ +syntax = "proto2"; + +package auth_service; + +// authenticates users, issuing tokens +service Auth { + // send credentials and join a workspace + rpc Login (WorkspaceJoinRequest) returns (Token); +} + +message Token { + required string token = 1; +} + +message WorkspaceJoinRequest { + required string workspace_id = 1; + required string username = 2; + required string password = 3; +} diff --git a/src/lib.rs b/src/lib.rs index c305f03..bcceca7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -173,7 +173,8 @@ pub mod proto { pub mod buffer_service { tonic::include_proto!("buffer_service"); } pub mod cursor_service { tonic::include_proto!("cursor_service"); } pub mod workspace_service { tonic::include_proto!("workspace_service"); } + pub mod auth_service { tonic::include_proto!("auth_service"); } } pub use errors::Error; -pub use errors::Result; \ No newline at end of file +pub use errors::Result; From 4ae31df3ffd7958052020bbc547fd4b88cf1d243 Mon Sep 17 00:00:00 2001 From: alemi Date: Thu, 1 Feb 2024 17:54:56 +0100 Subject: [PATCH 24/42] chore: moved select_buffer under buffer::tools needed to more appropriately subdivide feature flags --- src/buffer/mod.rs | 5 ++++- src/{ => buffer}/tools.rs | 2 +- src/lib.rs | 3 --- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/{ => buffer}/tools.rs (99%) diff --git a/src/buffer/mod.rs b/src/buffer/mod.rs index 7722e9b..dfaff9f 100644 --- a/src/buffer/mod.rs +++ b/src/buffer/mod.rs @@ -9,6 +9,9 @@ /// buffer controller implementation pub mod controller; +/// assorted helpers to handle buffer controllers +pub mod tools; + pub(crate) mod worker; -pub use controller::BufferController as Controller; \ No newline at end of file +pub use controller::BufferController as Controller; diff --git a/src/tools.rs b/src/buffer/tools.rs similarity index 99% rename from src/tools.rs rename to src/buffer/tools.rs index bf5d997..5e1870d 100644 --- a/src/tools.rs +++ b/src/buffer/tools.rs @@ -47,4 +47,4 @@ pub async fn select_buffer( }, } } -} \ No newline at end of file +} diff --git a/src/lib.rs b/src/lib.rs index bcceca7..1b8e3be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -148,9 +148,6 @@ pub mod errors; #[cfg(feature = "client")] pub mod client; -/// assorted helpers -pub mod tools; - /// workspace operations #[cfg(feature = "client")] pub mod workspace; From d14f004f735ecbd4107f672d5f594aa30990be51 Mon Sep 17 00:00:00 2001 From: Camillo Schenone Date: Sun, 4 Feb 2024 13:25:26 +0100 Subject: [PATCH 25/42] feat: adding an initial proto version for the workspace event stream --- proto/workspace.proto | 16 +++++++++++++++- proto/workspace_service.proto | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/proto/workspace.proto b/proto/workspace.proto index b333a32..bafda01 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -19,6 +19,20 @@ message WorkspaceMessage { required string id = 1; //unused??? } +enum WorkspaceEventType { + USER_JOIN = 0; + USER_LEAVE = 1; + FILE_CREATE = 2; + FILE_DELETE = 3; + FILE_RENAME = 4; +} + +message WorkspaceEvent { + required string id = 1; // the id of the user, or the path of the file + required WorkspaceEventType event = 2; + optional string extra = 3; // new name? extra info on user events etc... +} + message JoinRequest { required string username = 1; required string password = 2; @@ -66,4 +80,4 @@ message UserList { message WorkspaceDetails { required string id = 1; -} \ No newline at end of file +} diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto index 54fe77b..597dc5a 100644 --- a/proto/workspace_service.proto +++ b/proto/workspace_service.proto @@ -13,6 +13,6 @@ service Workspace { rpc ListBuffers (workspace.BufferListRequest) returns (files.BufferTree); rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.UserList); //TODO discuss - rpc Join (workspace.JoinRequest) returns (workspace.Token); + rpc Join (workspace.JoinRequest) returns (stream workspace.WorkspaceEvent); rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer -} \ No newline at end of file +} From 3738f7beb45413c44e8e8c9894ebc5963b7e8620 Mon Sep 17 00:00:00 2001 From: frelodev Date: Mon, 5 Feb 2024 23:31:06 +0100 Subject: [PATCH 26/42] Cleanup of proto Co-authored-by: alemi.dev --- proto/{auth_service.proto => auth.proto} | 5 +- proto/{buffer_service.proto => buffer.proto} | 2 +- proto/common.proto | 18 ++++ proto/cursor.proto | 10 +- proto/cursor_service.proto | 11 -- proto/workspace.proto | 105 +++++++------------ proto/workspace_service.proto | 18 ---- 7 files changed, 70 insertions(+), 99 deletions(-) rename proto/{auth_service.proto => auth.proto} (94%) rename proto/{buffer_service.proto => buffer.proto} (92%) create mode 100644 proto/common.proto delete mode 100644 proto/cursor_service.proto delete mode 100644 proto/workspace_service.proto diff --git a/proto/auth_service.proto b/proto/auth.proto similarity index 94% rename from proto/auth_service.proto rename to proto/auth.proto index 4d5ef47..c62fe72 100644 --- a/proto/auth_service.proto +++ b/proto/auth.proto @@ -1,6 +1,7 @@ syntax = "proto2"; -package auth_service; +package auth; + // authenticates users, issuing tokens service Auth { @@ -8,10 +9,12 @@ service Auth { rpc Login (WorkspaceJoinRequest) returns (Token); } + message Token { required string token = 1; } + message WorkspaceJoinRequest { required string workspace_id = 1; required string username = 2; diff --git a/proto/buffer_service.proto b/proto/buffer.proto similarity index 92% rename from proto/buffer_service.proto rename to proto/buffer.proto index a10250a..1a3ba47 100644 --- a/proto/buffer_service.proto +++ b/proto/buffer.proto @@ -1,6 +1,6 @@ syntax = "proto2"; -package buffer_service; +package buffer; // handle buffer changes, keep in sync users service Buffer { diff --git a/proto/common.proto b/proto/common.proto new file mode 100644 index 0000000..982f738 --- /dev/null +++ b/proto/common.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +package common; + + +// payload identifying user +message UserIdentity { + // user identifier + required string id = 1; +} + +message UserList { + repeated UserIdentity users = 1; +} + +message Empty{ + //generic Empty message +} diff --git a/proto/cursor.proto b/proto/cursor.proto index 2c4ea18..9667c4f 100644 --- a/proto/cursor.proto +++ b/proto/cursor.proto @@ -1,8 +1,14 @@ syntax = "proto2"; package cursor; +import "common.proto"; + +// handle cursor events and broadcast to all users +service Cursor { + // subscribe to a workspace's cursor events + rpc Attach (stream cursor.CursorEvent) returns (stream cursor.CursorEvent); +} -import "user.proto"; // empty request message MovedResponse {} @@ -26,7 +32,7 @@ message CursorPosition { // cursor event, with user id and cursor position message CursorEvent { // user moving the cursor - required user.UserIdentity user = 1; + required common.UserIdentity user = 1; // new cursor position required CursorPosition position = 2; } \ No newline at end of file diff --git a/proto/cursor_service.proto b/proto/cursor_service.proto deleted file mode 100644 index 4044f3e..0000000 --- a/proto/cursor_service.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto2"; - -package cursor_service; -import "cursor.proto"; -import "user.proto"; - -// handle cursor events and broadcast to all users -service Cursor { - // subscribe to a workspace's cursor events - rpc Attach (stream cursor.CursorEvent) returns (stream cursor.CursorEvent); -} diff --git a/proto/workspace.proto b/proto/workspace.proto index bafda01..3405dfa 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -1,83 +1,56 @@ syntax = "proto2"; package workspace; -import "user.proto"; import "files.proto"; +import "auth.proto"; +import "common.proto"; -message Empty {} - -message TreeRequest {} // empty -message UserRequest {} -message CursorResponse {} -message UserListRequest {} - -message WorkspaceUserList { - repeated user.UserIdentity user = 1; -} - -message WorkspaceMessage { - required string id = 1; //unused??? -} - -enum WorkspaceEventType { - USER_JOIN = 0; - USER_LEAVE = 1; - FILE_CREATE = 2; - FILE_DELETE = 3; - FILE_RENAME = 4; +service Workspace { + rpc CreateWorkspace (workspace.WorkspaceId) returns (common.Empty); + + rpc RequestAccess (workspace.BufferPath) returns (auth.Token); + rpc LeaveWorkspace (workspace.WorkspaceId) returns (common.Empty); + rpc CreateBuffer (workspace.BufferPath) returns (common.Empty); + rpc ListBuffers (common.Empty) returns (files.BufferTree); + rpc ListUsers (common.Empty) returns (common.UserList); + rpc ListBufferUsers (workspace.BufferPath) returns (common.UserList); //TODO discuss + rpc Attach (common.Empty) returns (stream workspace.WorkspaceEvent); + rpc Delete (workspace.BufferPath) returns (common.Empty); //deletes buffer } message WorkspaceEvent { - required string id = 1; // the id of the user, or the path of the file - required WorkspaceEventType event = 2; - optional string extra = 3; // new name? extra info on user events etc... + message UserJoin { + required common.UserIdentity id = 1; + } + message UserLeave { + required common.UserIdentity id = 1; + } + message FileCreate { + required string path = 1; + } + message FileRename { + required string before = 1; + required string after = 2; + } + message FileDelete { + required string path = 1; + } + + oneof event { + UserJoin join = 1; + UserLeave leave = 2; + FileCreate create = 3; + FileRename rename = 4; + FileDelete delete = 5; + } } -message JoinRequest { - required string username = 1; - required string password = 2; -} - -message AttachRequest { - required string path = 1; -} - -message Token { - required string token = 1; -} - -enum FileEventType { - CREATE = 0; - DELETE = 1; - RENAME = 2; -} - -message FileEvent { - required string bufferbuffertree = 1; - required FileEventType type = 2; -} - -enum UserEventType { - JOIN = 0; - LEAVE = 1; -} - -message UserEvent { - required user.UserIdentity user = 1; - required UserEventType type = 2; -} - -message BufferPayload { +message BufferPath { // buffer path to operate onto required string path = 1; } -message BufferListRequest {} -message UserList { - repeated user.UserIdentity users = 1; -} - -message WorkspaceDetails { +message WorkspaceId { required string id = 1; } diff --git a/proto/workspace_service.proto b/proto/workspace_service.proto deleted file mode 100644 index 597dc5a..0000000 --- a/proto/workspace_service.proto +++ /dev/null @@ -1,18 +0,0 @@ -syntax = "proto2"; - -package workspace_service; -import "user.proto"; -import "files.proto"; -import "workspace.proto"; - -service Workspace { - rpc Attach (workspace.AttachRequest) returns (workspace.Token); - rpc LeaveWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); - rpc CreateWorkspace (workspace.WorkspaceDetails) returns (workspace.Empty); - rpc CreateBuffer (workspace.BufferPayload) returns (workspace.Empty); - rpc ListBuffers (workspace.BufferListRequest) returns (files.BufferTree); - rpc ListUsers (workspace.UserListRequest) returns (workspace.UserList); - rpc ListBufferUsers (workspace.BufferPayload) returns (workspace.UserList); //TODO discuss - rpc Join (workspace.JoinRequest) returns (stream workspace.WorkspaceEvent); - rpc Delete (workspace.BufferPayload) returns (workspace.Empty); //deletes buffer -} From 1cf17dc151ff5eb171c071b41d74643f088bc983 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 01:09:28 +0100 Subject: [PATCH 27/42] chore: proto cleanup and simplification reuse as much as possible, keep rpc messages close with their rpc, helper struct for uuid with into() and from(). also replaced the simple things, such as imports and struct fields --- build.rs | 8 +++--- proto/auth.proto | 12 ++++----- proto/buffer.proto | 13 +++++++--- proto/common.proto | 16 ++++++------ proto/cursor.proto | 12 ++++----- proto/files.proto | 5 ---- proto/user.proto | 10 -------- proto/workspace.proto | 33 +++++++++++-------------- src/buffer/worker.rs | 10 +++----- src/client.rs | 42 +++++++++++++++++-------------- src/cursor/controller.rs | 11 +++------ src/cursor/worker.rs | 8 +++--- src/lib.rs | 53 +++++++++++++++++++++++++++++++++++----- src/workspace.rs | 33 ++++++------------------- 14 files changed, 135 insertions(+), 131 deletions(-) delete mode 100644 proto/user.proto diff --git a/build.rs b/build.rs index c035671..504944e 100644 --- a/build.rs +++ b/build.rs @@ -5,14 +5,12 @@ fn main() -> Result<(), Box> { // .build_transport(cfg!(feature = "transport")) .compile( &[ - "proto/user.proto", + "proto/common.proto", "proto/cursor.proto", "proto/files.proto", + "proto/auth.proto", "proto/workspace.proto", - "proto/buffer_service.proto", - "proto/cursor_service.proto", - "proto/workspace_service.proto", - "proto/auth_service.proto", + "proto/buffer.proto", ], &["proto"], )?; diff --git a/proto/auth.proto b/proto/auth.proto index c62fe72..39f7a20 100644 --- a/proto/auth.proto +++ b/proto/auth.proto @@ -2,21 +2,19 @@ syntax = "proto2"; package auth; - // authenticates users, issuing tokens service Auth { - // send credentials and join a workspace + // send credentials and join a workspace, returns ready to use token rpc Login (WorkspaceJoinRequest) returns (Token); } - message Token { required string token = 1; } - +// TODO one-request-to-do-it-all from login to workspace access message WorkspaceJoinRequest { - required string workspace_id = 1; - required string username = 2; - required string password = 3; + required string username = 1; + required string password = 2; + optional string workspace_id = 3; } diff --git a/proto/buffer.proto b/proto/buffer.proto index 1a3ba47..e018440 100644 --- a/proto/buffer.proto +++ b/proto/buffer.proto @@ -1,15 +1,20 @@ syntax = "proto2"; +import "common.proto"; + package buffer; // handle buffer changes, keep in sync users service Buffer { // attach to a buffer and receive operations - rpc Attach (stream Operation) returns (stream Operation); + rpc Attach (stream Operation) returns (stream BufferEvent); } message Operation { required bytes data = 1; - optional string user = 2; - optional string path = 3; -} \ No newline at end of file +} + +message BufferEvent { + required Operation op = 1; + required common.Identity user = 2; +} diff --git a/proto/common.proto b/proto/common.proto index 982f738..6b7ce31 100644 --- a/proto/common.proto +++ b/proto/common.proto @@ -3,16 +3,16 @@ syntax = "proto2"; package common; -// payload identifying user -message UserIdentity { - // user identifier +// a wrapper payload representing an uuid +message Identity { + // uuid bytes, as string required string id = 1; } -message UserList { - repeated UserIdentity users = 1; +// a collection of identities +message IdentityList { + repeated Identity users = 1; } -message Empty{ - //generic Empty message -} +//generic Empty message +message Empty { } diff --git a/proto/cursor.proto b/proto/cursor.proto index 9667c4f..0b4861b 100644 --- a/proto/cursor.proto +++ b/proto/cursor.proto @@ -2,17 +2,15 @@ syntax = "proto2"; package cursor; import "common.proto"; +import "files.proto"; // handle cursor events and broadcast to all users service Cursor { // subscribe to a workspace's cursor events - rpc Attach (stream cursor.CursorEvent) returns (stream cursor.CursorEvent); + rpc Attach (stream cursor.CursorPosition) returns (stream cursor.CursorEvent); } -// empty request -message MovedResponse {} - // a tuple indicating row and column message RowCol { required int32 row = 1; @@ -22,7 +20,7 @@ message RowCol { // cursor position object message CursorPosition { // path of current buffer this cursor is into - required string buffer = 1; + required files.BufferNode buffer = 1; // cursor start position required RowCol start = 2; // cursor end position @@ -32,7 +30,7 @@ message CursorPosition { // cursor event, with user id and cursor position message CursorEvent { // user moving the cursor - required common.UserIdentity user = 1; + required common.Identity user = 1; // new cursor position required CursorPosition position = 2; -} \ No newline at end of file +} diff --git a/proto/files.proto b/proto/files.proto index 4423582..5df3461 100644 --- a/proto/files.proto +++ b/proto/files.proto @@ -9,8 +9,3 @@ message BufferNode { message BufferTree { repeated BufferNode buffers = 1; } - -message WorkspaceFileTree { - // list of strings may be more efficient but it's a lot more hassle - required string payload = 1; // spappolata di json -} \ No newline at end of file diff --git a/proto/user.proto b/proto/user.proto deleted file mode 100644 index 322a935..0000000 --- a/proto/user.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto2"; - -package user; - - -// payload identifying user -message UserIdentity { - // user identifier - required string id = 1; -} diff --git a/proto/workspace.proto b/proto/workspace.proto index 3405dfa..6fedcae 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -1,29 +1,29 @@ syntax = "proto2"; package workspace; + +import "common.proto"; import "files.proto"; import "auth.proto"; -import "common.proto"; service Workspace { - rpc CreateWorkspace (workspace.WorkspaceId) returns (common.Empty); + rpc Attach (common.Empty) returns (stream WorkspaceEvent); - rpc RequestAccess (workspace.BufferPath) returns (auth.Token); - rpc LeaveWorkspace (workspace.WorkspaceId) returns (common.Empty); - rpc CreateBuffer (workspace.BufferPath) returns (common.Empty); + rpc CreateBuffer (files.BufferNode) returns (common.Empty); + rpc AccessBuffer (files.BufferNode) returns (BufferCredentials); + rpc DeleteBuffer (files.BufferNode) returns (common.Empty); + rpc ListBuffers (common.Empty) returns (files.BufferTree); - rpc ListUsers (common.Empty) returns (common.UserList); - rpc ListBufferUsers (workspace.BufferPath) returns (common.UserList); //TODO discuss - rpc Attach (common.Empty) returns (stream workspace.WorkspaceEvent); - rpc Delete (workspace.BufferPath) returns (common.Empty); //deletes buffer + rpc ListUsers (common.Empty) returns (common.IdentityList); + rpc ListBufferUsers (files.BufferNode) returns (common.IdentityList); //TODO discuss } message WorkspaceEvent { message UserJoin { - required common.UserIdentity id = 1; + required common.Identity user = 1; } message UserLeave { - required common.UserIdentity id = 1; + required common.Identity user = 1; } message FileCreate { required string path = 1; @@ -45,12 +45,7 @@ message WorkspaceEvent { } } -message BufferPath { - // buffer path to operate onto - required string path = 1; -} - - -message WorkspaceId { - required string id = 1; +message BufferCredentials { + required common.Identity id = 1; + required auth.Token token = 2; } diff --git a/src/buffer/worker.rs b/src/buffer/worker.rs index 031fc57..cccf1d9 100644 --- a/src/buffer/worker.rs +++ b/src/buffer/worker.rs @@ -11,7 +11,7 @@ use woot::woot::Woot; use crate::errors::IgnorableError; use crate::api::controller::ControllerWorker; use crate::api::TextChange; -use crate::proto::buffer_service::Operation; +use crate::proto::buffer::{BufferEvent, Operation}; use super::controller::BufferController; @@ -66,7 +66,7 @@ impl BufferWorker { impl ControllerWorker for BufferWorker { type Controller = BufferController; type Tx = mpsc::Sender; - type Rx = Streaming; + type Rx = Streaming; fn subscribe(&self) -> BufferController { BufferController::new( @@ -130,8 +130,6 @@ impl ControllerWorker for BufferWorker { for op in ops { let operation = Operation { data: postcard::to_extend(&op, Vec::new()).unwrap(), - user: None, - path: Some(self.name.clone()) }; match tx.send(operation).await { @@ -151,8 +149,8 @@ impl ControllerWorker for BufferWorker { res = rx.message() => match res { Err(_e) => break, Ok(None) => break, - Ok(Some(change)) => match postcard::from_bytes::(&change.data) { - Ok(op) => { + Ok(Some(change)) => match postcard::from_bytes::(&change.op.data) { + Ok(op) => { // TODO here in change we receive info about the author, maybe propagate? self.buffer.merge(op); self.content.send(self.buffer.view()).unwrap_or_warn("could not send buffer update"); for tx in self.pollers.drain(..) { diff --git a/src/client.rs b/src/client.rs index 35b3faa..6b72c00 100644 --- a/src/client.rs +++ b/src/client.rs @@ -11,13 +11,19 @@ use tonic::service::Interceptor; use tonic::transport::{Channel, Endpoint}; use uuid::Uuid; -use crate::api::controller::ControllerWorker; -use crate::cursor::worker::CursorWorker; -use crate::proto::buffer_service::buffer_client::BufferClient; -use crate::proto::cursor_service::cursor_client::CursorClient; -use crate::proto::workspace::{JoinRequest, Token, WorkspaceDetails}; -use crate::proto::workspace_service::workspace_client::WorkspaceClient; -use crate::workspace::Workspace; +use crate::proto::auth::auth_client::AuthClient; +use crate::{ + api::controller::ControllerWorker, + cursor::worker::CursorWorker, + proto::{ + common::Empty, + buffer::buffer_client::BufferClient, + cursor::cursor_client::CursorClient, + auth::{Token, WorkspaceJoinRequest}, + workspace::workspace_client::WorkspaceClient, + }, + workspace::Workspace +}; /// codemp client manager /// @@ -49,9 +55,10 @@ impl Interceptor for ClientInterceptor { #[derive(Debug, Clone)] pub(crate) struct Services { - pub(crate) workspace: crate::proto::workspace_service::workspace_client::WorkspaceClient>, - pub(crate) buffer: crate::proto::buffer_service::buffer_client::BufferClient>, - pub(crate) cursor: crate::proto::cursor_service::cursor_client::CursorClient>, + pub(crate) workspace: WorkspaceClient>, + pub(crate) buffer: BufferClient>, + pub(crate) cursor: CursorClient>, + pub(crate) auth: AuthClient, } // TODO meno losco @@ -90,14 +97,13 @@ impl Client { }) } - /// creates a new workspace (and joins it implicitly), returns an [tokio::sync::RwLock] to interact with it - pub async fn create_workspace(&mut self, workspace_id: &str) -> crate::Result>> { - let mut workspace_client = self.services.workspace.clone(); - workspace_client.create_workspace( - tonic::Request::new(WorkspaceDetails { id: workspace_id.to_string() }) - ).await?; - - self.join_workspace(workspace_id).await + pub async fn login(&self, username: String, password: String, workspace_id: Option) -> crate::Result<()> { + Ok(self.token_tx.send( + self.services.auth.clone() + .login(WorkspaceJoinRequest { username, password, workspace_id}) + .await? + .into_inner() + )?) } /// join a workspace, returns an [tokio::sync::RwLock] to interact with it diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index af1f783..a311150 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -6,7 +6,7 @@ use tokio::sync::{mpsc, broadcast::{self, error::{TryRecvError, RecvError}}, Mut use tonic::async_trait; use uuid::Uuid; -use crate::{api::Controller, errors::IgnorableError, proto::{cursor::{CursorEvent, CursorPosition}, user::UserIdentity}}; +use crate::{api::Controller, errors::IgnorableError, proto::cursor::{CursorEvent, CursorPosition}}; /// the cursor controller implementation /// @@ -22,7 +22,7 @@ use crate::{api::Controller, errors::IgnorableError, proto::{cursor::{CursorEven #[derive(Debug)] pub struct CursorController { user_id: Uuid, - op: mpsc::UnboundedSender, + op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, stop: mpsc::UnboundedSender<()>, @@ -37,7 +37,7 @@ impl Drop for CursorController { impl CursorController { pub(crate) fn new( user_id: Uuid, - op: mpsc::UnboundedSender, + op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, stop: mpsc::UnboundedSender<()>, @@ -56,10 +56,7 @@ impl Controller for CursorController { if cursor.start > cursor.end { std::mem::swap(&mut cursor.start, &mut cursor.end); } - Ok(self.op.send(CursorEvent { - user: UserIdentity { id: self.user_id.to_string() }, - position: cursor, - })?) + Ok(self.op.send(cursor)?) } /// try to receive without blocking, but will still block on stream mutex diff --git a/src/cursor/worker.rs b/src/cursor/worker.rs index 1d27c6b..f4c0072 100644 --- a/src/cursor/worker.rs +++ b/src/cursor/worker.rs @@ -4,14 +4,14 @@ use tokio::sync::{mpsc, broadcast::{self}, Mutex, watch}; use tonic::{Streaming, async_trait}; use uuid::Uuid; -use crate::{api::controller::ControllerWorker, errors::IgnorableError, proto::cursor::CursorEvent}; +use crate::{api::controller::ControllerWorker, errors::IgnorableError, proto::cursor::{CursorPosition, CursorEvent}}; use super::controller::CursorController; pub(crate) struct CursorWorker { user_id: Uuid, - producer: mpsc::UnboundedSender, - op: mpsc::UnboundedReceiver, + producer: mpsc::UnboundedSender, + op: mpsc::UnboundedReceiver, changed: watch::Sender, last_op: watch::Receiver, channel: Arc>, @@ -41,7 +41,7 @@ impl CursorWorker { #[async_trait] impl ControllerWorker for CursorWorker { type Controller = CursorController; - type Tx = mpsc::Sender; + type Tx = mpsc::Sender; type Rx = Streaming; fn subscribe(&self) -> CursorController { diff --git a/src/lib.rs b/src/lib.rs index 1b8e3be..5b602ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -163,14 +163,55 @@ pub use woot; #[cfg(feature = "transport")] #[allow(non_snake_case)] pub mod proto { - pub mod user { tonic::include_proto!("user"); } + pub mod common { + tonic::include_proto!("common"); + + impl From for Identity { + fn from(id: uuid::Uuid) -> Self { + Identity { id: id.to_string() } + } + } + + impl From<&uuid::Uuid> for Identity { + fn from(id: &uuid::Uuid) -> Self { + Identity { id: id.to_string() } + } + } + + impl From for uuid::Uuid { + fn from(value: Identity) -> Self { + uuid::Uuid::parse_str(&value.id).expect("invalid uuid in identity") + } + } + + impl From<&Identity> for uuid::Uuid { + fn from(value: &Identity) -> Self { + uuid::Uuid::parse_str(&value.id).expect("invalid uuid in identity") + } + } + } + + + pub mod files { + tonic::include_proto!("files"); + + impl From for BufferNode { + fn from(value: String) -> Self { + BufferNode { path: value } + } + } + + impl From for String { + fn from(value: BufferNode) -> Self { + value.path + } + } + } + + pub mod buffer { tonic::include_proto!("buffer"); } pub mod cursor { tonic::include_proto!("cursor"); } - pub mod files { tonic::include_proto!("files"); } pub mod workspace { tonic::include_proto!("workspace"); } - pub mod buffer_service { tonic::include_proto!("buffer_service"); } - pub mod cursor_service { tonic::include_proto!("cursor_service"); } - pub mod workspace_service { tonic::include_proto!("workspace_service"); } - pub mod auth_service { tonic::include_proto!("auth_service"); } + pub mod auth { tonic::include_proto!("auth"); } } pub use errors::Error; diff --git a/src/workspace.rs b/src/workspace.rs index 1e00f5e..a964eeb 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -2,11 +2,8 @@ use std::{collections::{BTreeMap, BTreeSet}, str::FromStr, sync::Arc}; use tokio::sync::mpsc; use uuid::Uuid; use crate::{ - proto::{user::UserIdentity, workspace::{AttachRequest, BufferListRequest, BufferPayload, Token, UserListRequest}}, - api::controller::ControllerWorker, - buffer::{self, worker::BufferWorker}, - client::Services, - cursor + api::controller::ControllerWorker, buffer::{self, worker::BufferWorker}, client::Services, cursor, + proto::{auth::Token, common::{Identity, Empty}, files::BufferNode, workspace::{WorkspaceEvent, workspace_event::{Event as WorkspaceEventInner, FileCreate, FileDelete, FileRename, UserJoin, UserLeave}}} }; //TODO may contain more info in the future @@ -15,20 +12,6 @@ pub struct UserInfo { pub uuid: Uuid } -impl From for UserInfo { - fn from(uuid: Uuid) -> Self { - UserInfo { - uuid - } - } -} - -impl From for Uuid { - fn from(uid: UserIdentity) -> Uuid { - Uuid::from_str(&uid.id).expect("expected an uuid") - } -} - pub struct Workspace { id: String, user_id: Uuid, @@ -70,7 +53,7 @@ impl Workspace { pub async fn create(&mut self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); workspace_client.create_buffer( - tonic::Request::new(BufferPayload { path: path.to_string() }) + tonic::Request::new(BufferNode { path: path.to_string() }) ).await?; // add to filetree @@ -115,7 +98,7 @@ impl Workspace { pub async fn fetch_buffers(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let buffers = workspace_client.list_buffers( - tonic::Request::new(BufferListRequest {}) + tonic::Request::new(Empty {}) ).await?.into_inner().buffers; self.filetree.clear(); @@ -130,7 +113,7 @@ impl Workspace { pub async fn fetch_users(&mut self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let users = BTreeSet::from_iter(workspace_client.list_users( - tonic::Request::new(UserListRequest {}) + tonic::Request::new(Empty {}) ).await?.into_inner().users.into_iter().map(Uuid::from)); // only keep userinfo for users that still exist @@ -150,7 +133,7 @@ impl Workspace { pub async fn list_buffer_users(&mut self, path: &str) -> crate::Result> { let mut workspace_client = self.services.workspace.clone(); let buffer_users = workspace_client.list_buffer_users( - tonic::Request::new(BufferPayload { path: path.to_string() }) + tonic::Request::new(BufferNode { path: path.to_string() }) ).await?.into_inner().users; Ok(buffer_users) @@ -167,8 +150,8 @@ impl Workspace { /// delete a buffer pub async fn delete(&mut self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); - workspace_client.delete( - tonic::Request::new(BufferPayload { path: path.to_string() }) + workspace_client.delete_buffer( + tonic::Request::new(BufferNode { path: path.to_string() }) ).await?; self.filetree.remove(path); From f61836e4caf8c2a76930f0b2cd41b12b8d896b99 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 01:11:36 +0100 Subject: [PATCH 28/42] chore: we don't need user_id this deep actually it's stored in our token anyway --- src/cursor/controller.rs | 5 +---- src/cursor/worker.rs | 9 ++------- src/prelude.rs | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/cursor/controller.rs b/src/cursor/controller.rs index a311150..a959b1a 100644 --- a/src/cursor/controller.rs +++ b/src/cursor/controller.rs @@ -4,7 +4,6 @@ use tokio::sync::{mpsc, broadcast::{self, error::{TryRecvError, RecvError}}, Mutex, watch}; use tonic::async_trait; -use uuid::Uuid; use crate::{api::Controller, errors::IgnorableError, proto::cursor::{CursorEvent, CursorPosition}}; @@ -21,7 +20,6 @@ use crate::{api::Controller, errors::IgnorableError, proto::cursor::{CursorEvent /// upon dropping this handle will stop the associated worker #[derive(Debug)] pub struct CursorController { - user_id: Uuid, op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, @@ -36,13 +34,12 @@ impl Drop for CursorController { impl CursorController { pub(crate) fn new( - user_id: Uuid, op: mpsc::UnboundedSender, last_op: Mutex>, stream: Mutex>, stop: mpsc::UnboundedSender<()>, ) -> Self { - CursorController { user_id, op, last_op, stream, stop } + CursorController { op, last_op, stream, stop } } } diff --git a/src/cursor/worker.rs b/src/cursor/worker.rs index f4c0072..3f45dfc 100644 --- a/src/cursor/worker.rs +++ b/src/cursor/worker.rs @@ -2,14 +2,12 @@ use std::sync::Arc; use tokio::sync::{mpsc, broadcast::{self}, Mutex, watch}; use tonic::{Streaming, async_trait}; -use uuid::Uuid; use crate::{api::controller::ControllerWorker, errors::IgnorableError, proto::cursor::{CursorPosition, CursorEvent}}; use super::controller::CursorController; pub(crate) struct CursorWorker { - user_id: Uuid, producer: mpsc::UnboundedSender, op: mpsc::UnboundedReceiver, changed: watch::Sender, @@ -19,14 +17,13 @@ pub(crate) struct CursorWorker { stop_control: mpsc::UnboundedSender<()>, } -impl CursorWorker { - pub(crate) fn new(user_id: Uuid) -> Self { +impl Default for CursorWorker { + fn default() -> Self { let (op_tx, op_rx) = mpsc::unbounded_channel(); let (cur_tx, _cur_rx) = broadcast::channel(64); let (end_tx, end_rx) = mpsc::unbounded_channel(); let (change_tx, change_rx) = watch::channel(CursorEvent::default()); Self { - user_id, producer: op_tx, op: op_rx, changed: change_tx, @@ -46,7 +43,6 @@ impl ControllerWorker for CursorWorker { fn subscribe(&self) -> CursorController { CursorController::new( - self.user_id.clone(), self.producer.clone(), Mutex::new(self.last_op.clone()), Mutex::new(self.channel.subscribe()), @@ -58,7 +54,6 @@ impl ControllerWorker for CursorWorker { loop { tokio::select!{ Ok(Some(cur)) = rx.message() => { - if Uuid::from(cur.user.clone()) == self.user_id { continue } self.channel.send(cur.clone()).unwrap_or_warn("could not broadcast event"); self.changed.send(cur).unwrap_or_warn("could not update last event"); }, diff --git a/src/prelude.rs b/src/prelude.rs index 3a2934b..077a94b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -31,4 +31,4 @@ pub use crate::{ proto::cursor::CursorPosition as CodempCursorPosition, proto::cursor::CursorEvent as CodempCursorEvent, proto::cursor::RowCol as CodempRowCol, -}; \ No newline at end of file +}; From 948a1b4de5330cbc156ecd65fe506a6586372405 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 01:12:05 +0100 Subject: [PATCH 29/42] feat: workspace streaming attach and lists to join a workspace, attach to it. to leave a workspace, close the channel. on such channel you get workspace events (new buffers, user leave, ...). must fetch current buffers and users upon join. if workspace doesn't exist, server should create it on attach also dashmap everywhere to get/put simple --- Cargo.toml | 1 + src/client.rs | 59 ++++++++++++--------------- src/workspace.rs | 101 +++++++++++++++++++++++++---------------------- 3 files changed, 81 insertions(+), 80 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7cca7ef..4948bb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ tokio-stream = { version = "0.1", optional = true } lazy_static = { version = "1.4", optional = true } serde = { version = "1.0.193", features = ["derive"] } postcard = "1.0.8" +dashmap = "5.5.3" [build-dependencies] tonic-build = "0.9" diff --git a/src/client.rs b/src/client.rs index 6b72c00..520ac91 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,13 +2,14 @@ //! //! codemp client manager, containing grpc services -use std::collections::BTreeMap; use std::sync::Arc; -use tokio::sync::{mpsc, RwLock}; +use dashmap::DashMap; +use tokio::sync::mpsc; use tonic::service::interceptor::InterceptedService; use tonic::service::Interceptor; use tonic::transport::{Channel, Endpoint}; +use tonic::IntoRequest; use uuid::Uuid; use crate::proto::auth::auth_client::AuthClient; @@ -33,7 +34,7 @@ use crate::{ pub struct Client { user_id: Uuid, token_tx: Arc>, - pub workspaces: BTreeMap>>, + workspaces: Arc>>, services: Arc } @@ -62,7 +63,7 @@ pub(crate) struct Services { } // TODO meno losco -fn parse_codemp_connection_string<'a>(string: &'a str) -> (String, String) { +fn parse_codemp_connection_string(string: &str) -> (String, String) { let url = string.replace("codemp://", ""); let (host, workspace) = url.split_once('/').unwrap(); (format!("http://{}", host), workspace.to_string()) @@ -86,14 +87,15 @@ impl Client { let buffer = BufferClient::with_interceptor(channel.clone(), inter.clone()); let cursor = CursorClient::with_interceptor(channel.clone(), inter.clone()); let workspace = WorkspaceClient::with_interceptor(channel.clone(), inter.clone()); + let auth = AuthClient::new(channel); let user_id = uuid::Uuid::new_v4(); Ok(Client { user_id, token_tx: Arc::new(token_tx), - workspaces: BTreeMap::new(), - services: Arc::new(Services { workspace, buffer, cursor }) + workspaces: Arc::new(DashMap::default()), + services: Arc::new(Services { workspace, buffer, cursor, auth }) }) } @@ -107,52 +109,43 @@ impl Client { } /// join a workspace, returns an [tokio::sync::RwLock] to interact with it - pub async fn join_workspace(&mut self, workspace_id: &str) -> crate::Result>> { - self.token_tx.send(self.services.workspace.clone().join( - tonic::Request::new(JoinRequest { username: "".to_string(), password: "".to_string() }) //TODO - ).await?.into_inner())?; + pub async fn join_workspace(&mut self, workspace: &str) -> crate::Result> { + let ws_stream = self.services.workspace.clone().attach(Empty{}.into_request()).await?.into_inner(); let (tx, rx) = mpsc::channel(10); - let stream = self.services.cursor.clone() + let cur_stream = self.services.cursor.clone() .attach(tokio_stream::wrappers::ReceiverStream::new(rx)) .await? .into_inner(); - let worker = CursorWorker::new(self.user_id.clone()); + let worker = CursorWorker::default(); let controller = Arc::new(worker.subscribe()); tokio::spawn(async move { tracing::debug!("controller worker started"); - worker.work(tx, stream).await; + worker.work(tx, cur_stream).await; tracing::debug!("controller worker stopped"); }); - let lock = Arc::new(RwLock::new( - Workspace::new( - workspace_id.to_string(), - self.user_id, - self.token_tx.clone(), - controller, - self.services.clone() - ).await? + let ws = Arc::new(Workspace::new( + workspace.to_string(), + self.user_id, + self.token_tx.clone(), + controller, + self.services.clone() )); - self.workspaces.insert(workspace_id.to_string(), lock.clone()); + ws.fetch_users().await?; + ws.fetch_buffers().await?; - Ok(lock) - } + ws.run_actor(ws_stream); - /// leave given workspace, disconnecting buffer and cursor controllers - pub async fn leave_workspace(&self, workspace_id: &str) -> crate::Result<()> { - let mut workspace_client = self.services.workspace.clone(); - workspace_client.leave_workspace( - tonic::Request::new(WorkspaceDetails { id: workspace_id.to_string() }) - ).await?; - - Ok(()) + self.workspaces.insert(workspace.to_string(), ws.clone()); + + Ok(ws) } /// accessor for user id pub fn user_id(&self) -> Uuid { - self.user_id.clone() + self.user_id } } diff --git a/src/workspace.rs b/src/workspace.rs index a964eeb..605ef1a 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -1,5 +1,7 @@ -use std::{collections::{BTreeMap, BTreeSet}, str::FromStr, sync::Arc}; +use std::{collections::BTreeSet, sync::Arc}; use tokio::sync::mpsc; +use dashmap::{DashMap, DashSet}; +use tonic::Streaming; use uuid::Uuid; use crate::{ api::controller::ControllerWorker, buffer::{self, worker::BufferWorker}, client::Services, cursor, @@ -14,43 +16,60 @@ pub struct UserInfo { pub struct Workspace { id: String, - user_id: Uuid, + user_id: Uuid, // reference to global user id token: Arc>, cursor: Arc, - buffers: BTreeMap>, - filetree: BTreeSet, - users: BTreeMap, + buffers: Arc>>, + pub(crate) filetree: Arc>, + pub(crate) users: Arc>, services: Arc } impl Workspace { /// create a new buffer and perform initial fetch operations - pub(crate) async fn new( + pub(crate) fn new( id: String, user_id: Uuid, token: Arc>, cursor: Arc, services: Arc - ) -> crate::Result { - let mut ws = Workspace { + ) -> Self { + Workspace { id, user_id, token, cursor, - buffers: BTreeMap::new(), - filetree: BTreeSet::new(), - users: BTreeMap::new(), + buffers: Arc::new(DashMap::default()), + filetree: Arc::new(DashSet::default()), + users: Arc::new(DashMap::default()), services - }; + } + } - ws.fetch_buffers().await?; - ws.fetch_users().await?; - - Ok(ws) + pub(crate) fn run_actor(&self, mut stream: Streaming) { + let users = self.users.clone(); + let filetree = self.filetree.clone(); + let name = self.id(); + tokio::spawn(async move { + loop { + match stream.message().await { + Err(e) => break tracing::error!("workspace '{}' stream closed: {}", name, e), + Ok(None) => break tracing::info!("leaving workspace {}", name), + Ok(Some(WorkspaceEvent { event: None })) => tracing::warn!("workspace {} received empty event", name), + Ok(Some(WorkspaceEvent { event: Some(ev) })) => match ev { + WorkspaceEventInner::Join(UserJoin { user }) => { users.insert(user.clone().into(), UserInfo { uuid: user.into() }); }, + WorkspaceEventInner::Leave(UserLeave { user }) => { users.remove(&user.into()); }, + WorkspaceEventInner::Create(FileCreate { path }) => { filetree.insert(path); }, + WorkspaceEventInner::Rename(FileRename { before, after }) => { filetree.remove(&before); filetree.insert(after); }, + WorkspaceEventInner::Delete(FileDelete { path }) => { filetree.remove(&path); }, + }, + } + } + }); } /// create a new buffer in current workspace - pub async fn create(&mut self, path: &str) -> crate::Result<()> { + pub async fn create(&self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); workspace_client.create_buffer( tonic::Request::new(BufferNode { path: path.to_string() }) @@ -69,17 +88,16 @@ impl Workspace { /// /// to interact with such buffer use [crate::api::Controller::send] or /// [crate::api::Controller::recv] to exchange [crate::api::TextChange] - pub async fn attach(&mut self, path: &str) -> crate::Result> { + pub async fn attach(&self, path: &str) -> crate::Result> { let mut worskspace_client = self.services.workspace.clone(); - let mut request = tonic::Request::new(AttachRequest { path: path.to_string() }); - request.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(path).expect("could not represent path as byte sequence")); - self.token.send(worskspace_client.attach(request).await?.into_inner())?; + let request = tonic::Request::new(BufferNode { path: path.to_string() }); + let credentials = worskspace_client.access_buffer(request).await?.into_inner(); + self.token.send(credentials.token)?; let (tx, rx) = mpsc::channel(10); - let stream = self.services.buffer.clone() - .attach(tokio_stream::wrappers::ReceiverStream::new(rx)) - .await? - .into_inner(); + let mut req = tonic::Request::new(tokio_stream::wrappers::ReceiverStream::new(rx)); + req.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(path).expect("could not represent path as byte sequence")); + let stream = self.services.buffer.clone().attach(req).await?.into_inner(); let worker = BufferWorker::new(self.user_id, path); let controller = Arc::new(worker.subscribe()); @@ -95,7 +113,7 @@ impl Workspace { } /// fetch a list of all buffers in a workspace - pub async fn fetch_buffers(&mut self) -> crate::Result<()> { + pub async fn fetch_buffers(&self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let buffers = workspace_client.list_buffers( tonic::Request::new(Empty {}) @@ -110,19 +128,16 @@ impl Workspace { } /// fetch a list of all users in a workspace - pub async fn fetch_users(&mut self) -> crate::Result<()> { + pub async fn fetch_users(&self) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); let users = BTreeSet::from_iter(workspace_client.list_users( tonic::Request::new(Empty {}) ).await?.into_inner().users.into_iter().map(Uuid::from)); - // only keep userinfo for users that still exist - self.users.retain(|k, _v| users.contains(k)); - - let _users = self.users.clone(); // damnnn rust - users.iter() - .filter(|u| _users.contains_key(u)) - .for_each(|u| { self.users.insert(*u, UserInfo::from(*u)); }); + self.users.clear(); + for u in users { + self.users.insert(u, UserInfo { uuid: u }); + } Ok(()) } @@ -130,7 +145,7 @@ impl Workspace { /// get a list of the users attached to a specific buffer /// /// TODO: discuss implementation details - pub async fn list_buffer_users(&mut self, path: &str) -> crate::Result> { + pub async fn list_buffer_users(&self, path: &str) -> crate::Result> { let mut workspace_client = self.services.workspace.clone(); let buffer_users = workspace_client.list_buffer_users( tonic::Request::new(BufferNode { path: path.to_string() }) @@ -139,16 +154,8 @@ impl Workspace { Ok(buffer_users) } - /// detach from a specific buffer, returns false if there - pub fn detach(&mut self, path: &str) -> bool { - match &mut self.buffers.remove(path) { - None => false, - Some(_) => true - } - } - /// delete a buffer - pub async fn delete(&mut self, path: &str) -> crate::Result<()> { + pub async fn delete(&self, path: &str) -> crate::Result<()> { let mut workspace_client = self.services.workspace.clone(); workspace_client.delete_buffer( tonic::Request::new(BufferNode { path: path.to_string() }) @@ -167,11 +174,11 @@ impl Workspace { /// get a new reference to a buffer controller, if any is active to given path pub fn buffer_by_name(&self, path: &str) -> Option> { - self.buffers.get(path).cloned() + self.buffers.get(path).map(|x| x.clone()) } /// get the currently cached "filetree" pub fn filetree(&self) -> Vec { self.filetree.iter().map(|f| f.clone()).collect() - } -} \ No newline at end of file + } +} From 2fabec6e2e6abf0be851556f8524feaf1312f57c Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 01:15:32 +0100 Subject: [PATCH 30/42] docs: added notes about leaking buffer id to client --- proto/workspace.proto | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/proto/workspace.proto b/proto/workspace.proto index 6fedcae..26eaa04 100644 --- a/proto/workspace.proto +++ b/proto/workspace.proto @@ -15,7 +15,7 @@ service Workspace { rpc ListBuffers (common.Empty) returns (files.BufferTree); rpc ListUsers (common.Empty) returns (common.IdentityList); - rpc ListBufferUsers (files.BufferNode) returns (common.IdentityList); //TODO discuss + rpc ListBufferUsers (files.BufferNode) returns (common.IdentityList); } message WorkspaceEvent { @@ -45,6 +45,9 @@ message WorkspaceEvent { } } +// TODO this is very ugly because we can't just return a new token (which is already smelly but whatev), we also need to tell the underlying id so that +// the client can put it as metadata while attaching, because it can't really know the underlying id that the server is using for each buffer without +// parsing the token itself. meehhhhhh, this bleeds underlying implementation to the upper levels, how can we avoid this?? message BufferCredentials { required common.Identity id = 1; required auth.Token token = 2; From 42ae4c0152661546053f8ef31eff9b7ea2c0e329 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 01:18:24 +0100 Subject: [PATCH 31/42] fix: proper deps for features --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4948bb6..09adeb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ tracing = "0.1" # woot codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.1", optional = true } # proto +uuid = { version = "1.3.1", features = ["v4"], optional = true } tonic = { version = "0.9", features = ["tls", "tls-roots"], optional = true } prost = { version = "0.11.8", optional = true } # api @@ -20,7 +21,6 @@ tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "ful async-trait = { version = "0.1", optional = true } # client md5 = { version = "0.7.0", optional = true } -uuid = { version = "1.3.1", features = ["v4"], optional = true } serde_json = { version = "1", optional = true } tokio-stream = { version = "0.1", optional = true } # global @@ -36,7 +36,7 @@ tonic-build = "0.9" default = ["client"] api = ["woot", "dep:similar", "dep:tokio", "dep:async-trait"] woot = ["dep:codemp-woot"] -transport = ["dep:prost", "dep:tonic"] +transport = ["dep:prost", "dep:tonic", "dep:uuid"] client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json"] server = ["transport"] global = ["client", "dep:lazy_static"] From d78362c745c10804d1a680948f18a194dc0e0aa8 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 03:47:37 +0100 Subject: [PATCH 32/42] feat: getter for workspace name doesn't really make sense: it's user-defined because we connect to workspace contained inside token but store such connection with user given name --- src/client.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client.rs b/src/client.rs index 520ac91..d61beb2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -144,6 +144,10 @@ impl Client { Ok(ws) } + pub fn get_workspace(&self, id: &str) -> Option> { + self.workspaces.get(id).map(|x| x.clone()) + } + /// accessor for user id pub fn user_id(&self) -> Uuid { self.user_id From ed1ce45e7f58c593902242e81d3a8d7f5f27816f Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 04:41:13 +0100 Subject: [PATCH 33/42] fix: send path received from server --- src/workspace.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workspace.rs b/src/workspace.rs index 605ef1a..86c4301 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -96,7 +96,7 @@ impl Workspace { let (tx, rx) = mpsc::channel(10); let mut req = tonic::Request::new(tokio_stream::wrappers::ReceiverStream::new(rx)); - req.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(path).expect("could not represent path as byte sequence")); + req.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(credentials.id.id).expect("could not represent path as byte sequence")); let stream = self.services.buffer.clone().attach(req).await?.into_inner(); let worker = BufferWorker::new(self.user_id, path); From 99a268185a4b305c940947d6bd43828ffa7180f6 Mon Sep 17 00:00:00 2001 From: alemi Date: Wed, 7 Feb 2024 21:24:31 +0100 Subject: [PATCH 34/42] fix: increase channels buffer size maybe helps? idk probably not actually --- src/client.rs | 2 +- src/workspace.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index d61beb2..bfd1e84 100644 --- a/src/client.rs +++ b/src/client.rs @@ -112,7 +112,7 @@ impl Client { pub async fn join_workspace(&mut self, workspace: &str) -> crate::Result> { let ws_stream = self.services.workspace.clone().attach(Empty{}.into_request()).await?.into_inner(); - let (tx, rx) = mpsc::channel(10); + let (tx, rx) = mpsc::channel(256); let cur_stream = self.services.cursor.clone() .attach(tokio_stream::wrappers::ReceiverStream::new(rx)) .await? diff --git a/src/workspace.rs b/src/workspace.rs index 86c4301..9c42b23 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -94,7 +94,7 @@ impl Workspace { let credentials = worskspace_client.access_buffer(request).await?.into_inner(); self.token.send(credentials.token)?; - let (tx, rx) = mpsc::channel(10); + let (tx, rx) = mpsc::channel(256); let mut req = tonic::Request::new(tokio_stream::wrappers::ReceiverStream::new(rx)); req.metadata_mut().insert("path", tonic::metadata::MetadataValue::try_from(credentials.id.id).expect("could not represent path as byte sequence")); let stream = self.services.buffer.clone().attach(req).await?.into_inner(); From c9a36ea8ec3dc07dc8b1d7a78826b2a59d4f7f03 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 00:35:08 +0100 Subject: [PATCH 35/42] fix: cleaned up code, fixed multi-op change issues --- Cargo.toml | 2 +- src/api/change.rs | 35 +++++++++++++++++++++--- src/buffer/worker.rs | 64 +++++++++++--------------------------------- 3 files changed, 49 insertions(+), 52 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 09adeb1..87b0f18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ name = "codemp" # core tracing = "0.1" # woot -codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.1", optional = true } +codemp-woot = { git = "ssh://git@github.com/codewithotherpeopleandchangenamelater/woot.git", features = ["serde"], tag = "v0.1.2", optional = true } # proto uuid = { version = "1.3.1", features = ["v4"], optional = true } tonic = { version = "0.9", features = ["tls", "tls-roots"], optional = true } diff --git a/src/api/change.rs b/src/api/change.rs index 041e628..9fbf0a5 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -3,7 +3,8 @@ //! an editor-friendly representation of a text change in a buffer //! to easily interface with codemp from various editors -use crate::proto::cursor::RowCol; +#[cfg(feature = "woot")] +use crate::woot::{WootResult, woot::Woot, crdt::{TextEditor, CRDT, Op}}; /// an editor-friendly representation of a text change in a buffer /// @@ -59,6 +60,33 @@ impl TextChange { } } + #[cfg(feature = "woot")] + pub fn transform(self, woot: &Woot) -> WootResult> { + let mut out = Vec::new(); + if self.is_empty() { return Ok(out); } // no-op + let view = woot.view(); + let Some(span) = view.get(self.span.clone()) else { + return Err(crate::woot::WootError::OutOfBounds); + }; + let diff = similar::TextDiff::from_chars(span, &self.content); + for (i, diff) in diff.iter_all_changes().enumerate() { + match diff.tag() { + similar::ChangeTag::Equal => {}, + similar::ChangeTag::Delete => match woot.delete_one(self.span.start + i) { + Err(e) => tracing::error!("could not create deletion: {}", e), + Ok(op) => out.push(op), + }, + similar::ChangeTag::Insert => { + match woot.insert(self.span.start + i, diff.value()) { + Ok(mut op) => out.append(&mut op), + Err(e) => tracing::error!("could not create insertion: {}", e), + } + }, + } + } + Ok(out) + } + /// returns true if this TextChange deletes existing text pub fn is_deletion(&self) -> bool { !self.span.is_empty() @@ -84,11 +112,12 @@ impl TextChange { /// convert from byte index to row and column /// txt must be the whole content of the buffer, in order to count lines - pub fn index_to_rowcol(txt: &str, index: usize) -> RowCol { + #[cfg(feature = "transport")] + pub fn index_to_rowcol(txt: &str, index: usize) -> crate::proto::cursor::RowCol { // FIXME might panic, use .get() let row = txt[..index].matches('\n').count() as i32; let col = txt[..index].split('\n').last().unwrap_or("").len() as i32; - RowCol { row, col } + crate::proto::cursor::RowCol { row, col } } } diff --git a/src/buffer/worker.rs b/src/buffer/worker.rs index cccf1d9..e98ea7c 100644 --- a/src/buffer/worker.rs +++ b/src/buffer/worker.rs @@ -1,11 +1,10 @@ use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; -use similar::{TextDiff, ChangeTag}; use tokio::sync::{watch, mpsc, oneshot}; use tonic::{async_trait, Streaming}; use uuid::Uuid; -use woot::crdt::{Op, CRDT, TextEditor}; +use woot::crdt::{Op, CRDT}; use woot::woot::Woot; use crate::errors::IgnorableError; @@ -95,53 +94,22 @@ impl ControllerWorker for BufferWorker { // received a text change from editor res = self.operations.recv() => match res { - None => break, - Some(change) => { - if !change.is_empty() { - let view = self.buffer.view(); - match view.get(change.span.clone()) { - None => tracing::error!("received illegal span from client: {:?} but buffer is of len {}", change.span, view.len()), - Some(span) => { - let diff = TextDiff::from_chars(span, &change.content); - - let mut i = 0; - let mut ops = Vec::new(); - for diff in diff.iter_all_changes() { - match diff.tag() { - ChangeTag::Equal => i += 1, - ChangeTag::Delete => match self.buffer.delete(change.span.start + i) { - Ok(op) => ops.push(op), - Err(e) => tracing::error!("could not apply deletion: {}", e), - }, - ChangeTag::Insert => { - for c in diff.value().chars() { - match self.buffer.insert(change.span.start + i, c) { - Ok(op) => { - ops.push(op); - i += 1; - }, - Err(e) => tracing::error!("could not apply insertion: {}", e), - } - } - }, - } - } - - for op in ops { - let operation = Operation { - data: postcard::to_extend(&op, Vec::new()).unwrap(), - }; - - match tx.send(operation).await { - Err(e) => tracing::error!("server refused to broadcast {}: {}", op, e), - Ok(()) => { - self.content.send(self.buffer.view()).unwrap_or_warn("could not send buffer update"); - }, - } - } - }, + None => break tracing::debug!("stopping: editor closed channel"), + Some(change) => match change.transform(&self.buffer) { + Err(e) => break tracing::error!("could not apply operation from client: {}", e), + Ok(ops) => { + for op in ops { + self.buffer.merge(op.clone()); + let operation = Operation { + data: postcard::to_extend(&op, Vec::new()).unwrap(), + }; + if let Err(e) = tx.send(operation).await { + tracing::error!("server refused to broadcast {}: {}", op, e); + } } - } + self.content.send(self.buffer.view()) + .unwrap_or_warn("could not send buffer update"); + }, } }, From 45c1106d982c1e2f2bcc3075ec33169d1061663b Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 00:35:49 +0100 Subject: [PATCH 36/42] chore: cleanup dependencies and features --- Cargo.toml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 87b0f18..48c95c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,27 +17,23 @@ tonic = { version = "0.9", features = ["tls", "tls-roots"], optional = true } prost = { version = "0.11.8", optional = true } # api similar = { version = "2.2", features = ["inline"], optional = true } -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync", "full"], optional = true } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "sync"], optional = true } async-trait = { version = "0.1", optional = true } # client md5 = { version = "0.7.0", optional = true } serde_json = { version = "1", optional = true } tokio-stream = { version = "0.1", optional = true } -# global -lazy_static = { version = "1.4", optional = true } serde = { version = "1.0.193", features = ["derive"] } -postcard = "1.0.8" -dashmap = "5.5.3" +dashmap = { version = "5.5.3", optional = true } +postcard = { version = "1.0.8", optional = true } [build-dependencies] tonic-build = "0.9" [features] -default = ["client"] -api = ["woot", "dep:similar", "dep:tokio", "dep:async-trait"] -woot = ["dep:codemp-woot"] +default = [] +api = ["woot", "dep:tokio", "dep:async-trait"] +woot = ["dep:codemp-woot", "dep:similar"] transport = ["dep:prost", "dep:tonic", "dep:uuid"] -client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json"] -server = ["transport"] -global = ["client", "dep:lazy_static"] -sync = ["client"] +client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json", "dep:dashmap", "dep:postcard"] +server = ["transport", "woot"] From 47127bbb41d803d7f2d12371dd09e2643789b630 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 00:39:07 +0100 Subject: [PATCH 37/42] chore: back from transport to proto --- Cargo.toml | 12 ++++++------ build.rs | 2 +- src/api/change.rs | 2 +- src/lib.rs | 12 +++++------- src/prelude.rs | 2 +- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 48c95c2..10cbf57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,9 +31,9 @@ postcard = { version = "1.0.8", optional = true } tonic-build = "0.9" [features] -default = [] -api = ["woot", "dep:tokio", "dep:async-trait"] -woot = ["dep:codemp-woot", "dep:similar"] -transport = ["dep:prost", "dep:tonic", "dep:uuid"] -client = ["transport", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json", "dep:dashmap", "dep:postcard"] -server = ["transport", "woot"] +default = [] +api = ["woot", "dep:tokio", "dep:async-trait"] +woot = ["dep:codemp-woot", "dep:similar"] +proto = ["dep:prost", "dep:tonic", "dep:uuid"] +client = ["proto", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json", "dep:dashmap", "dep:postcard"] +server = ["proto", "woot"] diff --git a/build.rs b/build.rs index 504944e..d4f38c1 100644 --- a/build.rs +++ b/build.rs @@ -2,7 +2,7 @@ fn main() -> Result<(), Box> { tonic_build::configure() // .build_client(cfg!(feature = "client")) // .build_server(cfg!(feature = "server")) // FIXME if false, build fails???? - // .build_transport(cfg!(feature = "transport")) + // .build_transport(cfg!(feature = "proto")) .compile( &[ "proto/common.proto", diff --git a/src/api/change.rs b/src/api/change.rs index 9fbf0a5..79b6982 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -112,7 +112,7 @@ impl TextChange { /// convert from byte index to row and column /// txt must be the whole content of the buffer, in order to count lines - #[cfg(feature = "transport")] + #[cfg(feature = "proto")] pub fn index_to_rowcol(txt: &str, index: usize) -> crate::proto::cursor::RowCol { // FIXME might panic, use .get() let row = txt[..index].matches('\n').count() as i32; diff --git a/src/lib.rs b/src/lib.rs index 5b602ac..68ad511 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,12 +21,10 @@ //! immediately but instead deferred until compatible. //! //! ## features -//! * `woot` : include the underlying CRDT library and re-exports it (default enabled) -//! * `api` : include traits for core interfaces under [api] (default enabled) -//! * `proto` : include GRCP protocol definitions under [proto] (default enabled) -//! * `client`: include the local [client] implementation (default enabled) -//! * `global`: provide a lazy_static global INSTANCE in [instance::global] -//! * `sync` : wraps the [instance::a_sync::Instance] holder into a sync variant: [instance::sync::Instance] +//! * `woot` : include the underlying CRDT library and re-exports it (default enabled) +//! * `api` : include traits for core interfaces under [api] (default enabled) +//! * `proto` : include GRCP protocol definitions under [proto] (default enabled) +//! * `client` : include the local [client] implementation (default enabled) //! //! ## examples //! while the [client::Client] itself is the core structure implementing all methods, plugins will mostly @@ -160,7 +158,7 @@ pub mod prelude; pub use woot; /// protocol types and services auto-generated by grpc -#[cfg(feature = "transport")] +#[cfg(feature = "proto")] #[allow(non_snake_case)] pub mod proto { pub mod common { diff --git a/src/prelude.rs b/src/prelude.rs index 077a94b..c01864c 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -26,7 +26,7 @@ pub use crate::{ buffer::Controller as CodempBufferController, }; -#[cfg(feature = "transport")] +#[cfg(feature = "proto")] pub use crate::{ proto::cursor::CursorPosition as CodempCursorPosition, proto::cursor::CursorEvent as CodempCursorEvent, From e0d4360d0976e10dd490b3745144c2ac493b6f6c Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 00:59:04 +0100 Subject: [PATCH 38/42] feat: BufferNode also From<&str> --- src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 68ad511..3ceb4ff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -199,6 +199,12 @@ pub mod proto { } } + impl From<&str> for BufferNode { + fn from(value: &str) -> Self { + BufferNode { path: value.to_string() } + } + } + impl From for String { fn from(value: BufferNode) -> Self { value.path From f7062378281f73db7517ddf0034da247e141f194 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 01:03:38 +0100 Subject: [PATCH 39/42] chore: removed blocking_recv just block_on(recv()) ... --- src/api/controller.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/api/controller.rs b/src/api/controller.rs index 465a9f7..cf45684 100644 --- a/src/api/controller.rs +++ b/src/api/controller.rs @@ -58,11 +58,4 @@ pub trait Controller : Sized + Send + Sync { /// attempt to receive a value without blocking, return None if nothing is available fn try_recv(&self) -> Result>; - - /// sync variant of [Self::recv], blocking invoking thread - /// this calls [Controller::recv] inside a [tokio::runtime::Runtime::block_on] - #[cfg(feature = "sync")] - fn blocking_recv(&self, rt: &tokio::runtime::Handle) -> Result { - rt.block_on(self.recv()) - } } From 4fdd2a79c444cf3a50b97c9173ce2a2bfd1b9599 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 01:04:10 +0100 Subject: [PATCH 40/42] fix: api doesn't really require woot or similar --- Cargo.toml | 2 +- src/api/change.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 10cbf57..8f25c02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ tonic-build = "0.9" [features] default = [] -api = ["woot", "dep:tokio", "dep:async-trait"] +api = ["dep:async-trait"] woot = ["dep:codemp-woot", "dep:similar"] proto = ["dep:prost", "dep:tonic", "dep:uuid"] client = ["proto", "api", "dep:tokio", "dep:tokio-stream", "dep:uuid", "dep:md5", "dep:serde_json", "dep:dashmap", "dep:postcard"] diff --git a/src/api/change.rs b/src/api/change.rs index 79b6982..aa597ba 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -30,6 +30,7 @@ pub struct TextChange { } impl TextChange { + #[cfg(feature = "woot")] /// create a new TextChange from the difference of given strings pub fn from_diff(before: &str, after: &str) -> TextChange { let diff = similar::TextDiff::from_chars(before, after); From a622ac773c99738bf691b6a2f8ca8f8f039c78c0 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 01:04:24 +0100 Subject: [PATCH 41/42] docs: updated main doc page examples --- src/lib.rs | 90 +++++++++++++++++++++++++----------------------------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3ceb4ff..5f953ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,59 +27,77 @@ //! * `client` : include the local [client] implementation (default enabled) //! //! ## examples -//! while the [client::Client] itself is the core structure implementing all methods, plugins will mostly -//! interact with [Instance] managers. +//! the [client::Client] itself is the core structure implementing all methods, plugins should +//! attempt to keep a single instance at any time +//! +//! working sessions are [workspace::Workspace] and while managing multiple ones is in theory +//! possible, it's not really working right now due to how authentication is managed //! //! ### async -//! this library is natively async and thus async usage should be preferred if possible with -//! [instance::a_sync::Instance] -//! //! ```rust,no_run //! use codemp::api::{Controller, TextChange}; -//! # use codemp::instance::a_sync::Instance; +//! # use codemp::client::Client; //! //! # async fn async_example() -> codemp::Result<()> { -//! let session = Instance::default(); // create global session -//! session.connect("http://alemi.dev:50051").await?; // connect to remote server +//! // creating a client session will immediately attempt to connect +//! let mut session = Client::new("http://alemi.dev:50053").await?; +//! +//! // login first, obtaining a new token granting access to 'some_workspace' +//! session.login( +//! "username".to_string(), +//! "password".to_string(), +//! Some("some_workspace".to_string()) +//! ).await?; //! -//! // join a remote workspace, obtaining a cursor controller -//! let cursor = session.join("some_workspace").await?; -//! cursor.send( // move cursor -//! codemp::proto::CursorPosition { +//! // join a remote workspace, obtaining a workspace handle +//! let workspace = session.join_workspace("some_workspace").await?; +//! +//! workspace.cursor().send( // move cursor +//! codemp::proto::cursor::CursorPosition { //! buffer: "test.txt".into(), -//! start: Some(codemp::proto::RowCol { row: 0, col: 0 }), -//! end: Some(codemp::proto::RowCol { row: 0, col: 1 }), +//! start: codemp::proto::cursor::RowCol { row: 0, col: 0 }, +//! end: codemp::proto::cursor::RowCol { row: 0, col: 1 }, //! } //! )?; -//! let op = cursor.recv().await?; // listen for event +//! let op = workspace.cursor().recv().await?; // receive event from server //! println!("received cursor event: {:?}", op); //! //! // attach to a new buffer and execute operations on it -//! session.create("test.txt", None).await?; // create new buffer -//! let buffer = session.attach("test.txt").await?; // attach to it +//! workspace.create("test.txt").await?; // create new buffer +//! let buffer = workspace.attach("test.txt").await?; // attach to it //! let local_change = TextChange { span: 0..0, content: "hello!".into() }; //! buffer.send(local_change)?; // insert some text -//! let remote_change = buffer.recv().await?; +//! let remote_change = buffer.recv().await?; // await remote change //! # //! # Ok(()) //! # } //! ``` //! //! ### sync -//! if async is not viable, including the feature `sync` will provide a sync-only [instance::sync::Instance] variant +//! if async is not viable, a solution might be keeping a global tokio runtime and blocking on it: //! //! ```rust,no_run -//! # use codemp::instance::sync::Instance; +//! # use codemp::client::Client; //! # use std::sync::Arc; //! # use codemp::api::Controller; //! # //! # fn sync_example() -> codemp::Result<()> { -//! let session = Instance::default(); // instantiate sync variant -//! session.connect("http://alemi.dev:50051")?; // connect to server +//! let rt = tokio::runtime::Runtime::new().unwrap(); +//! let mut session = rt.block_on( // using block_on allows calling async code +//! Client::new("http://alemi.dev:50051") +//! )?; +//! +//! rt.block_on(session.login( +//! "username".to_string(), +//! "password".to_string(), +//! Some("some_workspace".to_string()) +//! ))?; +//! +//! let workspace = rt.block_on(session.join_workspace("some_workspace"))?; //! //! // attach to buffer and blockingly receive events -//! let buffer = session.attach("test.txt")?; // attach to buffer, must already exist -//! while let Ok(op) = buffer.blocking_recv(session.rt()) { // must pass runtime +//! let buffer = rt.block_on(workspace.attach("test.txt"))?; // attach to buffer, must already exist +//! while let Ok(op) = rt.block_on(buffer.recv()) { // must pass runtime //! println!("received buffer event: {:?}", op); //! } //! # @@ -87,30 +105,6 @@ //! # } //! ``` //! -//! ### global -//! if instantiating the [Instance] manager is not possible, adding the feature `global` will -//! provide a static lazyly-allocated global reference: [struct@instance::global::INSTANCE]. -//! -//! ```rust,no_run -//! # use codemp::instance::sync::Instance; -//! # use std::sync::Arc; -//! use codemp::prelude::*; // prelude includes everything with "Codemp" in front -//! # fn global_example() -> codemp::Result<()> { -//! CODEMP_INSTANCE.connect("http://alemi.dev:50051")?; // connect to server -//! let cursor = CODEMP_INSTANCE.join("some_workspace")?; // join workspace -//! std::thread::spawn(move || { -//! loop { -//! match cursor.try_recv() { -//! Ok(Some(event)) => println!("received cursor event: {:?}", event), // there might be more -//! Ok(None) => std::thread::sleep(std::time::Duration::from_millis(10)), // wait for more -//! Err(e) => break, // channel closed -//! } -//! } -//! }); -//! # Ok(()) -//! # } -//! ``` -//! //! ## references //! //! ![another cool pic coz why not](https://alemi.dev/img/about-slice-2.png) From 9422f9a21651ee49f8dbebd867fe262e71b60f62 Mon Sep 17 00:00:00 2001 From: alemi Date: Fri, 9 Feb 2024 01:16:16 +0100 Subject: [PATCH 42/42] docs: updated docs --- src/api/change.rs | 1 + src/api/controller.rs | 3 +-- src/lib.rs | 28 ++++++++++++++++------------ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/api/change.rs b/src/api/change.rs index aa597ba..c04e18d 100644 --- a/src/api/change.rs +++ b/src/api/change.rs @@ -62,6 +62,7 @@ impl TextChange { } #[cfg(feature = "woot")] + /// consume the [TextChange], transforming it into a Vec of [woot::crdt::Op] pub fn transform(self, woot: &Woot) -> WootResult> { let mut out = Vec::new(); if self.is_empty() { return Ok(out); } // no-op diff --git a/src/api/controller.rs b/src/api/controller.rs index cf45684..18e5136 100644 --- a/src/api/controller.rs +++ b/src/api/controller.rs @@ -19,8 +19,7 @@ pub(crate) trait ControllerWorker { /// /// this generic trait is implemented by actors managing stream procedures. /// events can be enqueued for dispatching without blocking ([Controller::send]), and an async blocking -/// api ([Controller::recv]) is provided to wait for server events. Additional sync blocking -/// ([Controller::blocking_recv]) is implemented if feature `sync` is enabled. +/// api ([Controller::recv]) is provided to wait for server events. /// /// * if possible, prefer a pure [Controller::recv] consumer, awaiting for events /// * if async is not feasible a [Controller::poll]/[Controller::try_recv] approach is possible diff --git a/src/lib.rs b/src/lib.rs index 5f953ec..5421e05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,8 +5,8 @@ //! > the core library of the codemp project, driving all editor plugins //! //! ## structure -//! The main entrypoint is the [Instance] object, that maintains a connection and can -//! be used to join workspaces or attach to buffers. It contains the underlying [client::Client] and +//! The main entrypoint is the [Client] object, that maintains a connection and can +//! be used to join workspaces or attach to buffers. It contains the underlying [Workspace] and //! stores active controllers. //! //! Some actions will return structs implementing the [api::Controller] trait. These can be polled @@ -21,26 +21,21 @@ //! immediately but instead deferred until compatible. //! //! ## features -//! * `woot` : include the underlying CRDT library and re-exports it (default enabled) //! * `api` : include traits for core interfaces under [api] (default enabled) +//! * `woot` : include the underlying CRDT library and re-exports it (default enabled) //! * `proto` : include GRCP protocol definitions under [proto] (default enabled) //! * `client` : include the local [client] implementation (default enabled) //! //! ## examples -//! the [client::Client] itself is the core structure implementing all methods, plugins should -//! attempt to keep a single instance at any time -//! -//! working sessions are [workspace::Workspace] and while managing multiple ones is in theory -//! possible, it's not really working right now due to how authentication is managed +//! most methods are split between the [Client] itself and the current [Workspace] //! //! ### async //! ```rust,no_run //! use codemp::api::{Controller, TextChange}; -//! # use codemp::client::Client; //! //! # async fn async_example() -> codemp::Result<()> { //! // creating a client session will immediately attempt to connect -//! let mut session = Client::new("http://alemi.dev:50053").await?; +//! let mut session = codemp::Client::new("http://alemi.dev:50053").await?; //! //! // login first, obtaining a new token granting access to 'some_workspace' //! session.login( @@ -73,18 +68,19 @@ //! # } //! ``` //! +//! it's always possible to get a [Workspace] reference using [Client::get_workspace] +//! //! ### sync //! if async is not viable, a solution might be keeping a global tokio runtime and blocking on it: //! //! ```rust,no_run -//! # use codemp::client::Client; //! # use std::sync::Arc; //! # use codemp::api::Controller; //! # //! # fn sync_example() -> codemp::Result<()> { //! let rt = tokio::runtime::Runtime::new().unwrap(); //! let mut session = rt.block_on( // using block_on allows calling async code -//! Client::new("http://alemi.dev:50051") +//! codemp::Client::new("http://alemi.dev:50051") //! )?; //! //! rt.block_on(session.login( @@ -214,3 +210,11 @@ pub mod proto { pub use errors::Error; pub use errors::Result; + +#[cfg(feature = "client")] +pub use client::Client; + +#[cfg(feature = "client")] +pub use workspace::Workspace; + +