Merge branch 'dev' into pyo3_bump

.github/workflows/ci.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - dev
-      - main
 
 env:
   CARGO_TERM_COLOR: always
@@ -14,12 +13,19 @@ jobs:
     name: Rust project - latest
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
+        features:
+          - rust
+          - lua
+          - java
+          - js
+          - python
         toolchain:
           - stable
         #  - beta
         #  - nightly
-        # disable other toolchains to save on github runners
+        # TODO should re-enable future toolchains so we get warnings on breaking changes
     steps:
       - uses: arduino/setup-protoc@v2
       - uses: actions/checkout@v3
@@ -27,5 +33,5 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.ACTIONS_SSH_DEPLOY_KEY }}
       - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
-      - run: cargo build --verbose --all-features
-      - run: cargo test --verbose --all-features
+      - run: cargo build --verbose --features=${{ matrix.features }}
+      - run: cargo test --verbose --features=${{ matrix.features }}

Cargo.toml

@@ -50,6 +50,7 @@ pyo3-build-config = { version = "0.19", optional = true }
 
 [features]
 default = []
+rust = [] # used for ci matrix
 lua = ["mlua", "tracing-subscriber"]
 java = ["lazy_static", "jni", "tracing-subscriber"]
 js = ["napi-build", "tracing-subscriber", "napi", "napi-derive"]

README.md

@@ -1,22 +1,102 @@
 # codemp
-This project is heavily inspired by Microsoft Live Share plugin for Visual Studio (Code).
-While the functionality is incredibly good, I often find issues or glitches which slow me down, and being locked to only use Visual Studio products is limiting.
-I decided to write my own solution, and to make it open source, so that any editor can integrate it with a plugin.
 
-### Documentation
-build the crate documentation with `cargo doc` and access the codemp page with the html redirect `docs.html`
+<a href="https://codemp.dev"><img alt="codemp logo" align="center" src="https://codemp.dev/codemp-t.png" height="100" /></a>
 
-# Design
-## Client/Server
-While initially a P2P system seemed interesting, I don't think it would easily scale with many users (due to having to integrate many changes on each client).
-I decided to build a client/server architecture, with a central "Workspace" managed by the server application and multiple clients connecting to it.
-Each client will only have to care about keeping itself in sync with the server (remembering local-only changes and acknowledged changes), leaving the task of keeping track of differences to the server.
+### code multiplexer
 
-## Plugins
-This software will probably be distribuited as a standalone binary that editors can use to connect to a "Workspace". A dynamic library object might also be a choice.
-Each editor plugin must be responsible of mapping codemp functionality to actual editor capabilities, bridging codemp client to the editor itself. The client should be able to handle a session autonomously.
+> CodeMP is a **collaborative** text editing plugin to work remotely.
+It seamlessly integrates in your editor providing remote cursors and instant text synchronization,
+as well as a remote virtual workspace for you and your team.
 
-## Text Synchronization
-A non destructive way to sync changes across clients is necessary.
-I initially explored CRDTs, but implementation seemed complex with little extra benefits from "more traditional" approaches (Operational Transforms).
-This has to be investigated more.
+> CodeMP is build with state-of-the-art CRDT technology, guaranteeing eventual consistency.
+This means everyone in a workspace will always be working on the exact same file _eventually_:
+even under unreliable networks or constrained resources, the underlying CRDT will always reach a 
+convergent state across all users. Even with this baseline, CodeMP's proto is optimized for speed 
+and low network footprint, meaning even slow connections can provide stable real-time editing.
+
+# using this project
+CodeMP is available for many editors as plugins.
+
+Currently we support:
+ - [VSCode](https://github.com/hexedtech/codemp-vscode)
+ - [Intellij](https://github.com/hexedtech/codemp-intellij)
+ - [NeoVim](https://github.com/hexedtech/codemp-nvim)
+ - [Sublime Text](https://github.com/hexedtech/codemp-sublime)
+
+# using this library
+This is the main client library for codemp. It exposes functions to interact with the codemp client itself, its workspaces and buffers.
+
+All memory is managed by the library itself, which gives out always atomic reference-counted pointers to internally mutable objects. The host program needs only to connect a client first, and from that reference can retrieve every other necessary component.
+
+### from rust
+This library is primarily a rust crate, so rust applications will get the best possible integration.
+
+Just `cargo add codemp` and check the docs for some examples.
+
+### from supported languages
+This library provides first-class bindings for:
+ - java
+ - javascript
+ - python
+ - lua
+
+For any of these languages, just add `codemp` as a dependency in your project.
+
+The API should perfectly mimic what rust exposes underneath, so the main rust docs can still be used as reference for available methods and objects.
+
+### from other languages
+> [!WARNING]
+> The common C bindings are still not available
+
+Any other language with C ffi capabilities can use codemp via its bare C bindings.
+This will be more complex and may require wrapping the native calls underneath.
+
+# documentation
+This project is mainly a rust crate, so the most up-to-date and extended documentation will be found on docs.rs.
+ - Check [docs.rs/codemp](https://docs.rs/codemp) for our full documentation!
+
+# architecture
+CodeMP is built from scratch to guarantee impeccable performance and accuracy.
+The following architectural choices are driven by this very strict requirement.
+
+## interop: FFI
+The first challenge of developing such a system is adoption: getting all your colleagues to switch to your editor is not going to happen. Supporting a multitude of plugins in different languages and possibly different architectures however is a daunting task even for larger teams.
+
+Our solution is a single common native library, developed in safe and performant Rust, which can be used by any plugin with a thin layer of glue code to provide native bindings.
+
+This allows us to maintain a single client codebase and multiple plugins, rather than multiple clients and plugins, with the cost of FFI complexity.
+
+We took a gamble which paid off: our team was capable enough to handle cross compiling and multiple bindings, and can now focus on first-class integration in each editor API.
+
+## synchronization: CRDT
+Our investigations in the field of text synchronization for multi agent editing showed that there are mostly two approached to solve the problem: Operational Transforms (older, more used) and Conflict-free Replicated Data Structures (CRDTs, a newer technology)
+
+While initial prototypes used OT to achieve syncrhonization, we quickly found issues. The editor is not under our plugin's control, and could always apply new insertions/deletions while processing remote changes. This was a huge issue with OTs, as it would require control over the integration process.
+
+We introduced CRDTs first with a hand-crafted naive approach, and were very impressed by the results. Because of the nature of CRDTs, we have an internal state which is always kept in sync with the server (and all other peers), and this state can then be finely synchronized with the effective editor state. Edits coming while integrating just branch more, and our inner CRDT merges those seemlessly.
+
+We recently swapped our internal library for a production-grade solution: [diamond-types](https://github.com/josephg/diamond-types), with even more impressive results: we jumped from processing ~2 thousand operations per second to an astonishing **~8 million**, a `1000x` improvement!
+
+## layout: star (client/server)
+Network layout posed a challenging decision: a distributed system could provide lower latency but a centralized arbiter could dramatically reduce necessary resources for each peer.
+
+We want codemp to be a viable solution on low power devices in unreliable networks, so opted to a centralized approach.
+
+While for small work groups the benefits are negligible, bigger sessions dramatically benefit from having a central server which handles reduntant merging and skips irrelevant operations, while masking IPs and removing the problem of punching through NATs.
+
+We hope to provide a solution capable of scaling to hundreds or thousands of concurrent users, in order to open new uses in conferences, competitions, teaching and live entertainment.
+
+## protocol: streams (grpc)
+The underlying network structure is really important to achieve good performance. We need a binary stream to quickly beam back and forth operations.
+
+GRPC provides this, encapsulating is convenient to use primitives, while also providing request/response procedures.
+
+We plan to experiment with laminar and capnproto for the fast cursor and operation streams, but we will probably retain an http-based approach for workspace management and authentication.
+
+# contributing
+> [!NOTE]
+> This project is maintained by [hexedtech](https://hexed.technology).
+
+If you find bugs or would like to see new features implemented, be sure to open an issue on this repository.
+
+In case you wished to contribute code, that's great! We love external contributions, but we require you to **sign our CLA first** (which is not yet ready, TODO!)

dist/java/src/mp/code/Workspace.java

@@ -1,6 +1,7 @@
 package mp.code;
 
 import java.util.Optional;
+import java.util.UUID;
 
 import mp.code.data.DetachResult;
 import mp.code.exceptions.CodeMPException;
@@ -57,8 +58,8 @@ public class Workspace {
 		fetch_buffers(this.ptr);
 	}
 
-	private static native String[] list_buffer_users(long self, String path) throws CodeMPException;
-	public String[] listBufferUsers(String path) throws CodeMPException {
+	private static native UUID[] list_buffer_users(long self, String path) throws CodeMPException;
+	public UUID[] listBufferUsers(String path) throws CodeMPException {
 		return list_buffer_users(this.ptr, path);
 	}
 
@@ -104,8 +105,10 @@ public class Workspace {
 			} else return Optional.empty();
 		}
 
-		public boolean hasFileTreeUpdated() {
-			return type == Type.FILE_TREE_UPDATED;
+		public Optional<String> getTargetBuffer() {
+			if(this.type == Type.FILE_TREE_UPDATED) {
+				return Optional.of(this.argument);
+			} else return Optional.empty();
 		}
 
 		private enum Type {

src/ffi/java/buffer.rs

@@ -1,4 +1,4 @@
-use jni::{objects::{JClass, JObject, JString, JValueGen}, sys::{jlong, jobject, jstring}, JNIEnv};
+use jni::{objects::{JClass, JObject, JValueGen}, sys::{jlong, jobject, jstring}, JNIEnv};
 
 use crate::api::Controller;
 

src/ffi/java/mod.rs

@@ -79,3 +79,25 @@ impl<T> JExceptable<T> for Result<T, uuid::Error> where T: Default {
 		self.unwrap_or_default()
 	}
 }
+
+/// Allows easy conversion for various types into Java objects.
+/// This is essentially the same as [TryInto], but that can't be emplemented on non-local types.
+pub(crate) trait JObjectify<'local> {
+	/// The error type, likely to be [jni::errors::Error].
+	type Error;
+
+	/// Attempts to convert the given object to a [jni::objects::JObject].
+	fn jobjectify(self, env: &mut jni::JNIEnv<'local>) -> Result<jni::objects::JObject<'local>, Self::Error>;
+}
+
+impl<'local> JObjectify<'local> for uuid::Uuid {
+	type Error = jni::errors::Error;
+	fn jobjectify(self, env: &mut jni::JNIEnv<'local>) -> Result<jni::objects::JObject<'local>, jni::errors::Error> {
+		env.find_class("java/util/UUID").and_then(|class| {
+			let (msb, lsb) = self.as_u64_pair();
+			let msb = i64::from_ne_bytes(msb.to_ne_bytes());
+			let lsb = i64::from_ne_bytes(lsb.to_ne_bytes());
+			env.new_object(&class, "(JJ)V", &[jni::objects::JValueGen::Long(msb), jni::objects::JValueGen::Long(lsb)])
+		})
+	}
+}

src/ffi/java/workspace.rs

@@ -1,7 +1,7 @@
 use jni::{objects::{JClass, JObject, JString, JValueGen}, sys::{jlong, jobject, jobjectArray, jstring}, JNIEnv};
 use crate::Workspace;
 
-use super::{JExceptable, RT};
+use super::{JExceptable, JObjectify, RT};
 
 /// Gets the workspace id.
 #[no_mangle]
@@ -166,11 +166,11 @@ pub extern "system" fn Java_mp_code_Workspace_list_1buffer_1users<'local>(
 	let users = RT.block_on(workspace.list_buffer_users(&buffer))
 		.jexcept(&mut env);
 
-	env.find_class("java/lang/String")
-		.and_then(|class| env.new_object_array(users.len() as i32, class, JObject::null()))
+	env.find_class("java/util/UUID")
+		.and_then(|class| env.new_object_array(users.len() as i32, &class, JObject::null()))
 		.map(|arr| {
 			for (idx, user) in users.iter().enumerate() {
-				env.new_string(&user.id)
+				user.id.jobjectify(&mut env)
 					.and_then(|id| env.set_object_array_element(&arr, idx as i32, id))
 					.jexcept(&mut env);
 			}
@@ -204,17 +204,14 @@ pub extern "system" fn Java_mp_code_Workspace_event(
 	RT.block_on(workspace.event())
 		.map(|event| {
 			let (name, arg) = match event {
-				crate::api::Event::FileTreeUpdated => ("FILE_TREE_UPDATED", None),
-				crate::api::Event::UserJoin(arg) => ("USER_JOIN", Some(arg)),
-				crate::api::Event::UserLeave(arg) => ("USER_LEAVE", Some(arg)),
+				crate::api::Event::FileTreeUpdated(arg) => ("FILE_TREE_UPDATED", env.new_string(arg).unwrap_or_default()),
+				crate::api::Event::UserJoin(arg) => ("USER_JOIN", env.new_string(arg).unwrap_or_default()),
+				crate::api::Event::UserLeave(arg) => ("USER_LEAVE", env.new_string(arg).unwrap_or_default()),
 			};
 			let event_type = env.find_class("mp/code/Workspace$Event$Type")
 				.and_then(|class| env.get_static_field(class, name, "Lmp/code/Workspace/Event/Type;"))
 				.and_then(|f| f.l())
 				.jexcept(&mut env);
-			let arg = arg.map(|s| env.new_string(s).jexcept(&mut env))
-				.unwrap_or_default();
-
 			env.find_class("mp/code/Workspace$Event").and_then(|class|
 				env.new_object(
 					class,

src/ffi/js/buffer.rs

@@ -7,8 +7,8 @@ use crate::buffer::controller::BufferController;
 
 #[napi]
 impl BufferController {
-	#[napi(ts_args_type = "fun: (event: TextChange) => void")]
-	pub fn callback(&self, fun: napi::JsFunction) -> napi::Result<()>{ 
+	#[napi(js_name = "callback", ts_args_type = "fun: (event: TextChange) => void")]
+	pub fn jscallback(&self, fun: napi::JsFunction) -> napi::Result<()>{ 
 		let tsfn : ThreadsafeFunction<crate::api::TextChange, Fatal> = 
 		fun.create_threadsafe_function(0,
 			|ctx : ThreadSafeCallContext<crate::api::TextChange>| {
@@ -41,4 +41,4 @@ impl BufferController {
 	pub async fn js_send(&self, op: TextChange) -> napi::Result<()> {
 		Ok(self.send(op).await?)
 	}
-}
+}

src/ffi/js/cursor.rs

@@ -42,8 +42,8 @@ impl From<crate::api::Cursor> for JsCursor {
 
 #[napi]
 impl CursorController {
-	#[napi(ts_args_type = "fun: (event: Cursor) => void")]
-	pub fn callback(&self, fun: napi::JsFunction) -> napi::Result<()>{ 
+	#[napi(js_name = "callback", ts_args_type = "fun: (event: Cursor) => void")]
+	pub fn jscallback(&self, fun: napi::JsFunction) -> napi::Result<()>{ 
 		let tsfn : ThreadsafeFunction<JsCursor, ErrorStrategy::Fatal> = 
 		fun.create_threadsafe_function(0,
 			|ctx : ThreadSafeCallContext<JsCursor>| {

src/ffi/js/workspace.rs

@@ -12,8 +12,8 @@ impl Workspace {
 	}
 	
 	#[napi(js_name = "filetree")]
-	pub fn js_filetree(&self) -> Vec<String> {
-		self.filetree()
+	pub fn js_filetree(&self, filter: Option<&str>) -> Vec<String> {
+		self.filetree(filter)
 	}
 
 	#[napi(js_name = "cursor")]
@@ -41,4 +41,4 @@ impl Workspace {
 		Ok(self.delete(&path).await?)
 	}
 
-}
+}

src/ffi/lua.rs

@@ -166,12 +166,15 @@ impl LuaUserData for CodempWorkspace {
 			});
 			Ok(())
 		});
+
+		methods.add_method("filetree", |_, this, (filter,):(Option<String>,)|
+			Ok(this.filetree(filter.as_deref()))
+		);
 	}
 
 	fn add_fields<'lua, F: LuaUserDataFields<'lua, Self>>(fields: &mut F) {
 		fields.add_field_method_get("id", |_, this| Ok(this.id()));
 		fields.add_field_method_get("cursor", |_, this| Ok(this.cursor()));
-		fields.add_field_method_get("filetree", |_, this| Ok(this.filetree()));
 		fields.add_field_method_get("active_buffers", |_, this| Ok(this.buffer_list()));
 		// fields.add_field_method_get("users", |_, this| Ok(this.0.users())); // TODO
 	}

src/ffi/python/workspace.rs

@@ -83,7 +83,7 @@ impl Workspace {
 	}
 
 	#[pyo3(name = "filetree")]
-	fn pyfiletree(&self) -> Vec<String> {
-		self.filetree()
+	fn pyfiletree(&self, filter: Option<&str>) -> Vec<String> {
+		self.filetree(filter)
 	}
 }

src/workspace/worker.rs

@@ -334,8 +334,11 @@ impl Workspace {
 
 	/// get the currently cached "filetree"
 	// #[cfg_attr(feature = "js", napi)] // https://github.com/napi-rs/napi-rs/issues/1120
-	pub fn filetree(&self) -> Vec<String> {
-		self.0.filetree.iter().map(|f| f.clone()).collect()
+	pub fn filetree(&self, filter: Option<&str>) -> Vec<String> {
+		self.0.filetree.iter()
+			.filter(|f| filter.map_or(true, |flt| f.starts_with(flt)))
+			.map(|f| f.clone())
+			.collect()
 	}
 }