fix: generic JsonVec that accepts null

not the cleanest solution but should be generic and transparent

Cargo.toml

@@ -9,7 +9,7 @@ members = [
 	"web",
 	"utils/httpsign",
 	"utils/mdhtml",
-	"utils/uriproxy"
+	"utils/uriproxy",
 ]
 
 [package]
@@ -31,11 +31,11 @@ path = "main.rs"
 toml = "0.8"
 tracing = "0.1"
 tracing-subscriber = "0.3"
-sea-orm = "0.12"
 clap = { version = "4.5", features = ["derive"] }
 signal-hook = "0.3"
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
 tokio = { version = "1.35", features = ["full"] } # TODO slim this down
+sea-orm = { version = "0.12", features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls"] }
 futures = "0.3"
 
 upub = { path = "upub/core" }

apb/Cargo.toml

@@ -18,7 +18,6 @@ chrono = { version = "0.4", features = ["serde"] }
 thiserror = "1"
 paste = "1.0"
 tracing = "0.1"
-async-trait = "0.1"
 serde_json = { version = "1", optional = true }
 sea-orm = { version = "0.12", optional = true, default-features = false }
 reqwest = { version = "0.12", features = ["json"], optional = true }

upub/cli/Cargo.toml

@@ -20,6 +20,6 @@ uuid = { version = "1.8", features = ["v4"] }
 chrono = { version = "0.4", features = ["serde"] }
 openssl = "0.10" # TODO handle pubkeys with a smaller crate
 clap = { version = "4.5", features = ["derive"] }
-sea-orm = { version = "0.12", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"] }
+sea-orm = "0.12"
 futures = "0.3"
 mdhtml = { path = "../../utils/mdhtml/" }

upub/cli/src/faker.rs

@@ -1,4 +1,4 @@
-use upub::model::{addressing, config, credential, activity, object, actor, Audience};
+use upub::{ext::JsonVec, model::{activity, actor, addressing, config, credential, object}};
 use openssl::rsa::Rsa;
 use sea_orm::{ActiveValue::NotSet, IntoActiveModel};
 
@@ -21,8 +21,8 @@ pub async fn faker(ctx: upub::Context, count: i64) -> Result<(), sea_orm::DbErr>
 		followers: None,
 		followers_count: 0,
 		statuses_count: count as i32,
-		fields: vec![],
+		fields: JsonVec::default(),
-		also_known_as: vec![],
+		also_known_as: JsonVec::default(),
 		moved_to: None,
 		icon: Some("https://cdn.alemi.dev/social/circle-square.png".to_string()),
 		image: Some("https://cdn.alemi.dev/social/someriver-xs.jpg".to_string()),
@@ -90,10 +90,10 @@ pub async fn faker(ctx: upub::Context, count: i64) -> Result<(), sea_orm::DbErr>
 			likes: Set(0),
 			announces: Set(0),
 			audience: Set(None),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
-			bto: Set(Audience::default()),
+			bto: Set(JsonVec::default()),
-			cc: Set(Audience(vec![])),
+			cc: Set(JsonVec(vec![])),
-			bcc: Set(Audience::default()),
+			bcc: Set(JsonVec::default()),
 			url: Set(None),
 			sensitive: Set(false),
 		}).exec(db).await?;
@@ -106,10 +106,10 @@ pub async fn faker(ctx: upub::Context, count: i64) -> Result<(), sea_orm::DbErr>
 			object: Set(Some(format!("{domain}/objects/{oid}"))),
 			target: Set(None),
 			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i as u64)),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
-			bto: Set(Audience::default()),
+			bto: Set(JsonVec::default()),
-			cc: Set(Audience(vec![])),
+			cc: Set(JsonVec(vec![])),
-			bcc: Set(Audience::default()),
+			bcc: Set(JsonVec::default()),
 		}).exec(db).await?;
 	}
 

upub/core/Cargo.toml

@@ -21,7 +21,6 @@ openssl = "0.10" # TODO handle pubkeys with a smaller crate
 base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }
 uuid = { version = "1.8", features = ["v4"] }
-regex = "1.10"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_default = "0.1"
@@ -29,10 +28,10 @@ serde-inline-default = "0.2"
 toml = "0.8"
 uriproxy = { path = "../../utils/uriproxy" }
 httpsign = { path = "../../utils/httpsign/" }
+jsonvec = { path = "../../utils/jsonvec/" }
 jrd = "0.1"
 tracing = "0.1"
-tokio = { version = "1.35", features = ["full"] } # TODO slim this down
+sea-orm = { version = "0.12", features = ["macros"] }
-sea-orm = { version = "0.12", features = ["macros", "sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls"] }
 reqwest = { version = "0.12", features = ["json"] }
 apb = { path = "../../apb", features = ["unstructured", "orm", "did-core", "activitypub-miscellaneous-terms", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
 # nodeinfo = "0.0.2" # the version on crates.io doesn't re-export necessary types to build the struct!!!

upub/core/src/ext.rs

@@ -48,3 +48,66 @@ impl<T, E: std::error::Error> LoggableError for Result<T, E> {
 		}
 	}
 }
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct JsonVec<T>(pub Vec<T>);
+
+impl<T> From<Vec<T>> for JsonVec<T> {
+	fn from(value: Vec<T>) -> Self {
+		JsonVec(value)
+	}
+}
+
+impl<T> Default for JsonVec<T> {
+	fn default() -> Self {
+		JsonVec(Vec::new())
+	}
+}
+
+impl<T: serde::de::DeserializeOwned> sea_orm::TryGetableFromJson for JsonVec<T> {}
+
+impl<T: serde::ser::Serialize> std::convert::From<JsonVec<T>> for sea_orm::Value {
+	fn from(source: JsonVec<T>) -> Self {
+		sea_orm::Value::Json(serde_json::to_value(&source).ok().map(std::boxed::Box::new))
+	}
+}
+
+impl<T: serde::de::DeserializeOwned + TypeName> sea_orm::sea_query::ValueType for JsonVec<T> {
+	fn try_from(v: sea_orm::Value) -> Result<Self, sea_orm::sea_query::ValueTypeErr> {
+		match v {
+			sea_orm::Value::Json(Some(json)) => Ok(
+				serde_json::from_value(*json).map_err(|_| sea_orm::sea_query::ValueTypeErr)?,
+			),
+			sea_orm::Value::Json(None) => Ok(JsonVec::default()),
+			_ => Err(sea_orm::sea_query::ValueTypeErr),
+		}
+	}
+
+	fn type_name() -> String {
+		format!("JsonVec_{}", T::type_name())
+	}
+
+	fn array_type() -> sea_orm::sea_query::ArrayType {
+		sea_orm::sea_query::ArrayType::Json
+	}
+
+	fn column_type() -> sea_orm::sea_query::ColumnType {
+		sea_orm::sea_query::ColumnType::Json
+	}
+}
+
+impl<T> sea_orm::sea_query::Nullable for JsonVec<T> {
+	fn null() -> sea_orm::Value {
+		sea_orm::Value::Json(None)
+	}
+}
+
+pub trait TypeName {
+	fn type_name() -> String;
+}
+
+impl TypeName for String {
+	fn type_name() -> String {
+		"String".to_string()
+	}
+}

upub/core/src/init.rs

@@ -1,7 +1,7 @@
 use openssl::rsa::Rsa;
 use sea_orm::{ActiveValue::{NotSet, Set}, DatabaseConnection, EntityTrait};
 
-use crate::model;
+use crate::{ext::JsonVec, model};
 
 #[derive(Debug, thiserror::Error)]
 pub enum InitError {
@@ -34,9 +34,9 @@ pub async fn application(
 					domain: Set(domain.clone()),
 					preferred_username: Set(domain.clone()),
 					actor_type: Set(apb::ActorType::Application),
-					also_known_as: Set(vec![]),
+					also_known_as: Set(JsonVec::default()),
 					moved_to: Set(None),
-					fields: Set(vec![]), // TODO we could put some useful things here actually
+					fields: Set(JsonVec::default()), // TODO we could put some useful things here actually
 					private_key: Set(Some(privk)),
 					public_key: Set(pubk),
 					following: Set(None),

upub/core/src/model/activity.rs

@@ -1,7 +1,7 @@
 use apb::{ActivityMut, ActivityType, BaseMut, ObjectMut};
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::model::Audience;
+use crate::ext::JsonVec;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "activities")]
@@ -14,10 +14,10 @@ pub struct Model {
 	pub actor: String,
 	pub object: Option<String>,
 	pub target: Option<String>,
-	pub to: Audience,
+	pub to: JsonVec<String>,
-	pub bto: Audience,
+	pub bto: JsonVec<String>,
-	pub cc: Audience,
+	pub cc: JsonVec<String>,
-	pub bcc: Audience,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 }
 

upub/core/src/model/actor.rs

@@ -2,7 +2,9 @@ use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
 use apb::{field::OptionalString, ActorMut, ActorType, BaseMut, DocumentMut, EndpointsMut, ObjectMut, PublicKeyMut};
 
-#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, sea_orm::FromJsonQueryResult)]
+use crate::ext::{JsonVec, TypeName};
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub struct Field {
 	pub name: String,
 	pub content: String,
@@ -12,6 +14,12 @@ pub struct Field {
 	pub field_type: String,
 }
 
+impl TypeName for Field {
+	fn type_name() -> String {
+		"Field".to_string()
+	}
+}
+
 impl<T: apb::Object> From<T> for Field {
 	fn from(value: T) -> Self {
 		Field {
@@ -37,7 +45,7 @@ pub struct Model {
 	pub image: Option<String>,
 	pub icon: Option<String>,
 	pub preferred_username: String,
-	pub fields: Vec<Field>,
+	pub fields: JsonVec<Field>,
 	pub inbox: Option<String>,
 	pub shared_inbox: Option<String>,
 	pub outbox: Option<String>,
@@ -50,7 +58,7 @@ pub struct Model {
 	pub private_key: Option<String>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
-	pub also_known_as: Vec<String>,
+	pub also_known_as: JsonVec<String>,
 	pub moved_to: Option<String>,
 }
 
@@ -208,7 +216,7 @@ impl Model {
 					.set_url(apb::Node::link(i.clone()))
 			)))
 			.set_attachment(apb::Node::array(
-				self.fields
+				self.fields.0
 					.into_iter()
 					.filter_map(|x| serde_json::to_value(x).ok())
 					.collect()
@@ -233,7 +241,7 @@ impl Model {
 				apb::new()
 					.set_shared_inbox(self.shared_inbox.as_deref())
 			))
-			.set_also_known_as(apb::Node::links(self.also_known_as))
+			.set_also_known_as(apb::Node::links(self.also_known_as.0))
 			.set_moved_to(apb::Node::maybe_link(self.moved_to))
 			.set_discoverable(Some(true))
 	}

upub/core/src/model/mod.rs

@@ -19,23 +19,3 @@ pub mod dislike;
 pub mod hashtag;
 pub mod mention;
 pub mod attachment;
-
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, sea_orm::FromJsonQueryResult)]
-pub struct Audience(pub Vec<String>);
-
-impl<T: apb::Base> From<apb::Node<T>> for Audience {
-	fn from(value: apb::Node<T>) -> Self {
-		use apb::field::OptionalString;
-
-		Audience(
-			match value {
-				apb::Node::Empty => vec![],
-				apb::Node::Link(l) => l.href().map(|x| vec![x.to_string()]).unwrap_or_default(),
-				apb::Node::Object(o) => if let Ok(id) = o.id() { vec![id.to_string()] } else { vec![] },
-				apb::Node::Array(arr) => arr.into_iter().filter_map(|l| l.id().str()).collect(),
-			}
-		)
-	}
-}
-

upub/core/src/model/object.rs

@@ -1,7 +1,7 @@
 use apb::{BaseMut, CollectionMut, DocumentMut, ObjectMut, ObjectType};
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use super::Audience;
+use crate::ext::JsonVec;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "objects")]
@@ -24,10 +24,10 @@ pub struct Model {
 	pub announces: i32,
 	pub replies: i32,
 	pub context: Option<String>,
-	pub to: Audience,
+	pub to: JsonVec<String>,
-	pub bto: Audience,
+	pub bto: JsonVec<String>,
-	pub cc: Audience,
+	pub cc: JsonVec<String>,
-	pub bcc: Audience,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
 

upub/core/src/traits/admin.rs

@@ -1,5 +1,7 @@
 use sea_orm::{ActiveValue::{NotSet, Set}, DbErr, EntityTrait};
 
+use crate::ext::JsonVec;
+
 #[async_trait::async_trait]
 pub trait Administrable {
 	async fn register_user(
@@ -35,13 +37,13 @@ impl Administrable for crate::Context {
 			domain: Set(domain),
 			summary: Set(summary),
 			preferred_username: Set(username.clone()),
-			fields: Set(vec![]),
+			fields: Set(JsonVec::default()),
 			following: Set(None),
 			following_count: Set(0),
 			followers: Set(None),
 			followers_count: Set(0),
 			statuses_count: Set(0),
-			also_known_as: Set(vec![]),
+			also_known_as: Set(JsonVec::default()),
 			moved_to: Set(None),
 			icon: Set(avatar_url),
 			image: Set(banner_url),

upub/core/src/traits/normalize.rs

@@ -203,10 +203,10 @@ impl AP {
 			object: activity.object().id().str(),
 			target: activity.target().id().str(),
 			published: activity.published().unwrap_or(chrono::Utc::now()),
-			to: activity.to().into(),
+			to: activity.to().all_ids().into(),
-			bto: activity.bto().into(),
+			bto: activity.bto().all_ids().into(),
-			cc: activity.cc().into(),
+			cc: activity.cc().all_ids().into(),
-			bcc: activity.bcc().into(),
+			bcc: activity.bcc().all_ids().into(),
 		})
 	}
 
@@ -287,10 +287,10 @@ impl AP {
 			announces: object.shares().get()
 				.map_or(0, |x| x.total_items().unwrap_or(0)) as i32,
 			audience: object.audience().id().str(),
-			to: object.to().into(),
+			to: object.to().all_ids().into(),
-			bto: object.bto().into(),
+			bto: object.bto().all_ids().into(),
-			cc: object.cc().into(),
+			cc: object.cc().all_ids().into(),
-			bcc: object.bcc().into(),
+			bcc: object.bcc().all_ids().into(),
 
 			sensitive: object.sensitive().unwrap_or(false),
 		})
@@ -333,13 +333,12 @@ impl AP {
 			summary: actor.summary().str(),
 			icon: actor.icon().get().and_then(|x| x.url().id().str()),
 			image: actor.image().get().and_then(|x| x.url().id().str()),
-			fields: actor.attachment().flat().into_iter().filter_map(|x| Some(crate::model::actor::Field::from(x.extract()?))).collect(),
 			inbox: actor.inbox().id().str(),
 			outbox: actor.outbox().id().str(),
 			shared_inbox: actor.endpoints().get().and_then(|x| x.shared_inbox().str()),
 			followers: actor.followers().id().str(),
 			following: actor.following().id().str(),
-			also_known_as: actor.also_known_as().flat().into_iter().filter_map(|x| x.id().str()).collect(),
+			also_known_as: actor.also_known_as().flat().into_iter().filter_map(|x| x.id().str()).collect::<Vec<String>>().into(),
 			moved_to: actor.moved_to().id().str(),
 			published: actor.published().unwrap_or(chrono::Utc::now()),
 			updated: chrono::Utc::now(),
@@ -348,6 +347,12 @@ impl AP {
 			statuses_count: actor.statuses_count().unwrap_or(0) as i32,
 			public_key: actor.public_key().get().ok_or(apb::FieldErr("publicKey"))?.public_key_pem().to_string(),
 			private_key: None, // there's no way to transport privkey over AP json, must come from DB
+			fields: actor.attachment()
+				.flat()
+				.into_iter()
+				.filter_map(|x| Some(crate::model::actor::Field::from(x.extract()?)))
+				.collect::<Vec<crate::model::actor::Field>>()
+				.into(),
 		})
 	}
 

upub/routes/Cargo.toml

@@ -13,10 +13,7 @@ readme = "README.md"
 [dependencies]
 thiserror = "1"
 rand = "0.8"
-sha2 = "0.10"
 sha256 = "1.5" # TODO ughhh
-hmac = "0.12"
-base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

upub/worker/Cargo.toml

@@ -16,12 +16,10 @@ tracing = "0.1"
 async-trait = "0.1"
 serde_json = "1"
 sea-orm = "0.12"
-jrd = "0.1"
 regex = "1.10"
 chrono = { version = "0.4", features = ["serde"] }
 tokio = { version = "1.35", features = ["full"] } # TODO slim this down
 reqwest = { version = "0.12", features = ["json"] }
 apb = { path = "../../apb", features = ["unstructured", "orm", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
-httpsign = { path = "../../utils/httpsign/" }
 mdhtml = { path = "../../utils/mdhtml/" }
 upub = { path = "../core/" }

web/Cargo.toml

@@ -12,7 +12,6 @@ repository = "https://git.alemi.dev/upub.git"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-trait = "0.1"
 lazy_static = "1.4"
 tracing = "0.1"
 tracing-subscriber = "0.3"