fix: object updates should only touch some fields

also reject updates for other types

.tci

@@ -1,19 +1,19 @@
 #!/bin/bash
 
 echo "building release binary"
-cargo build --release --all-features -j 1 # limit memory usage
+cargo build --release --all-features -j 4
 echo "stopping service"
 systemctl --user stop upub
 echo "installing new binary"
 cp ./target/release/upub /opt/bin/upub
 echo "migrating database"
-/opt/bin/upub --db "sqlite:///srv/tci/upub.db" --domain https://upub.alemi.dev migrate
+/opt/bin/upub -c /etc/upub/config.toml migrate
 echo "restarting service"
 systemctl --user start upub
 echo "rebuilding frontend"
 cd web
-CARGO_BUILD_JOBS=1 /opt/bin/trunk build --release --public-url 'https://upub.alemi.dev/web'
+CARGO_BUILD_JOBS=4 /opt/bin/trunk build --release --public-url 'https://dev.upub.social/web'
 echo "deploying frontend"
-rm /srv/http/upub/web/*
+rm /srv/http/upub/dev/web/*
-mv ./dist/* /srv/http/upub/web/
+mv ./dist/* /srv/http/upub/dev/web/
 echo "done"

Cargo.toml

@@ -1,9 +1,20 @@
 [workspace]
-members = ["apb", "web", "mdhtml", "uriproxy"]
+members = [
+	"apb",
+	"upub/core",
+	"upub/cli",
+	"upub/migrations",
+	"upub/routes",
+	"upub/worker",
+	"web",
+	"utils/httpsign",
+	"utils/mdhtml",
+	"utils/uriproxy",
+]
 
 [package]
-name = "upub"
+name = "upub-bin"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 authors = [ "alemi <me@alemi.dev>" ]
 description = "Traits and types to handle ActivityPub objects"
@@ -12,46 +23,30 @@ keywords = ["activitypub", "activitystreams", "json"]
 repository = "https://git.alemi.dev/upub.git"
 readme = "README.md"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "upub"
+path = "main.rs"
 
 [dependencies]
-thiserror = "1"
-rand = "0.8"
-sha256 = "1.5"
-openssl = "0.10" # TODO handle pubkeys with a smaller crate
-base64 = "0.22"
-chrono = { version = "0.4", features = ["serde"] }
-uuid = { version = "1.8", features = ["v4"] }
-regex = "1.10"
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-serde_default = "0.1"
-serde-inline-default = "0.2"
 toml = "0.8"
-mdhtml = { path = "mdhtml", features = ["markdown"] }
-uriproxy = { path = "uriproxy" }
-jrd = "0.1"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 clap = { version = "4.5", features = ["derive"] }
+signal-hook = "0.3"
+signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
+tokio = { version = "1.40", features = ["full"] } # TODO slim this down
+sea-orm = { version = "1.0", features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls"] }
 futures = "0.3"
-tokio = { version = "1.35", features = ["full"] } # TODO slim this down
+
-sea-orm = { version = "0.12", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"] }
+upub = { path = "upub/core" }
-reqwest = { version = "0.12", features = ["json"] }
+upub-cli = { path = "upub/cli", optional = true }
-axum = "0.7"
+upub-migrations = { path = "upub/migrations", optional = true }
-tower-http = { version = "0.5", features = ["cors", "trace"] }
+upub-routes = { path = "upub/routes", optional = true }
-apb = { path = "apb", features = ["unstructured", "orm", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
+upub-worker = { path = "upub/worker", optional = true }
-# nodeinfo = "0.0.2" # the version on crates.io doesn't re-export necessary types to build the struct!!!
-nodeinfo = { git = "https://codeberg.org/thefederationinfo/nodeinfo-rs", rev = "e865094804" }
-# migrations
-sea-orm-migration = { version = "0.12", optional = true }
-# mastodon
-mastodon-async-entities = { version = "1.1.0", optional = true }
-time = { version = "0.3", features = ["serde"], optional = true }
-async-recursion = "1.1"
 
 [features]
-default = ["mastodon", "migrations", "cli"]
+default = ["serve", "migrate", "cli", "worker"]
-cli = []
+serve = ["dep:upub-routes"]
-migrations = ["dep:sea-orm-migration"]
+migrate = ["dep:upub-migrations"]
-mastodon = ["dep:mastodon-async-entities", "dep:time"]
+cli = ["dep:upub-cli"]
+worker = ["dep:upub-worker"]

README.md

@@ -1,7 +1,7 @@
 # μpub
-> micro social network, federated
+> [micro social network, federated](https://join.upub.social)
 
-![screenshot of upub simple frontend](https://cdn.alemi.dev/proj/upub/fe/20240514.png)
+![screenshot of upub simple frontend](https://cdn.alemi.dev/proj/upub/fe/20240704.png)
 
 μpub aims to be a private, lightweight, modular and **secure** [ActivityPub](https://www.w3.org/TR/activitypub/) server
 
@@ -13,7 +13,7 @@ all interactions happen with ActivityPub's client-server methods (basically POST
 
 development is still active, so expect more stuff to come! since most fediverse software uses Mastodon's API, μpub plans to implement it as an optional feature, becoming eventually compatible with most existing frontends and mobile applications, but focus right now is on producing something specific to μpub needs
 
-a test instance is _usually_ available at [upub.alemi.dev](https://upub.alemi.dev)
+a test instance is available at [dev.upub.social](https://dev.upub.social)
 
 ## about the database schema
 im going to be very real i tried to do migrations but its getting super messy so until further notice assume db to be volatile. next change may be a migration (easy!) or a whole db rebuild (aaaaaaaaaa...), so if you're not comfortable with either manually exporting/importing or dropping and starting from scratch, **you really shouldn't put upub in prod yet**!
@@ -34,6 +34,33 @@ most instances will have "authorized fetch" which kind of makes the issue less b
 
 note that followers get expanded: addressing to example.net/actor/followers will address to anyone following actor that the server knows of, at that time
 
+## media caching
+μpub doesn't download remote media to both minimize local resources requirement and avoid storing media that remotes want gone. to prevent leaking local user ip addresses, all media links are cloaked and proxied.
+
+while this just works for small instances, larger servers should set up aggressive caching on `/proxy/...` path
+
+for example, on `nginx`:
+```nginx
+proxy_cache_path /tmp/upub/cache levels=1:2 keys_zone=upub_cache:100m max_size=50g inactive=168h use_temp_path=off;
+
+server {
+	location /proxy/ {
+		# use our configured cache
+		slice              1m;
+		proxy_set_header   Range $slice_range;
+		chunked_transfer_encoding on;
+		proxy_ignore_client_abort on;
+		proxy_buffering    on;
+		proxy_cache        upub_cache;
+		proxy_cache_key    $host$uri$is_args$args$slice_range;
+		proxy_cache_valid  200 206 301 304 168h;
+		proxy_cache_lock   on;
+		proxy_pass         http://127.0.0.1/;
+	}
+}
+
+```
+
 ## contributing
 
 all help is extremely welcome! development mostly happens on [moonlit.technology](https://moonlit.technology/alemi/upub.git), but there's a [github mirror](https://github.com/alemidev/upub) available too
@@ -63,18 +90,15 @@ don't hesitate to get in touch, i'd be thrilled to showcase the project to you!
  - [x] backend config
  - [x] frontend config
  - [x] optimize `addressing` database schema
- - [ ] mentions, notifications
+ - [x] mentions, notifications
- - [ ] hashtags
+ - [x] hashtags
+ - [x] remote media proxy
+ - [x] user fields
+ - [ ] better editing via web frontend
+ - [ ] upload media
  - [ ] public vs unlisted for discovery
  - [ ] mastodon-like search bar
  - [ ] polls
- - [ ] better editing via web frontend
- - [ ] remote media proxy
- - [ ] upload media
- - [ ] user fields
  - [ ] lists
  - [ ] full mastodon api
  - [ ] get rid of internal ids from code
-
-## what about the name?
-μpub (or simply `upub`) means "[micro](https://en.wikipedia.org/wiki/International_System_of_Units#Prefixes)-pub", but could also be read "upub", "you-pub" or "mu-pub"

apb/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "apb"
-version = "0.1.1"
+version = "0.2.2"
 edition = "2021"
 authors = [ "alemi <me@alemi.dev>" ]
 description = "Traits and types to handle ActivityPub objects"
 license = "MIT"
 keywords = ["activitypub", "activitystreams", "json"]
-repository = "https://git.alemi.dev/upub.git"
+repository = "https://moonlit.technology/alemi/upub"
 readme = "README.md"
 
 [lib]
@@ -18,9 +18,8 @@ chrono = { version = "0.4", features = ["serde"] }
 thiserror = "1"
 paste = "1.0"
 tracing = "0.1"
-async-trait = "0.1"
 serde_json = { version = "1", optional = true }
-sea-orm = { version = "0.12", optional = true }
+sea-orm = { version = "1.0", optional = true, default-features = false }
 reqwest = { version = "0.12", features = ["json"], optional = true }
 
 [features]
@@ -32,6 +31,9 @@ activitypub-fe = [] # https://ns.alemi.dev/as/fe/#
 ostatus = [] # https://ostatus.org#  , but it redirects and 403??? just need this for conversation
 toot = [] # http://joinmastodon.org/ns#  , mastodon is weird tho??
 litepub = [] # incomplete, https://litepub.social/
+did-core = [] # incomplete, may be cool to support all of this: https://www.w3.org/TR/did-core/
+# full jsonld utilities
+jsonld = []
 # builtin utils
 send = []
 orm = ["dep:sea-orm"]

apb/src/field.rs

@@ -0,0 +1,18 @@
+#[derive(Debug, thiserror::Error)]
+#[error("missing field '{0}'")]
+pub struct FieldErr(pub &'static str);
+
+pub type Field<T> = Result<T, FieldErr>;
+
+
+// TODO this trait is really ad-hoc and has awful naming...
+
+pub trait OptionalString {
+	fn str(self) -> Option<String>;
+}
+
+impl OptionalString for Field<&str> {
+	fn str(self) -> Option<String> {
+		self.ok().map(|x| x.to_string())
+	}
+}

apb/src/jsonld.rs

@@ -1,16 +1,7 @@
-// TODO
+use crate::Object;
-// move this file somewhere else
-// it's not a route
-// maybe under src/server/jsonld.rs ??
-
-use apb::Object;
-use axum::response::{IntoResponse, Response};
 
 pub trait LD {
 	fn ld_context(self) -> Self;
-	fn new_object() -> serde_json::Value {
-		serde_json::Value::Object(serde_json::Map::default())
-	}
 }
 
 impl LD for serde_json::Value {
@@ -21,7 +12,7 @@ impl LD for serde_json::Value {
 			ctx.insert("sensitive".to_string(), serde_json::Value::String("as:sensitive".into()));
 			ctx.insert("quoteUrl".to_string(), serde_json::Value::String("as:quoteUrl".into()));
 			match o_type {
-				Some(apb::ObjectType::Actor(_)) => {
+				Ok(crate::ObjectType::Actor(_)) => {
 					ctx.insert("counters".to_string(), serde_json::Value::String("https://ns.alemi.dev/as/counters/#".into()));
 					ctx.insert("followingCount".to_string(), serde_json::Value::String("counters:followingCount".into()));
 					ctx.insert("followersCount".to_string(), serde_json::Value::String("counters:followersCount".into()));
@@ -30,18 +21,24 @@ impl LD for serde_json::Value {
 					ctx.insert("followingMe".to_string(), serde_json::Value::String("fe:followingMe".into()));
 					ctx.insert("followedByMe".to_string(), serde_json::Value::String("fe:followedByMe".into()));
 				},
-				Some(_) => {
+				Ok(
+					crate::ObjectType::Note
+					| crate::ObjectType::Article
+					| crate::ObjectType::Event
+					| crate::ObjectType::Document(crate::DocumentType::Page) // TODO why Document lemmyyyyyy
+				) => {
 					ctx.insert("fe".to_string(), serde_json::Value::String("https://ns.alemi.dev/as/fe/#".into()));
 					ctx.insert("likedByMe".to_string(), serde_json::Value::String("fe:likedByMe".into()));
 					ctx.insert("ostatus".to_string(), serde_json::Value::String("http://ostatus.org#".into()));
 					ctx.insert("conversation".to_string(), serde_json::Value::String("ostatus:conversation".into()));
 				},
-				None => {},
+				_ => {},
 			}
 			obj.insert(
 				"@context".to_string(),
 				serde_json::Value::Array(vec![
 					serde_json::Value::String("https://www.w3.org/ns/activitystreams".into()),
+					serde_json::Value::String("https://w3id.org/security/v1".into()),
 					serde_json::Value::Object(ctx),
 				]),
 			);
@@ -51,15 +48,3 @@ impl LD for serde_json::Value {
 		self
 	}
 }
-
-// got this from https://github.com/kitsune-soc/kitsune/blob/b023a12b687dd9a274233a5a9950f2de5e192344/kitsune/src/http/responder.rs
-// i was trying to do it with middlewares but this is way cleaner
-pub struct JsonLD<T>(pub T);
-impl<T: serde::Serialize> IntoResponse for JsonLD<T> {
-	fn into_response(self) -> Response {
-		(
-			[("Content-Type", "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")],
-			axum::Json(self.0)
-		).into_response()
-	}
-}

apb/src/key.rs

@@ -1,7 +1,7 @@
 // TODO technically this is not part of ActivityStreams
 
 pub trait PublicKey : super::Base {
-	fn owner(&self) -> Option<&str> { None }
+	fn owner(&self) -> crate::Field<&str> { Err(crate::FieldErr("owner")) }
 	fn public_key_pem(&self) -> &str;
 }
 

apb/src/lib.rs

@@ -88,17 +88,28 @@
 
 
 mod macros;
-pub(crate) use macros::{strenum, getter, setter};
+pub(crate) use macros::strenum;
+
+#[cfg(feature = "unstructured")]
+pub(crate) use macros::{getter, setter};
 
 mod node;
 pub use node::Node;
 
-pub mod server;
 pub mod target;
 
 mod key;
 pub use key::{PublicKey, PublicKeyMut};
 
+pub mod field;
+pub use field::{Field, FieldErr};
+
+#[cfg(feature = "jsonld")]
+mod jsonld;
+
+#[cfg(feature = "jsonld")]
+pub use jsonld::LD;
+
 mod types;
 pub use types::{
 	base::{Base, BaseMut, BaseType},
@@ -125,3 +136,8 @@ pub use types::{
 		tombstone::{Tombstone, TombstoneMut},
 	},
 };
+
+#[cfg(feature = "unstructured")]
+pub fn new() -> serde_json::Value {
+	serde_json::Value::Object(serde_json::Map::default())
+}

apb/src/macros.rs

@@ -38,6 +38,12 @@ macro_rules! strenum {
 				$($deep($inner),)*
 			}
 
+			impl std::fmt::Display for $enum_name {
+				fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+					write!(f, "{}", self.as_ref())
+				}
+			}
+
 			impl AsRef<str> for $enum_name {
 				fn as_ref(&self) -> &str {
 					match self {
@@ -91,7 +97,7 @@ macro_rules! strenum {
 				}
 			
 				fn column_type() -> sea_orm::sea_query::ColumnType {
-					sea_orm::sea_query::ColumnType::String(Some(24))
+					sea_orm::sea_query::ColumnType::String(sea_orm::sea_query::table::StringLen::N(24))
 				}
 			}
 			
@@ -108,114 +114,109 @@ macro_rules! strenum {
 
 pub(crate) use strenum;
 
+#[cfg(feature = "unstructured")]
 macro_rules! getter {
 	($name:ident -> type $t:ty) => {
-		fn $name(&self) -> Option<$t> {
+		paste::paste! {
-			self.get("type")?.as_str()?.try_into().ok()
+			fn [< $name:snake >] (&self) -> $crate::Field<$t> {
-		}
+				self.get("type")
-	};
+					.and_then(|x| x.as_str())
-
+					.and_then(|x| x.try_into().ok())
-	($name:ident -> bool) => {
+					.ok_or($crate::FieldErr("type"))
-		fn $name(&self) -> Option<bool> {
-			self.get(stringify!($name))?.as_bool()
-		}
-	};
-
-	($name:ident -> &str) => {
-		fn $name(&self) -> Option<&str> {
-			self.get(stringify!($name))?.as_str()
-		}
-	};
-
-	($name:ident::$rename:ident -> bool) => {
-		fn $name(&self) -> Option<bool> {
-			self.get(stringify!($rename))?.as_bool()
-		}
-	};
-
-	($name:ident::$rename:ident -> &str) => {
-		fn $name(&self) -> Option<&str> {
-			self.get(stringify!($rename))?.as_str()
-		}
-	};
-
-	($name:ident -> f64) => {
-		fn $name(&self) -> Option<f64> {
-			self.get(stringify!($name))?.as_f64()
-		}
-	};
-
-	($name:ident::$rename:ident -> f64) => {
-		fn $name(&self) -> Option<f64> {
-			self.get(stringify!($rename))?.as_f64()
-		}
-	};
-
-	($name:ident -> u64) => {
-		fn $name(&self) -> Option<u64> {
-			self.get(stringify!($name))?.as_u64()
-		}
-	};
-
-	($name:ident::$rename:ident -> u64) => {
-		fn $name(&self) -> Option<u64> {
-			self.get(stringify!($rename))?.as_u64()
-		}
-	};
-
-	($name:ident -> chrono::DateTime<chrono::Utc>) => {
-		fn $name(&self) -> Option<chrono::DateTime<chrono::Utc>> {
-			Some(
-				chrono::DateTime::parse_from_rfc3339(
-						self
-							.get(stringify!($name))?
-							.as_str()?
-					)
-					.ok()?
-					.with_timezone(&chrono::Utc)
-			)
-		}
-	};
-
-	($name:ident::$rename:ident -> chrono::DateTime<chrono::Utc>) => {
-		fn $name(&self) -> Option<chrono::DateTime<chrono::Utc>> {
-			Some(
-				chrono::DateTime::parse_from_rfc3339(
-						self
-							.get(stringify!($rename))?
-							.as_str()?
-					)
-					.ok()?
-					.with_timezone(&chrono::Utc)
-			)
-		}
-	};
-
-	($name:ident -> node $t:ty) => {
-		fn $name(&self) -> $crate::Node<$t> {
-			match self.get(stringify!($name)) {
-				Some(x) => $crate::Node::from(x.clone()),
-				None => $crate::Node::Empty,
 			}
 		}
 	};
 
-	($name:ident::$rename:ident -> node $t:ty) => {
+	($name:ident -> bool) => {
-		fn $name(&self) -> $crate::Node<$t> {
+		paste::paste! {
-			match self.get(stringify!($rename)) {
+			fn [< $name:snake >](&self) -> $crate::Field<bool> {
-				Some(x) => $crate::Node::from(x.clone()),
+				self.get(stringify!($name))
-				None => $crate::Node::Empty,
+					.and_then(|x| x.as_bool())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> &str) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<&str> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_str())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> f64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<f64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_f64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> u64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<u64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_u64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> i64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<i64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_i64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> chrono::DateTime<chrono::Utc>) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<chrono::DateTime<chrono::Utc>> {
+				Ok(
+					chrono::DateTime::parse_from_rfc3339(
+							self
+								.get(stringify!($name))
+								.and_then(|x| x.as_str())
+								.ok_or($crate::FieldErr(stringify!($name)))?
+						)
+						.map_err(|e| {
+							tracing::warn!("invalid time string ({e}), ignoring");
+							$crate::FieldErr(stringify!($name))
+						})?
+						.with_timezone(&chrono::Utc)
+				)
+			}
+		}
+	};
+
+	($name:ident -> node $t:ty) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Node<$t> {
+				match self.get(stringify!($name)) {
+					Some(x) => $crate::Node::from(x.clone()),
+					None => $crate::Node::Empty,
+				}
 			}
 		}
 	};
 }
 
+#[cfg(feature = "unstructured")]
 pub(crate) use getter;
 
+#[cfg(feature = "unstructured")]
 macro_rules! setter {
 	($name:ident -> bool) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<bool>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<bool>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::Bool(x))
 				);
@@ -224,20 +225,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> bool) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<bool>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::Bool(x))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> &str) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<&str>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<&str>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::String(x.to_string()))
 				);
@@ -246,20 +236,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> &str) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<&str>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::String(x.to_string()))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> u64) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<u64>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<u64>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
 				);
@@ -268,11 +247,11 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> u64) => {
+	($name:ident -> i64) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<u64>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<i64>) -> Self {
 				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
+					&mut self, stringify!($name), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
 				);
 				self
 			}
@@ -281,7 +260,7 @@ macro_rules! setter {
 
 	($name:ident -> chrono::DateTime<chrono::Utc>) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::String(x.to_rfc3339()))
 				);
@@ -290,20 +269,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> chrono::DateTime<chrono::Utc>) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::String(x.to_rfc3339()))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> node $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: $crate::Node<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: $crate::Node<$t>) -> Self {
 				$crate::macros::set_maybe_node(
 					&mut self, stringify!($name), val
 				);
@@ -314,7 +282,7 @@ macro_rules! setter {
 
 	($name:ident::$rename:ident -> node $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: $crate::Node<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: $crate::Node<$t>) -> Self {
 				$crate::macros::set_maybe_node(
 					&mut self, stringify!($rename), val
 				);
@@ -325,7 +293,7 @@ macro_rules! setter {
 
 	($name:ident -> type $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<$t>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, "type", val.map(|x| serde_json::Value::String(x.as_ref().to_string()))
 				);
@@ -335,6 +303,7 @@ macro_rules! setter {
 	};
 }
 
+#[cfg(feature = "unstructured")]
 pub(crate) use setter;
 
 #[cfg(feature = "unstructured")]

apb/src/node.rs

@@ -1,9 +1,13 @@
 /// ActivityPub object node, representing either nothing, something, a link to something or
 /// multiple things
 pub enum Node<T : super::Base> {
+	/// this document node holds multiple objects
 	Array(std::collections::VecDeque<Node<T>>), // TODO would be cool to make it Box<[T]> so that Node is just a ptr
+	/// this document node holds one object
 	Object(Box<T>),
+	/// this document node holds a reference to an object
 	Link(Box<dyn crate::Link + Sync + Send>), // TODO feature flag to toggle these maybe?
+	/// this document node is not present
 	Empty,
 }
 
@@ -99,21 +103,21 @@ impl<T : super::Base> Node<T> {
 	}
 
 	/// returns id of object: url for link, id for object, None if empty or array
-	pub fn id(&self) -> Option<String> {
+	pub fn id(&self) -> crate::Field<&str> {
 		match self {
-			Node::Empty => None,
+			Node::Empty => Err(crate::FieldErr("id")),
-			Node::Link(uri) => Some(uri.href().to_string()),
+			Node::Link(uri) => uri.href(),
-			Node::Object(obj) => Some(obj.id()?.to_string()),
+			Node::Object(obj) => obj.id(),
-			Node::Array(arr) => Some(arr.front()?.id()?.to_string()),
+			Node::Array(arr) => arr.front().map(|x| x.id()).ok_or(crate::FieldErr("id"))?,
 		}
 	}
 
-	pub fn ids(&self) -> Vec<String> {
+	pub fn all_ids(&self) -> Vec<String> {
 		match self {
 			Node::Empty => vec![],
-			Node::Link(uri) => vec![uri.href().to_string()],
+			Node::Link(uri) => uri.href().map(|x| vec![x.to_string()]).unwrap_or_default(),
 			Node::Object(x) => x.id().map_or(vec![], |x| vec![x.to_string()]),
-			Node::Array(x) => x.iter().filter_map(Self::id).collect()
+			Node::Array(x) => x.iter().filter_map(|x| Some(x.id().ok()?.to_string())).collect()
 		}
 	}
 
@@ -169,6 +173,14 @@ impl Node<serde_json::Value> {
 		)
 	}
 
+	pub fn maybe_array(values: Vec<serde_json::Value>) -> Self {
+		if values.is_empty() {
+			Node::Empty
+		} else {
+			Node::array(values)
+		}
+	}
+
 	#[cfg(feature = "fetch")]
 	pub async fn fetch(&mut self) -> reqwest::Result<&mut Self> {
 		if let Node::Link(link) = self {
@@ -205,6 +217,7 @@ impl From<&str> for Node<serde_json::Value> {
 #[cfg(feature = "unstructured")]
 impl From<serde_json::Value> for Node<serde_json::Value> {
 	fn from(value: serde_json::Value) -> Self {
+		use crate::Link;
 		match value {
 			serde_json::Value::String(uri) => Node::Link(Box::new(uri)),
 			serde_json::Value::Array(arr) => Node::Array(
@@ -213,9 +226,9 @@ impl From<serde_json::Value> for Node<serde_json::Value> {
 						.map(Node::from)
 				)
 			),
-			serde_json::Value::Object(_) => match value.get("href") {
+			serde_json::Value::Object(_) => match value.link_type() {
-				None => Node::Object(Box::new(value)),
+				Ok(_) => Node::Link(Box::new(value)),
-				Some(_) => Node::Link(Box::new(value)),
+				Err(_) => Node::Object(Box::new(value)),
 			},
 			_ => Node::Empty,
 		}
@@ -227,7 +240,7 @@ impl From<Node<serde_json::Value>> for serde_json::Value {
 	fn from(value: Node<serde_json::Value>) -> Self {
 		match value {
 			Node::Empty => serde_json::Value::Null,
-			Node::Link(l) => serde_json::Value::String(l.href().to_string()), // TODO there could be more
+			Node::Link(l) => serde_json::Value::String(l.href().unwrap_or_default().to_string()), // TODO there could be more
 			Node::Object(o) => *o,
 			Node::Array(arr) =>
 				serde_json::Value::Array(arr.into_iter().map(|x| x.into()).collect()),

apb/src/server.rs

@@ -1,33 +0,0 @@
-#[async_trait::async_trait]
-pub trait Outbox {
-	type Object: crate::Object;
-	type Activity: crate::Activity;
-	type Error: std::error::Error;
-
-	async fn create_note(&self, uid: String, object: Self::Object) -> Result<String, Self::Error>;
-	async fn create(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn like(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn follow(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn announce(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn accept(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn reject(&self, _uid: String, _activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn undo(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn delete(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn update(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-}
-
-#[async_trait::async_trait]
-pub trait Inbox {
-	type Activity: crate::Activity;
-	type Error: std::error::Error;
-
-	async fn create(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn like(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn follow(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn announce(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn accept(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn reject(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn undo(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn delete(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn update(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-}

apb/src/target.rs

@@ -3,15 +3,99 @@ use crate::Object;
 pub const PUBLIC : &str = "https://www.w3.org/ns/activitystreams#Public";
 
 pub trait Addressed {
-	fn addressed(&self) -> Vec<String>;
+	fn addressed(&self) -> Vec<String>; // TODO rename this? remate others? idk
+	fn mentioning(&self) -> Vec<String>;
+	// fn secondary_targets(&self) -> Vec<String>;
+	// fn public_targets(&self) -> Vec<String>;
+	// fn private_targets(&self) -> Vec<String>;
 }
 
 impl<T: Object> Addressed for T {
 	fn addressed(&self) -> Vec<String> {
-		let mut to : Vec<String> = self.to().ids();
+		let mut to : Vec<String> = self.to().all_ids();
-		to.append(&mut self.bto().ids());
+		to.append(&mut self.bto().all_ids());
-		to.append(&mut self.cc().ids());
+		to.append(&mut self.cc().all_ids());
-		to.append(&mut self.bcc().ids());
+		to.append(&mut self.bcc().all_ids());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to : Vec<String> = self.to().all_ids();
+		to.append(&mut self.bto().all_ids());
+		to
+	}
+
+	// fn secondary_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.cc().ids();
+	// 	to.append(&mut self.bcc().ids());
+	// 	to
+	// }
+
+	// fn public_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.to().ids();
+	// 	to.append(&mut self.cc().ids());
+	// 	to
+	// }
+
+	// fn private_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.bto().ids();
+	// 	to.append(&mut self.bcc().ids());
+	// 	to
+	// }
+}
+
+#[cfg(test)]
+mod test {
+	use super::Addressed;
+
+	#[test]
+	#[cfg(feature = "unstructured")]
+	fn addressed_trait_finds_all_targets_on_json_objects() {
+		let obj = serde_json::json!({
+			"id": "http://localhost:8080/obj/1",
+			"type": "Note",
+			"content": "hello world!",
+			"published": "2024-06-04T17:09:20+00:00",
+			"to": ["http://localhost:8080/usr/root/followers"],
+			"bto": ["https://localhost:8080/usr/secret"],
+			"cc": [crate::target::PUBLIC],
+			"bcc": [],
+		});
+
+		let addressed = obj.addressed();
+
+		assert_eq!(
+			addressed,
+			vec![
+				"http://localhost:8080/usr/root/followers".to_string(), 
+				"https://localhost:8080/usr/secret".to_string(),
+				crate::target::PUBLIC.to_string(),
+			]
+		);
+	}
+
+	#[test]
+	#[cfg(feature = "unstructured")]
+	fn primary_targets_only_finds_to_and_bto() {
+		let obj = serde_json::json!({
+			"id": "http://localhost:8080/obj/1",
+			"type": "Note",
+			"content": "hello world!",
+			"published": "2024-06-04T17:09:20+00:00",
+			"to": ["http://localhost:8080/usr/root/followers"],
+			"bto": ["https://localhost:8080/usr/secret"],
+			"cc": [crate::target::PUBLIC],
+			"bcc": [],
+		});
+
+		let addressed = obj.mentioning();
+
+		assert_eq!(
+			addressed,
+			vec![
+				"http://localhost:8080/usr/root/followers".to_string(), 
+				"https://localhost:8080/usr/secret".to_string(),
+			]
+		);
+	}
 }

apb/src/types/base.rs

@@ -9,8 +9,8 @@ crate::strenum! {
 }
 
 pub trait Base : crate::macros::MaybeSend {
-	fn id(&self) -> Option<&str> { None }
+	fn id(&self) -> crate::Field<&str> { Err(crate::FieldErr("id")) }
-	fn base_type(&self) -> Option<BaseType> { None }
+	fn base_type(&self) -> crate::Field<BaseType> { Err(crate::FieldErr("type")) }
 }
 
 
@@ -21,30 +21,35 @@ pub trait BaseMut : crate::macros::MaybeSend {
 
 
 impl Base for String {
-	fn id(&self) -> Option<&str> {
+	fn id(&self) -> crate::Field<&str> {
-		Some(self)
+		Ok(self)
 	}
 
-	fn base_type(&self) -> Option<BaseType> {
+	fn base_type(&self) -> crate::Field<BaseType> {
-		Some(BaseType::Link(LinkType::Link))
+		Ok(BaseType::Link(LinkType::Link))
 	}
 }
 
 #[cfg(feature = "unstructured")]
 impl Base for serde_json::Value {
-	fn base_type(&self) -> Option<BaseType> {
+	fn base_type(&self) -> crate::Field<BaseType> {
 		if self.is_string() {
-			Some(BaseType::Link(LinkType::Link))
+			Ok(BaseType::Link(LinkType::Link))
 		} else {
-			self.get("type")?.as_str()?.try_into().ok()
+			self.get("type")
+				.and_then(|x| x.as_str())
+				.and_then(|x| x.try_into().ok())
+				.ok_or(crate::FieldErr("type"))
 		}
 	}
 
-	fn id(&self) -> Option<&str> {
+	fn id(&self) -> crate::Field<&str> {
 		if self.is_string() {
-			self.as_str()
+			Ok(self.as_str().ok_or(crate::FieldErr("id"))?)
 		} else {
-			self.get("id").map(|x| x.as_str())?
+			self.get("id")
+				.and_then(|x| x.as_str())
+				.ok_or(crate::FieldErr("id"))
 		}
 	}
 }

apb/src/types/link.rs

@@ -1,3 +1,5 @@
+use crate::{Field, FieldErr};
+
 #[cfg(feature = "activitypub-miscellaneous-terms")]
 crate::strenum! {
 	pub enum LinkType {
@@ -16,73 +18,80 @@ crate::strenum! {
 }
 
 pub trait Link : crate::Base {
-	fn href(&self) -> &str;
+	fn link_type(&self) -> Field<LinkType> { Err(FieldErr("type")) }
-	fn rel(&self) -> Option<&str> { None }
+	fn href(&self) -> Field<&str>;
-	fn link_media_type(&self) -> Option<&str> { None } // also in obj
+	fn rel(&self) -> Field<&str> { Err(FieldErr("rel")) }
-	fn link_name(&self) -> Option<&str> { None }       // also in obj
+	fn media_type(&self) -> Field<&str> { Err(FieldErr("mediaType")) } // also in obj
-	fn hreflang(&self) -> Option<&str> { None }
+	fn name(&self) -> Field<&str> { Err(FieldErr("name")) }       // also in obj
-	fn height(&self) -> Option<u64> { None }
+	fn hreflang(&self) -> Field<&str> { Err(FieldErr("hreflang")) }
-	fn width(&self) -> Option<u64> { None }
+	fn height(&self) -> Field<u64> { Err(FieldErr("height")) }
-	fn link_preview(&self) -> Option<&str> { None }    // also in obj
+	fn width(&self) -> Field<u64> { Err(FieldErr("width")) }
+	fn preview(&self) -> Field<&str> { Err(FieldErr("linkPreview")) }    // also in obj
 }
 
 pub trait LinkMut : crate::BaseMut {
-	fn set_href(self, href: &str) -> Self;
+	fn set_link_type(self, val: Option<LinkType>) -> Self;
+	fn set_href(self, href: Option<&str>) -> Self;
 	fn set_rel(self, val: Option<&str>) -> Self;
-	fn set_link_media_type(self, val: Option<&str>) -> Self; // also in obj
+	fn set_media_type(self, val: Option<&str>) -> Self; // also in obj
-	fn set_link_name(self, val: Option<&str>) -> Self;       // also in obj
+	fn set_name(self, val: Option<&str>) -> Self;       // also in obj
 	fn set_hreflang(self, val: Option<&str>) -> Self;
 	fn set_height(self, val: Option<u64>) -> Self;
 	fn set_width(self, val: Option<u64>) -> Self;
-	fn set_link_preview(self, val: Option<&str>) -> Self;    // also in obj
+	fn set_preview(self, val: Option<&str>) -> Self;    // also in obj
 }
 
 impl Link for String {
-	fn href(&self) -> &str {
+	fn href(&self) -> Field<&str> {
-		self
+		Ok(self)
 	}
 }
 
 #[cfg(feature = "unstructured")]
 impl Link for serde_json::Value {
 	// TODO this can fail, but it should never do!
-	fn href(&self) -> &str {
+	fn href(&self) -> Field<&str> {
 		if self.is_string() {
-			self.as_str().unwrap_or("")
+			self.as_str().ok_or(FieldErr("href"))
 		} else {
-			self.get("href").map(|x| x.as_str().unwrap_or("")).unwrap_or("")
+			self.get("href").and_then(|x| x.as_str()).ok_or(FieldErr("href"))
 		}
 	}
 
+	crate::getter! { link_type -> type LinkType }
 	crate::getter! { rel -> &str }
-	crate::getter! { link_media_type::mediaType -> &str }
+	crate::getter! { mediaType -> &str }
-	crate::getter! { link_name::name -> &str }
+	crate::getter! { name -> &str }
 	crate::getter! { hreflang -> &str }
 	crate::getter! { height -> u64 }
 	crate::getter! { width -> u64 }
-	crate::getter! { link_preview::preview -> &str }
+	crate::getter! { preview -> &str }
 }
 
 #[cfg(feature = "unstructured")]
 impl LinkMut for serde_json::Value {
-	fn set_href(mut self, href: &str) -> Self {
+	fn set_href(mut self, href: Option<&str>) -> Self {
 		match &mut self {
 			serde_json::Value::Object(map) => {
-				map.insert(
+				match href {
-					"href".to_string(),
+					Some(href) => map.insert(
-					serde_json::Value::String(href.to_string())
+						"href".to_string(),
-				);
+						serde_json::Value::String(href.to_string())
+					),
+					None => map.remove("href"),
+				};
 			},
-			x => *x = serde_json::Value::String(href.to_string()),
+			x => *x = serde_json::Value::String(href.unwrap_or_default().to_string()),
 		}
 		self
 	}
 
+	crate::setter! { link_type -> type LinkType }
 	crate::setter! { rel -> &str }
-	crate::setter! { link_media_type::mediaType -> &str }
+	crate::setter! { mediaType -> &str }
-	crate::setter! { link_name::name -> &str }
+	crate::setter! { name -> &str }
 	crate::setter! { hreflang -> &str }
 	crate::setter! { height -> u64 }
 	crate::setter! { width -> u64 }
-	crate::setter! { link_preview::preview -> &str }
+	crate::setter! { preview -> &str }
 }

apb/src/types/object/activity/accept.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Accept : super::Activity {
-	fn accept_type(&self) -> Option<AcceptType> { None }
+	fn accept_type(&self) -> crate::Field<AcceptType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait AcceptMut : super::ActivityMut {

apb/src/types/object/activity/ignore.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Ignore : super::Activity {
-	fn ignore_type(&self) -> Option<IgnoreType> { None }
+	fn ignore_type(&self) -> crate::Field<IgnoreType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait IgnoreMut : super::ActivityMut {

apb/src/types/object/activity/intransitive.rs

@@ -10,7 +10,7 @@ strenum! {
 }
 
 pub trait IntransitiveActivity : super::Activity {
-	fn intransitive_activity_type(&self) -> Option<IntransitiveActivityType> { None }
+	fn intransitive_activity_type(&self) -> crate::Field<IntransitiveActivityType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait IntransitiveActivityMut : super::ActivityMut {

apb/src/types/object/activity/mod.rs

@@ -4,7 +4,7 @@ pub mod intransitive;
 pub mod offer;
 pub mod reject;
 
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 use accept::AcceptType;
 use reject::RejectType;
 use offer::OfferType;
@@ -73,13 +73,29 @@ crate::strenum! {
 }
 
 pub trait Activity : Object {
-	fn activity_type(&self) -> Option<ActivityType> { None }
+	fn activity_type(&self) -> Field<ActivityType> { Err(FieldErr("type")) }
+	/// Describes one or more entities that either performed or are expected to perform the activity.
+	/// Any single activity can have multiple actors. The actor MAY be specified using an indirect Link. 
 	fn actor(&self) -> Node<Self::Actor> { Node::Empty }
+	/// Describes an object of any kind.
+	/// The Object type serves as the base type for most of the other kinds of objects defined in the Activity Vocabulary, including other Core types such as Activity, IntransitiveActivity, Collection and OrderedCollection. 
 	fn object(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes the indirect object, or target, of the activity.
+	/// The precise meaning of the target is largely dependent on the type of action being described but will often be the object of the English preposition "to".
+	/// For instance, in the activity "John added a movie to his wishlist", the target of the activity is John's wishlist. An activity can have more than one target. 
 	fn target(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes the result of the activity.
+	/// For instance, if a particular action results in the creation of a new resource, the result property can be used to describe that new resource. 
 	fn result(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes an indirect object of the activity from which the activity is directed.
+	/// The precise meaning of the origin is the object of the English preposition "from".
+	/// For instance, in the activity "John moved an item to List B from List A", the origin of the activity is "List A".
 	fn origin(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies one or more objects used (or to be used) in the completion of an Activity.
 	fn instrument(&self) -> Node<Self::Object> { Node::Empty }
+
+	#[cfg(feature = "activitypub-fe")]
+	fn seen(&self) -> Field<bool> { Err(FieldErr("seen")) }
 }
 
 pub trait ActivityMut : ObjectMut {
@@ -90,6 +106,9 @@ pub trait ActivityMut : ObjectMut {
 	fn set_result(self, val: Node<Self::Object>) -> Self;
 	fn set_origin(self, val: Node<Self::Object>) -> Self;
 	fn set_instrument(self, val: Node<Self::Object>) -> Self;
+
+	#[cfg(feature = "activitypub-fe")]
+	fn set_seen(self, val: Option<bool>) -> Self;
 }
 
 #[cfg(feature = "unstructured")]
@@ -101,6 +120,9 @@ impl Activity for serde_json::Value {
 	crate::getter! { result -> node <Self as Object>::Object }
 	crate::getter! { origin -> node <Self as Object>::Object }
 	crate::getter! { instrument -> node <Self as Object>::Object }
+
+	#[cfg(feature = "activitypub-fe")]
+	crate::getter! { seen -> bool }
 }
 
 #[cfg(feature = "unstructured")]
@@ -112,4 +134,7 @@ impl ActivityMut for serde_json::Value {
 	crate::setter! { result -> node <Self as Object>::Object }
 	crate::setter! { origin -> node <Self as Object>::Object }
 	crate::setter! { instrument -> node <Self as Object>::Object }
+
+	#[cfg(feature = "activitypub-fe")]
+	crate::setter! { seen -> bool }
 }

apb/src/types/object/activity/offer.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Offer : super::Activity {
-	fn offer_type(&self) -> Option<OfferType> { None }
+	fn offer_type(&self) -> crate::Field<OfferType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait OfferMut : super::ActivityMut {

apb/src/types/object/activity/reject.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Reject : super::Activity {
-	fn reject_type(&self) -> Option<RejectType> { None }
+	fn reject_type(&self) -> crate::Field<RejectType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait RejectMut : super::ActivityMut {

apb/src/types/object/actor.rs

@@ -1,4 +1,4 @@
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 
 crate::strenum! {
 	pub enum ActorType {
@@ -14,51 +14,67 @@ pub trait Actor : Object {
 	type PublicKey : crate::PublicKey;
 	type Endpoints : Endpoints;
 
-	fn actor_type(&self) -> Option<ActorType> { None }
+	fn actor_type(&self) -> Field<ActorType> { Err(FieldErr("type")) }
-	fn preferred_username(&self) -> Option<&str> { None }
+	/// A short username which may be used to refer to the actor, with no uniqueness guarantees.
+	fn preferred_username(&self) -> Field<&str> { Err(FieldErr("preferredUsername")) }
+	/// A reference to an [ActivityStreams] OrderedCollection comprised of all the messages received by the actor; see 5.2 Inbox. 
 	fn inbox(&self) -> Node<Self::Collection>;
+	/// An [ActivityStreams] OrderedCollection comprised of all the messages produced by the actor; see 5.1 Outbox. 
 	fn outbox(&self) -> Node<Self::Collection>;
+	/// A link to an [ActivityStreams] collection of the actors that this actor is following; see 5.4 Following Collection
 	fn following(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A link to an [ActivityStreams] collection of the actors that follow this actor; see 5.3 Followers Collection. 
 	fn followers(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A link to an [ActivityStreams] collection of objects this actor has liked; see 5.5 Liked Collection.
 	fn liked(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A list of supplementary Collections which may be of interest.
 	fn streams(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A json object which maps additional (typically server/domain-wide) endpoints which may be useful either for this actor or someone referencing this actor.
+	/// This mapping may be nested inside the actor document as the value or may be a link to a JSON-LD document with these properties. 
 	fn endpoints(&self) -> Node<Self::Endpoints> { Node::Empty }
-	fn public_key(&self) -> Node<Self::PublicKey> { Node::Empty }
+	fn public_key(&self) -> Node<Self::PublicKey> { Node::Empty } // TODO hmmm where is this from??
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn moved_to(&self) -> Node<Self::Actor> { Node::Empty }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	fn manually_approves_followers(&self) -> Option<bool> { None }
+	fn manually_approves_followers(&self) -> Field<bool> { Err(FieldErr("manuallyApprovesFollowers")) }
+
+	#[cfg(feature = "did-core")]
+	fn also_known_as(&self) -> Node<Self::Actor> { Node::Empty }
 
 	#[cfg(feature = "activitypub-fe")]
-	fn following_me(&self) -> Option<bool> { None }
+	fn following_me(&self) -> Field<bool> { Err(FieldErr("followingMe")) }
 	#[cfg(feature = "activitypub-fe")]
-	fn followed_by_me(&self) -> Option<bool> { None }
+	fn followed_by_me(&self) -> Field<bool> { Err(FieldErr("followedByMe")) }
+	#[cfg(feature = "activitypub-fe")]
+	fn notifications(&self) -> Node<Self::Collection> { Node::Empty }
 
 	#[cfg(feature = "activitypub-counters")]
-	fn followers_count(&self) -> Option<u64> { None }
+	fn followers_count(&self) -> Field<u64> { Err(FieldErr("followersCount")) }
 	#[cfg(feature = "activitypub-counters")]
-	fn following_count(&self) -> Option<u64> { None }
+	fn following_count(&self) -> Field<u64> { Err(FieldErr("followingCount")) }
 	#[cfg(feature = "activitypub-counters")]
-	fn statuses_count(&self) -> Option<u64> { None }
+	fn statuses_count(&self) -> Field<u64> { Err(FieldErr("statusesCount")) }
 
 	#[cfg(feature = "toot")]
-	fn discoverable(&self) -> Option<bool> { None }
+	fn discoverable(&self) -> Field<bool> { Err(FieldErr("discoverable")) }
+	#[cfg(feature = "toot")]
+	fn featured(&self) -> Node<Self::Collection> { Node::Empty }
 }
 
 pub trait Endpoints : Object {
 	/// Endpoint URI so this actor's clients may access remote ActivityStreams objects which require authentication to access. To use this endpoint, the client posts an x-www-form-urlencoded id parameter with the value being the id of the requested ActivityStreams object. 
-	fn proxy_url(&self) -> Option<&str> { None }
+	fn proxy_url(&self) -> Field<&str> { Err(FieldErr("proxyUrl")) }
 	/// If OAuth 2.0 bearer tokens [RFC6749] [RFC6750] are being used for authenticating client to server interactions, this endpoint specifies a URI at which a browser-authenticated user may obtain a new authorization grant. 
-	fn oauth_authorization_endpoint(&self) -> Option<&str> { None }
+	fn oauth_authorization_endpoint(&self) -> Field<&str> { Err(FieldErr("oauthAuthorizationEndpoint")) }
 	/// If OAuth 2.0 bearer tokens [RFC6749] [RFC6750] are being used for authenticating client to server interactions, this endpoint specifies a URI at which a client may acquire an access token. 
-	fn oauth_token_endpoint(&self) -> Option<&str> { None }
+	fn oauth_token_endpoint(&self) -> Field<&str> { Err(FieldErr("oauthTokenEndpoint")) }
 	/// If Linked Data Signatures and HTTP Signatures are being used for authentication and authorization, this endpoint specifies a URI at which browser-authenticated users may authorize a client's public key for client to server interactions. 
-	fn provide_client_key(&self) -> Option<&str> { None }
+	fn provide_client_key(&self) -> Field<&str> { Err(FieldErr("provideClientKey")) }
 	/// If Linked Data Signatures and HTTP Signatures are being used for authentication and authorization, this endpoint specifies a URI at which a client key may be signed by the actor's key for a time window to act on behalf of the actor in interacting with foreign servers. 
-	fn sign_client_key(&self) -> Option<&str> { None }
+	fn sign_client_key(&self) -> Field<&str> { Err(FieldErr("signClientKey")) }
 	/// An optional endpoint used for wide delivery of publicly addressed activities and activities sent to followers. sharedInbox endpoints SHOULD also be publicly readable OrderedCollection objects containing objects addressed to the Public special collection. Reading from the sharedInbox endpoint MUST NOT present objects which are not addressed to the Public endpoint.
-	fn shared_inbox(&self) -> Option<&str> { None }
+	fn shared_inbox(&self) -> Field<&str> { Err(FieldErr("sharedInbox")) }
 }
 
 pub trait ActorMut : ObjectMut {
@@ -81,10 +97,15 @@ pub trait ActorMut : ObjectMut {
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn set_manually_approves_followers(self, val: Option<bool>) -> Self;
 
+	#[cfg(feature = "did-core")]
+	fn set_also_known_as(self, val: Node<Self::Actor>) -> Self;
+
 	#[cfg(feature = "activitypub-fe")]
 	fn set_following_me(self, val: Option<bool>) -> Self;
 	#[cfg(feature = "activitypub-fe")]
 	fn set_followed_by_me(self, val: Option<bool>) -> Self;
+	#[cfg(feature = "activitypub-fe")]
+	fn set_notifications(self, val: Node<Self::Collection>) -> Self;
 
 	#[cfg(feature = "activitypub-counters")]
 	fn set_followers_count(self, val: Option<u64>) -> Self;
@@ -95,6 +116,8 @@ pub trait ActorMut : ObjectMut {
 
 	#[cfg(feature = "toot")]
 	fn set_discoverable(self, val: Option<bool>) -> Self;
+	#[cfg(feature = "toot")]
+	fn set_featured(self, val: Node<Self::Collection>) -> Self;
 }
 
 pub trait EndpointsMut : ObjectMut {
@@ -117,46 +140,53 @@ impl Actor for serde_json::Value {
 	type PublicKey = serde_json::Value;
 	type Endpoints = serde_json::Value;
 
-	crate::getter! { actor_type -> type ActorType }
+	crate::getter! { actorType -> type ActorType }
-	crate::getter! { preferred_username::preferredUsername -> &str }
+	crate::getter! { preferredUsername -> &str }
 	crate::getter! { inbox -> node Self::Collection }
 	crate::getter! { outbox -> node Self::Collection }
 	crate::getter! { following -> node Self::Collection }
 	crate::getter! { followers -> node Self::Collection }
 	crate::getter! { liked -> node Self::Collection }
 	crate::getter! { streams -> node Self::Collection }
-	crate::getter! { public_key::publicKey -> node Self::PublicKey }
+	crate::getter! { publicKey -> node Self::PublicKey }
 	crate::getter! { endpoints -> node Self::Endpoints }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { moved_to::movedTo -> node Self::Actor }
+	crate::getter! { movedTo -> node Self::Actor }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { manually_approves_followers::manuallyApprovedFollowers -> bool }
+	crate::getter! { manuallyApprovesFollowers -> bool }
+
+	#[cfg(feature = "did-core")]
+	crate::getter! { alsoKnownAs -> node Self::Actor }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { following_me::followingMe -> bool }
+	crate::getter! { followingMe -> bool }
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { followed_by_me::followedByMe -> bool }
+	crate::getter! { followedByMe -> bool }
+	#[cfg(feature = "activitypub-fe")]
+	crate::getter! { notifications -> node Self::Collection }
 
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { following_count::followingCount -> u64 }
+	crate::getter! { followingCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { followers_count::followersCount -> u64 }
+	crate::getter! { followersCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { statuses_count::statusesCount -> u64 }
+	crate::getter! { statusesCount -> u64 }
 
 	#[cfg(feature = "toot")]
 	crate::getter! { discoverable -> bool }
+	#[cfg(feature = "toot")]
+	crate::getter! { featured -> node Self::Collection }
 }
 
 #[cfg(feature = "unstructured")]
 impl Endpoints for serde_json::Value {
-	crate::getter! { proxy_url::proxyUrl -> &str }
+	crate::getter! { proxyUrl -> &str }
-	crate::getter! { oauth_authorization_endpoint::oauthAuthorizationEndpoint -> &str }
+	crate::getter! { oauthAuthorizationEndpoint -> &str }
-	crate::getter! { oauth_token_endpoint::oauthTokenEndpoint -> &str }
+	crate::getter! { oauthTokenEndpoint -> &str }
-	crate::getter! { provide_client_key::provideClientKey -> &str }
+	crate::getter! { provideClientKey -> &str }
-	crate::getter! { sign_client_key::signClientKey -> &str }
+	crate::getter! { signClientKey -> &str }
-	crate::getter! { shared_inbox::sharedInbox -> &str }
+	crate::getter! { sharedInbox -> &str }
 }
 
 #[cfg(feature = "unstructured")]
@@ -165,43 +195,51 @@ impl ActorMut for serde_json::Value {
 	type Endpoints = serde_json::Value;
 
 	crate::setter! { actor_type -> type ActorType }
-	crate::setter! { preferred_username::preferredUsername -> &str }
+	crate::setter! { preferredUsername -> &str }
 	crate::setter! { inbox -> node Self::Collection }
 	crate::setter! { outbox -> node Self::Collection }
 	crate::setter! { following -> node Self::Collection }
 	crate::setter! { followers -> node Self::Collection }
 	crate::setter! { liked -> node Self::Collection }
 	crate::setter! { streams -> node Self::Collection }
-	crate::setter! { public_key::publicKey -> node Self::PublicKey }
+	crate::setter! { publicKey -> node Self::PublicKey }
 	crate::setter! { endpoints -> node Self::Endpoints }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { moved_to::movedTo -> node Self::Actor }
+	crate::setter! { movedTo -> node Self::Actor }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { manually_approves_followers::manuallyApprovedFollowers -> bool }
+	crate::setter! { manuallyApprovesFollowers -> bool }
+
+
+	#[cfg(feature = "did-core")]
+	crate::setter! { alsoKnownAs -> node Self::Actor }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { following_me::followingMe -> bool }
+	crate::setter! { followingMe -> bool }
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { followed_by_me::followedByMe -> bool }
+	crate::setter! { followedByMe -> bool }
+	#[cfg(feature = "activitypub-fe")]
+	crate::setter! { notifications -> node Self::Collection }
 
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { following_count::followingCount -> u64 }
+	crate::setter! { followingCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { followers_count::followersCount -> u64 }
+	crate::setter! { followersCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { statuses_count::statusesCount -> u64 }
+	crate::setter! { statusesCount -> u64 }
 
 	#[cfg(feature = "toot")]
 	crate::setter! { discoverable -> bool }
+	#[cfg(feature = "toot")]
+	crate::setter! { featured -> node Self::Collection }
 }
 
 #[cfg(feature = "unstructured")]
 impl EndpointsMut for serde_json::Value {
-	crate::setter! { proxy_url::proxyUrl -> &str }
+	crate::setter! { proxyUrl -> &str }
-	crate::setter! { oauth_authorization_endpoint::oauthAuthorizationEndpoint -> &str }
+	crate::setter! { oauthAuthorizationEndpoint -> &str }
-	crate::setter! { oauth_token_endpoint::oauthTokenEndpoint -> &str }
+	crate::setter! { oauthTokenEndpoint -> &str }
-	crate::setter! { provide_client_key::provideClientKey -> &str }
+	crate::setter! { provideClientKey -> &str }
-	crate::setter! { sign_client_key::signClientKey -> &str }
+	crate::setter! { signClientKey -> &str }
-	crate::setter! { shared_inbox::sharedInbox -> &str }
+	crate::setter! { sharedInbox -> &str }
 }

apb/src/types/object/collection/mod.rs

@@ -1,7 +1,7 @@
 pub mod page;
 pub use page::CollectionPage;
 
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 
 crate::strenum! {
 	pub enum CollectionType {
@@ -15,13 +15,20 @@ crate::strenum! {
 pub trait Collection : Object {
 	type CollectionPage : CollectionPage;
 
-	fn collection_type(&self) -> Option<CollectionType> { None }
+	fn collection_type(&self) -> Field<CollectionType> { Err(FieldErr("type")) }
 
-	fn total_items(&self) -> Option<u64> { None }
+	/// A non-negative integer specifying the total number of objects contained by the logical view of the collection.
+	/// This number might not reflect the actual number of items serialized within the Collection object instance. 
+	fn total_items(&self) -> Field<u64> { Err(FieldErr("totalItems")) }
+	/// In a paged Collection, indicates the page that contains the most recently updated member items. 
 	fn current(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// In a paged Collection, indicates the furthest preceeding page of items in the collection. 
 	fn first(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// In a paged Collection, indicates the furthest proceeding page of the collection.
 	fn last(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// Identifies the items contained in a collection. The items might be ordered or unordered.
 	fn items(&self) -> Node<Self::Object> { Node::Empty }
+	/// ??????????????? same as items but ordered?? spec just uses it without saying
 	fn ordered_items(&self) -> Node<Self::Object> { Node::Empty }
 }
 
@@ -42,12 +49,12 @@ impl Collection for serde_json::Value {
 	type CollectionPage = serde_json::Value;
 
 	crate::getter! { collection_type -> type CollectionType }
-	crate::getter! { total_items::totalItems -> u64 }
+	crate::getter! { totalItems -> u64 }
 	crate::getter! { current -> node Self::CollectionPage }
 	crate::getter! { first -> node Self::CollectionPage }
 	crate::getter! { last -> node Self::CollectionPage }
 	crate::getter! { items -> node <Self as Object>::Object }
-	crate::getter! { ordered_items::orderedItems -> node <Self as Object>::Object }
+	crate::getter! { orderedItems -> node <Self as Object>::Object }
 }
 
 #[cfg(feature = "unstructured")]
@@ -55,10 +62,10 @@ impl CollectionMut for serde_json::Value {
 	type CollectionPage = serde_json::Value;
 
 	crate::setter! { collection_type -> type CollectionType }
-	crate::setter! { total_items::totalItems -> u64 }
+	crate::setter! { totalItems -> u64 }
 	crate::setter! { current -> node Self::CollectionPage }
 	crate::setter! { first -> node Self::CollectionPage }
 	crate::setter! { last -> node Self::CollectionPage }
 	crate::setter! { items -> node <Self as Object>::Object }
-	crate::setter! { ordered_items::orderedItems -> node <Self as Object>::Object }
+	crate::setter! { orderedItems -> node <Self as Object>::Object }
 }

apb/src/types/object/collection/page.rs

@@ -14,14 +14,14 @@ pub trait CollectionPageMut : super::CollectionMut {
 
 #[cfg(feature = "unstructured")]
 impl CollectionPage for serde_json::Value {
-	crate::getter! { part_of::partOf -> node Self::Collection }
+	crate::getter! { partOf -> node Self::Collection }
 	crate::getter! { next -> node Self::CollectionPage }
 	crate::getter! { prev -> node Self::CollectionPage }
 }
 
 #[cfg(feature = "unstructured")]
 impl CollectionPageMut for serde_json::Value {
-	crate::setter! { part_of::partOf -> node Self::Collection }
+	crate::setter! { partOf -> node Self::Collection }
 	crate::setter! { next -> node Self::CollectionPage }
 	crate::setter! { prev -> node Self::CollectionPage }
 }

apb/src/types/object/document.rs

@@ -9,7 +9,7 @@ crate::strenum! {
 }
 
 pub trait Document : super::Object {
-	fn document_type(&self) -> Option<DocumentType> { None }
+	fn document_type(&self) -> crate::Field<DocumentType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait DocumentMut : super::ObjectMut {
@@ -19,10 +19,10 @@ pub trait DocumentMut : super::ObjectMut {
 
 #[cfg(feature = "unstructured")]
 impl Document for serde_json::Value {
-	crate::getter! { document_type -> type DocumentType }
+	crate::getter! { documentType -> type DocumentType }
 }
 
 #[cfg(feature = "unstructured")]
 impl DocumentMut for serde_json::Value {
-	crate::setter! { document_type -> type DocumentType }
+	crate::setter! { documentType -> type DocumentType }
 }

apb/src/types/object/mod.rs

@@ -7,7 +7,7 @@ pub mod place;
 pub mod profile;
 pub mod relationship;
 
-use crate::{Base, BaseMut, Node};
+use crate::{Base, BaseMut, Field, FieldErr, Node};
 
 use actor::ActorType;
 use document::DocumentType;
@@ -40,51 +40,96 @@ pub trait Object : Base {
 	type Document : crate::Document;
 	type Activity : crate::Activity;
 
-	fn object_type(&self) -> Option<ObjectType> { None }
+	fn object_type(&self) -> Field<ObjectType> { Err(FieldErr("type")) }
+	/// Identifies a resource attached or related to an object that potentially requires special handling
+	/// The intent is to provide a model that is at least semantically similar to attachments in email. 
 	fn attachment(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies one or more entities to which this object is attributed.
+	/// The attributed entities might not be Actors. For instance, an object might be attributed to the completion of another activity. 
 	fn attributed_to(&self) -> Node<Self::Actor> { Node::Empty }
-	fn audience(&self) -> Node<Self::Actor> { Node::Empty }
+	/// Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant
-	fn content(&self) -> Option<&str> { None } // TODO handle language maps
+	fn audience(&self) -> Node<Self::Object> { Node::Empty }
+	/// The content or textual representation of the Object encoded as a JSON string. By default, the value of content is HTML
+	/// The mediaType property can be used in the object to indicate a different content type
+	/// The content MAY be expressed using multiple language-tagged values
+	fn content(&self) -> Field<&str> { Err(FieldErr("content")) } // TODO handle language maps
+	/// Identifies the context within which the object exists or an activity was performed
+	/// The notion of "context" used is intentionally vague
+	/// The intended function is to serve as a means of grouping objects and activities that share a common originating context or purpose
+	/// An example could be all activities relating to a common project or event
 	fn context(&self) -> Node<Self::Object> { Node::Empty } 
-	fn name(&self) -> Option<&str> { None }       // also in link // TODO handle language maps
+	/// A simple, human-readable, plain-text name for the object. HTML markup MUST NOT be included. The name MAY be expressed using multiple language-tagged values
-	fn end_time(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	fn name(&self) -> Field<&str> { Err(FieldErr("name")) }       // also in link // TODO handle language maps
+	/// The date and time describing the actual or expected ending time of the object
+	/// When used with an Activity object, for instance, the endTime property specifies the moment the activity concluded or is expected to conclude. 
+	fn end_time(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("endTime")) }
+	/// Identifies the entity (e.g. an application) that generated the object
 	fn generator(&self) -> Node<Self::Actor> { Node::Empty }
+	/// Indicates an entity that describes an icon for this object
+	/// The image should have an aspect ratio of one (horizontal) to one (vertical) and should be suitable for presentation at a small size
 	fn icon(&self) -> Node<Self::Document> { Node::Empty }
+	/// Indicates an entity that describes an image for this object
+	/// Unlike the icon property, there are no aspect ratio or display size limitations assumed
 	fn image(&self) -> Node<Self::Document> { Node::Empty }
+	/// Indicates one or more entities for which this object is considered a response
 	fn in_reply_to(&self) -> Node<Self::Object> { Node::Empty }
+	/// Indicates one or more physical or logical locations associated with the object
 	fn location(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies an entity that provides a preview of this object
 	fn preview(&self) -> Node<Self::Object> { Node::Empty }    // also in link
-	fn published(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	/// The date and time at which the object was published
-	fn updated(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	fn published(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("published")) }
+	/// The date and time at which the object was updated
+	fn updated(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("updated")) }
+	/// Identifies a Collection containing objects considered to be responses to this object
 	fn replies(&self) -> Node<Self::Collection> { Node::Empty }
 	fn likes(&self) -> Node<Self::Collection> { Node::Empty }
 	fn shares(&self) -> Node<Self::Collection> { Node::Empty }
-	fn start_time(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	/// The date and time describing the actual or expected starting time of the object.
-	fn summary(&self) -> Option<&str> { None }
+	/// When used with an Activity object, for instance, the startTime property specifies the moment the activity began or is scheduled to begin. 
-	fn tag(&self) -> Node<Self::Object> { Node::Empty }
+	fn start_time(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("startTime")) }
+	/// A natural language summarization of the object encoded as HTML. Multiple language tagged summaries MAY be provided
+	fn summary(&self) -> Field<&str> { Err(FieldErr("summary")) }
+	/// One or more "tags" that have been associated with an objects. A tag can be any kind of Object
+	/// The key difference between attachment and tag is that the former implies association by inclusion, while the latter implies associated by reference
+	// TODO technically this is an object? but spec says that it works my reference, idk
+	fn tag(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies one or more links to representations of the object
 	fn url(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an entity considered to be part of the public primary audience of an Object
 	fn to(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an Object that is part of the private primary audience of this Object
 	fn bto(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an Object that is part of the public secondary audience of this Object
 	fn cc(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies one or more Objects that are part of the private secondary audience of this Object
 	fn bcc(&self) -> Node<Self::Link> { Node::Empty }
-	fn media_type(&self) -> Option<&str> { None } // also in link
+	/// When used on a Link, identifies the MIME media type of the referenced resource.
-	fn duration(&self) -> Option<&str> { None } // TODO how to parse xsd:duration ?
+	/// When used on an Object, identifies the MIME media type of the value of the content property.
+	/// If not specified, the content property is assumed to contain text/html content. 
+	fn media_type(&self) -> Field<&str> { Err(FieldErr("mediaType")) } // also in link
+	/// When the object describes a time-bound resource, such as an audio or video, a meeting, etc, the duration property indicates the object's approximate duration.
+	/// The value MUST be expressed as an xsd:duration as defined by [ xmlschema11-2], section 3.3.6 (e.g. a period of 5 seconds is represented as "PT5S"). 
+	fn duration(&self) -> Field<&str> { Err(FieldErr("duration")) } // TODO how to parse xsd:duration ?
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	fn sensitive(&self) -> Option<bool> { None }
+	fn sensitive(&self) -> Field<bool> { Err(FieldErr("sensitive")) }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn quote_url(&self) -> Node<Self::Object> { Node::Empty }
 
 	#[cfg(feature = "activitypub-fe")]
-	fn liked_by_me(&self) -> Option<bool> { None }
+	fn liked_by_me(&self) -> Field<bool> { Err(FieldErr("likedByMe")) }
 
 	#[cfg(feature = "ostatus")]
 	fn conversation(&self) -> Node<Self::Object> { Node::Empty }
 
-	fn as_activity(&self) -> Option<&Self::Activity> { None }
+	fn as_activity(&self) -> Result<&Self::Activity, FieldErr> { Err(FieldErr("type")) }
-	fn as_actor(&self) -> Option<&Self::Actor> { None }
+	fn as_actor(&self) -> Result<&Self::Actor, FieldErr> { Err(FieldErr("type")) }
-	fn as_collection(&self) -> Option<&Self::Collection> { None }
+	fn as_collection(&self) -> Result<&Self::Collection, FieldErr> { Err(FieldErr("type")) }
-	fn as_document(&self) -> Option<&Self::Document> { None }
+	fn as_document(&self) -> Result<&Self::Document, FieldErr> { Err(FieldErr("type")) }
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	fn value(&self) -> Field<&str> { Err(FieldErr("value")) }
 }
 
 pub trait ObjectMut : BaseMut {
@@ -134,6 +179,9 @@ pub trait ObjectMut : BaseMut {
 
 	#[cfg(feature = "ostatus")]
 	fn set_conversation(self, val: Node<Self::Object>) -> Self;
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	fn set_value(self, val: Option<&str>) -> Self;
 }
 
 #[cfg(feature = "unstructured")]
@@ -145,18 +193,18 @@ impl Object for serde_json::Value {
 	type Collection = serde_json::Value;
 	type Activity = serde_json::Value;
 
-	crate::getter! { object_type -> type ObjectType }
+	crate::getter! { objectType -> type ObjectType }
 	crate::getter! { attachment -> node <Self as Object>::Object }
-	crate::getter! { attributed_to::attributedTo -> node Self::Actor }
+	crate::getter! { attributedTo -> node Self::Actor }
 	crate::getter! { audience -> node Self::Actor }
 	crate::getter! { content -> &str }
 	crate::getter! { context -> node <Self as Object>::Object }
 	crate::getter! { name -> &str }
-	crate::getter! { end_time::endTime -> chrono::DateTime<chrono::Utc> }
+	crate::getter! { endTime -> chrono::DateTime<chrono::Utc> }
 	crate::getter! { generator -> node Self::Actor }
 	crate::getter! { icon -> node Self::Document }
 	crate::getter! { image -> node Self::Document }
-	crate::getter! { in_reply_to::inReplyTo -> node <Self as Object>::Object }
+	crate::getter! { inReplyTo -> node <Self as Object>::Object }
 	crate::getter! { location -> node <Self as Object>::Object }
 	crate::getter! { preview -> node <Self as Object>::Object }
 	crate::getter! { published -> chrono::DateTime<chrono::Utc> }
@@ -164,53 +212,56 @@ impl Object for serde_json::Value {
 	crate::getter! { replies -> node Self::Collection }
 	crate::getter! { likes -> node Self::Collection }
 	crate::getter! { shares -> node Self::Collection }
-	crate::getter! { start_time::startTime -> chrono::DateTime<chrono::Utc> }
+	crate::getter! { startTime -> chrono::DateTime<chrono::Utc> }
 	crate::getter! { summary -> &str }
 	crate::getter! { tag -> node <Self as Object>::Object }
 	crate::getter! { to -> node Self::Link }
 	crate::getter! { bto -> node Self::Link }
 	crate::getter! { cc -> node Self::Link }
 	crate::getter! { bcc -> node Self::Link }
-	crate::getter! { media_type::mediaType -> &str }
+	crate::getter! { mediaType -> &str }
 	crate::getter! { duration -> &str }
 	crate::getter! { url -> node Self::Link }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	crate::getter! { sensitive -> bool }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { quote_url::quoteUrl -> node <Self as Object>::Object }
+	crate::getter! { quoteUrl -> node <Self as Object>::Object }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { liked_by_me::likedByMe -> bool }
+	crate::getter! { likedByMe -> bool }
 
 	#[cfg(feature = "ostatus")]
 	crate::getter! { conversation -> node <Self as Object>::Object }
 
-	fn as_activity(&self) -> Option<&Self::Activity> {
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
-		match self.object_type() {
+	crate::getter! { value -> &str }
-			Some(ObjectType::Activity(_)) => Some(self),
+
-			_ => None,
+	fn as_activity(&self) -> Result<&Self::Activity, FieldErr> {
+		match self.object_type()? {
+			ObjectType::Activity(_) => Ok(self),
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_actor(&self) -> Option<&Self::Actor> {
+	fn as_actor(&self) -> Result<&Self::Actor, FieldErr> {
-		match self.object_type() {
+		match self.object_type()? {
-			Some(ObjectType::Actor(_)) => Some(self),
+			ObjectType::Actor(_) => Ok(self),
-			_ => None,
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_collection(&self) -> Option<&Self::Collection> {
+	fn as_collection(&self) -> Result<&Self::Collection, FieldErr> {
-		match self.object_type() {
+		match self.object_type()? {
-			Some(ObjectType::Collection(_)) => Some(self),
+			ObjectType::Collection(_) => Ok(self),
-			_ => None,
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_document(&self) -> Option<&Self::Document> {
+	fn as_document(&self) -> Result<&Self::Document, FieldErr> {
-		match self.object_type() {
+		match self.object_type()? {
-			Some(ObjectType::Document(_)) => Some(self),
+			ObjectType::Document(_) => Ok(self),
-			_ => None,
+			_ => Err(FieldErr("type")),
 		}
 	}
 }
@@ -225,16 +276,16 @@ impl ObjectMut for serde_json::Value {
 
 	crate::setter! { object_type -> type ObjectType }
 	crate::setter! { attachment -> node <Self as Object>::Object }
-	crate::setter! { attributed_to::attributedTo -> node Self::Actor }
+	crate::setter! { attributedTo -> node Self::Actor }
 	crate::setter! { audience -> node Self::Actor }
 	crate::setter! { content -> &str }
 	crate::setter! { context -> node <Self as Object>::Object }
 	crate::setter! { name -> &str }
-	crate::setter! { end_time::endTime -> chrono::DateTime<chrono::Utc> }
+	crate::setter! { endTime -> chrono::DateTime<chrono::Utc> }
 	crate::setter! { generator -> node Self::Actor }
 	crate::setter! { icon -> node Self::Document }
 	crate::setter! { image -> node Self::Document }
-	crate::setter! { in_reply_to::inReplyTo -> node <Self as Object>::Object }
+	crate::setter! { inReplyTo -> node <Self as Object>::Object }
 	crate::setter! { location -> node <Self as Object>::Object }
 	crate::setter! { preview -> node <Self as Object>::Object }
 	crate::setter! { published -> chrono::DateTime<chrono::Utc> }
@@ -242,25 +293,28 @@ impl ObjectMut for serde_json::Value {
 	crate::setter! { replies -> node Self::Collection }
 	crate::setter! { likes -> node Self::Collection }
 	crate::setter! { shares -> node Self::Collection }
-	crate::setter! { start_time::startTime -> chrono::DateTime<chrono::Utc> }
+	crate::setter! { startTime -> chrono::DateTime<chrono::Utc> }
 	crate::setter! { summary -> &str }
 	crate::setter! { tag -> node <Self as Object>::Object }
 	crate::setter! { to -> node Self::Link }
 	crate::setter! { bto -> node Self::Link}
 	crate::setter! { cc -> node Self::Link }
 	crate::setter! { bcc -> node Self::Link }
-	crate::setter! { media_type::mediaType -> &str }
+	crate::setter! { mediaType -> &str }
 	crate::setter! { duration -> &str }
 	crate::setter! { url -> node Self::Link }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	crate::setter! { sensitive -> bool }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { quote_url::quoteUrl -> node <Self as Object>::Object }
+	crate::setter! { quoteUrl -> node <Self as Object>::Object }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { liked_by_me::likedByMe -> bool }
+	crate::setter! { likedByMe -> bool }
 
 	#[cfg(feature = "ostatus")]
 	crate::setter! { conversation -> node <Self as Object>::Object }
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	crate::setter! { value -> &str }
 }

apb/src/types/object/place.rs

@@ -1,10 +1,12 @@
+use crate::{Field, FieldErr};
+
 pub trait Place : super::Object {
-	fn accuracy(&self) -> Option<f64> { None }
+	fn accuracy(&self) -> Field<f64> { Err(FieldErr("accuracy")) }
-	fn altitude(&self) -> Option<f64> { None }
+	fn altitude(&self) -> Field<f64> { Err(FieldErr("altitude")) }
-	fn latitude(&self) -> Option<f64> { None }
+	fn latitude(&self) -> Field<f64> { Err(FieldErr("latitude")) }
-	fn longitude(&self) -> Option<f64> { None }
+	fn longitude(&self) -> Field<f64> { Err(FieldErr("longitude")) }
-	fn radius(&self) -> Option<f64> { None }
+	fn radius(&self) -> Field<f64> { Err(FieldErr("radius")) }
-	fn units(&self) -> Option<&str> { None }
+	fn units(&self) -> Field<&str> { Err(FieldErr("units")) }
 }
 
 pub trait PlaceMut : super::ObjectMut {

apb/src/types/object/tombstone.rs

@@ -1,6 +1,6 @@
 pub trait Tombstone : super::Object {
-	fn former_type(&self) -> Option<crate::BaseType> { None }
+	fn former_type(&self) -> crate::Field<crate::BaseType> { Err(crate::FieldErr("formerType")) }
-	fn deleted(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	fn deleted(&self) -> crate::Field<chrono::DateTime<chrono::Utc>> { Err(crate::FieldErr("deleted")) }
 }
 
 pub trait TombstoneMut : super::ObjectMut {

main.rs

@@ -0,0 +1,284 @@
+use std::path::PathBuf;
+use clap::{Parser, Subcommand};
+use sea_orm::{ConnectOptions, Database};
+use signal_hook::consts::signal::*;
+use signal_hook_tokio::Signals;
+use futures::stream::StreamExt;
+use upub::{context, ext::LoggableError};
+
+#[cfg(feature = "cli")]
+use upub_cli as cli;
+
+#[cfg(feature = "migrate")]
+use upub_migrations as migrations;
+
+#[cfg(feature = "serve")]
+use upub_routes as routes;
+
+#[cfg(feature = "worker")]
+use upub_worker as worker;
+
+
+#[derive(Parser)]
+/// all names were taken
+struct Args {
+	#[clap(subcommand)]
+	/// command to run
+	command: Mode,
+
+	/// path to config file, leave empty to not use any
+	#[arg(short, long)]
+	config: Option<PathBuf>,
+
+	#[arg(long = "db")]
+	/// database connection uri, overrides config value
+	database: Option<String>,
+
+	#[arg(long)]
+	/// instance base domain, for AP ids, overrides config value
+	domain: Option<String>,
+
+	#[arg(long, default_value_t=false)]
+	/// run with debug level tracing
+	debug: bool,
+
+	#[arg(long)]
+	/// force set number of worker threads for async runtime, defaults to number of cores
+	threads: Option<usize>,
+}
+
+#[derive(Clone, Subcommand)]
+enum Mode {
+	/// print current or default configuration
+	Config,
+
+	#[cfg(feature = "migrate")]
+	/// apply database migrations
+	Migrate,
+
+	#[cfg(feature = "cli")]
+	/// run maintenance CLI tasks
+	Cli {
+		#[clap(subcommand)]
+		/// task to run
+		command: cli::CliCommand,
+	},
+
+	#[cfg(all(feature = "serve", feature = "worker"))]
+	/// start both api routes and background workers
+	Monolith {
+		#[arg(short, long, default_value="127.0.0.1:3000")]
+		/// addr to bind and serve onto
+		bind: String,
+
+		#[arg(short, long, default_value_t = 4)]
+		/// how many concurrent jobs to process with this worker
+		tasks: usize,
+
+		#[arg(short, long, default_value_t = 20)]
+		/// interval for polling new tasks
+		poll: u64,
+	},
+
+	#[cfg(feature = "serve")]
+	/// start api routes server
+	Serve {
+		#[arg(short, long, default_value="127.0.0.1:3000")]
+		/// addr to bind and serve onto
+		bind: String,
+	},
+
+	#[cfg(feature = "worker")]
+	/// start background job worker
+	Work {
+		/// only run tasks of this type, run all if not given
+		filter: Filter,
+
+		/// how many concurrent jobs to process with this worker
+		#[arg(short, long, default_value_t = 4)]
+		tasks: usize,
+
+		#[arg(short, long, default_value_t = 20)]
+		/// interval for polling new tasks
+		poll: u64,
+	},
+}
+
+fn main() {
+	let args = Args::parse();
+
+	tracing_subscriber::fmt()
+		.compact()
+		.with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO })
+		.init();
+
+	let config = upub::Config::load(args.config.as_ref());
+
+	if matches!(args.command, Mode::Config) {
+		println!("{}", toml::to_string_pretty(&config).expect("failed serializing config"));
+		return;
+	}
+
+	let mut runtime = tokio::runtime::Builder::new_multi_thread();
+
+	if let Some(threads) = args.threads {
+		runtime.worker_threads(threads);
+	}
+
+	runtime
+		.enable_io()
+		.enable_time()
+		.thread_name("upub-worker")
+		.build()
+		.expect("failed creating tokio async runtime")
+		.block_on(async { init(args, config).await })
+}
+
+async fn init(args: Args, config: upub::Config) {
+	let database = args.database.unwrap_or(config.datasource.connection_string.clone());
+	let domain = args.domain.unwrap_or(config.instance.domain.clone());
+
+	// TODO can i do connectoptions.into() or .connect() and skip these ugly bindings?
+	let mut opts = ConnectOptions::new(&database);
+
+	opts
+		.sqlx_logging(true)
+		.sqlx_logging_level(tracing::log::LevelFilter::Debug)
+		.max_connections(config.datasource.max_connections)
+		.min_connections(config.datasource.min_connections)
+		.acquire_timeout(std::time::Duration::from_secs(config.datasource.acquire_timeout_seconds))
+		.connect_timeout(std::time::Duration::from_secs(config.datasource.connect_timeout_seconds))
+		.sqlx_slow_statements_logging_settings(
+			if config.datasource.slow_query_warn_enable { tracing::log::LevelFilter::Warn } else { tracing::log::LevelFilter::Debug },
+			std::time::Duration::from_secs(config.datasource.slow_query_warn_seconds)
+		);
+
+	let db = Database::connect(opts)
+		.await.expect("error connecting to db");
+
+	#[cfg(feature = "migrate")]
+	if matches!(args.command, Mode::Migrate) {
+		use migrations::MigratorTrait;
+
+		migrations::Migrator::up(&db, None)
+			.await
+			.expect("error applying migrations");
+
+		return;
+	}
+
+	let (tx_wake, rx_wake) = tokio::sync::mpsc::unbounded_channel();
+	let wake = WakeToken(rx_wake);
+
+	let ctx = upub::Context::new(db, domain, config.clone(), Some(Box::new(WakerToken(tx_wake))))
+		.await.expect("failed creating server context");
+
+	#[cfg(feature = "cli")]
+	if let Mode::Cli { command } = args.command {
+		cli::run(ctx, command)
+			.await.expect("failed running cli task");
+
+		return;
+	}
+
+
+	// register signal handler only for long-lasting modes, such as server or worker
+	let (tx, rx) = tokio::sync::watch::channel(false);
+	let signals = Signals::new([SIGTERM, SIGINT]).expect("failed registering signal handler");
+	let handle = signals.handle();
+	let signals_task = tokio::spawn(handle_signals(signals, tx));
+	let stop = CancellationToken(rx);
+
+	match args.command {
+		#[cfg(feature = "serve")]
+		Mode::Serve { bind } =>
+			routes::serve(ctx, bind, stop)
+				.await.expect("failed serving api routes"),
+
+		#[cfg(feature = "worker")]
+		Mode::Work { filter, tasks, poll } =>
+			worker::spawn(ctx, tasks, poll, filter.into(), stop, wake)
+				.await.expect("failed running worker"),
+
+		#[cfg(all(feature = "serve", feature = "worker"))]
+		Mode::Monolith { bind, tasks, poll } => {
+			worker::spawn(ctx.clone(), tasks, poll, None, stop.clone(), wake);
+
+			routes::serve(ctx, bind, stop)
+				.await.expect("failed serving api routes");
+		},
+
+		Mode::Config => unreachable!(),
+		#[cfg(feature = "migrate")]
+		Mode::Migrate => unreachable!(),
+		#[cfg(feature = "cli")]
+		Mode::Cli { .. } => unreachable!(),
+	}
+
+	handle.close();
+	signals_task.await.expect("failed joining signal handler task");
+}
+
+struct WakerToken(tokio::sync::mpsc::UnboundedSender<()>);
+impl context::WakerToken for WakerToken {
+	fn wake(&self) {
+		self.0.send(()).warn_failed("failed waking up workers");
+	}
+}
+
+struct WakeToken(tokio::sync::mpsc::UnboundedReceiver<()>);
+
+impl worker::WakeToken for WakeToken {
+	async fn wait(&mut self) {
+		let _ = self.0.recv().await;
+	}
+}
+
+#[derive(Clone)]
+struct CancellationToken(tokio::sync::watch::Receiver<bool>);
+
+impl worker::StopToken for CancellationToken {
+	fn stop(&self) -> bool {
+		*self.0.borrow()
+	}
+}
+
+impl routes::ShutdownToken for CancellationToken {
+	async fn event(mut self) {
+		self.0.changed().await.warn_failed("cancellation token channel closed, stopping...");
+	}
+}
+
+async fn handle_signals(
+	mut signals: signal_hook_tokio::Signals,
+	tx: tokio::sync::watch::Sender<bool>,
+) {
+	while let Some(signal) = signals.next().await {
+		match signal {
+			SIGTERM | SIGINT => {
+				tracing::info!("received stop signal, closing tasks");
+				tx.send(true).info_failed("error sending stop signal to tasks")
+			},
+			_ => unreachable!(),
+		}
+	}
+}
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+enum Filter {
+	All,
+	Delivery,
+	Inbound,
+	Outbound,
+}
+
+impl From<Filter> for Option<upub::model::job::JobType> {
+	fn from(value: Filter) -> Self {
+		match value {
+			Filter::All => None,
+			Filter::Delivery => Some(upub::model::job::JobType::Delivery),
+			Filter::Inbound => Some(upub::model::job::JobType::Inbound),
+			Filter::Outbound => Some(upub::model::job::JobType::Outbound),
+		}
+	}
+}

mdhtml/src/lib.rs

@@ -1,96 +0,0 @@
-use html5ever::tendril::*;
-use html5ever::tokenizer::{BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer};
-use comrak::{markdown_to_html, Options};
-
-/// In our case, our sink only contains a tokens vector
-#[derive(Debug, Clone, Default)]
-struct Sink(String);
-
-impl TokenSink for Sink {
-	type Handle = ();
-
-	/// Each processed token will be handled by this method
-	fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
-		match token {
-			Token::TagToken(tag) => {
-				if !matches!(
-					tag.name.as_ref(),
-					"h1" | "h2" | "h3" 
-					| "hr" | "br" | "p" | "b" | "i"
-					| "blockquote" | "pre" | "code"
-					| "ul" | "ol" | "li"
-					| "img" | "a"
-				) { return TokenSinkResult::Continue } // skip this tag
-
-				self.0.push('<');
-
-				if !tag.self_closing && matches!(tag.kind, TagKind::EndTag) {
-					self.0.push('/');
-				}
-
-				self.0.push_str(tag.name.as_ref());
-
-				if !matches!(tag.kind, TagKind::EndTag) {
-					match tag.name.as_ref() {
-						"img" => for attr in tag.attrs {
-							match attr.name.local.as_ref() {
-								"src" => self.0.push_str(&format!(" src=\"{}\"", attr.value.as_ref())),
-								"title" => self.0.push_str(&format!(" title=\"{}\"", attr.value.as_ref())),
-								"alt" => self.0.push_str(&format!(" alt=\"{}\"", attr.value.as_ref())),
-								_ => {},
-							}
-						},
-						"a" => {
-							let any_attr = !tag.attrs.is_empty();
-							for attr in tag.attrs {
-								match attr.name.local.as_ref() {
-									"href" => self.0.push_str(&format!(" href=\"{}\"", attr.value.as_ref())),
-									"title" => self.0.push_str(&format!(" title=\"{}\"", attr.value.as_ref())),
-									_ => {},
-								}
-							}
-							if any_attr {
-								self.0.push_str(" rel=\"nofollow noreferrer\" target=\"_blank\"");
-							}
-						},
-						_ => {},
-					}
-				}
-
-				if tag.self_closing {
-					self.0.push('/');
-				}
-
-				self.0.push('>');
-			},
-			Token::CharacterTokens(txt) => self.0.push_str(txt.as_ref()),
-			Token::CommentToken(_) => {},
-			Token::DoctypeToken(_) => {},
-			Token::NullCharacterToken => {},
-			Token::EOFToken => {},
-			Token::ParseError(e) => tracing::error!("error parsing html: {e}"),
-		}
-		TokenSinkResult::Continue
-	}
-}
-
-pub fn safe_markdown(text: &str) -> String {
-	safe_html(&markdown_to_html(text, &Options::default()))
-}
-
-pub fn safe_html(text: &str) -> String {
-	let mut input = BufferQueue::default();
-	input.push_back(text.to_tendril().try_reinterpret().unwrap());
-
-	let sink = Sink::default();
-
-	let mut tok = Tokenizer::new(sink, Default::default());
-	let _ = tok.feed(&mut input);
-
-	if !input.is_empty() {
-		tracing::warn!("buffer input not empty after processing html");
-	}
-	tok.end();
-
-	tok.sink.0
-}

src/cli/fetch.rs

@@ -1,36 +0,0 @@
-use sea_orm::EntityTrait;
-
-use crate::server::{fetcher::Fetchable, normalizer::Normalizer, Context};
-
-pub async fn fetch(ctx: crate::server::Context, uri: String, save: bool) -> crate::Result<()> {
-	use apb::Base;
-
-	let mut node = apb::Node::link(uri.to_string());
-	node.fetch(&ctx).await?;
-
-	let obj = node.extract().expect("node still empty after fetch?");
-	let server = Context::server(&uri);
-
-	println!("{}", serde_json::to_string_pretty(&obj).unwrap());
-
-	if save {
-		match obj.base_type() {
-			Some(apb::BaseType::Object(apb::ObjectType::Actor(_))) => {
-				crate::model::actor::Entity::insert(
-					crate::model::actor::ActiveModel::new(&obj).unwrap()
-				).exec(ctx.db()).await.unwrap();
-			},
-			Some(apb::BaseType::Object(apb::ObjectType::Activity(_))) => {
-				ctx.insert_activity(obj, Some(server)).await.unwrap();
-			},
-			Some(apb::BaseType::Object(apb::ObjectType::Note)) => {
-				ctx.insert_object(obj, Some(server)).await.unwrap();
-			},
-			Some(apb::BaseType::Object(t)) => tracing::warn!("not implemented: {:?}", t),
-			Some(apb::BaseType::Link(_)) => tracing::error!("fetched another link?"),
-			None => tracing::error!("no type on object"),
-		}
-	}
-
-	Ok(())
-}

src/cli/relay.rs

@@ -1,41 +0,0 @@
-use sea_orm::{ActiveValue::{Set, NotSet}, ColumnTrait, EntityTrait, QueryFilter, QueryOrder};
-
-use crate::server::addresser::Addresser;
-
-pub async fn relay(ctx: crate::server::Context, actor: String, accept: bool) -> crate::Result<()> {
-	let aid = ctx.aid(&uuid::Uuid::new_v4().to_string());
-
-	let mut activity_model = crate::model::activity::ActiveModel {
-		internal: NotSet,
-		id: Set(aid.clone()),
-		activity_type: Set(apb::ActivityType::Follow),
-		actor: Set(ctx.base().to_string()),
-		object: Set(Some(actor.clone())),
-		target: Set(None),
-		published: Set(chrono::Utc::now()),
-		to: Set(crate::model::Audience(vec![actor.clone()])),
-		bto: Set(crate::model::Audience::default()),
-		cc: Set(crate::model::Audience(vec![apb::target::PUBLIC.to_string()])),
-		bcc: Set(crate::model::Audience::default()),
-	};
-
-	if accept {
-		let follow_req = crate::model::activity::Entity::find()
-			.filter(crate::model::activity::Column::ActivityType.eq("Follow"))
-			.filter(crate::model::activity::Column::Actor.eq(&actor))
-			.filter(crate::model::activity::Column::Object.eq(ctx.base()))
-			.order_by_desc(crate::model::activity::Column::Published)
-			.one(ctx.db())
-			.await?
-			.expect("no follow request to accept");
-		activity_model.activity_type = Set(apb::ActivityType::Accept(apb::AcceptType::Accept));
-		activity_model.object = Set(Some(follow_req.id));
-	};
-
-	crate::model::activity::Entity::insert(activity_model)
-		.exec(ctx.db()).await?;
-
-	ctx.dispatch(ctx.base(), vec![actor, apb::target::PUBLIC.to_string()], &aid, None).await?;
-
-	Ok(())
-}

src/cli/update.rs

@@ -1,45 +0,0 @@
-use futures::TryStreamExt;
-use sea_orm::{ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter};
-
-use crate::server::fetcher::Fetcher;
-
-pub async fn update_users(ctx: crate::server::Context, days: i64) -> crate::Result<()> {
-	let mut count = 0;
-	let mut insertions = Vec::new();
-
-	{
-		let mut stream = crate::model::actor::Entity::find()
-			.filter(crate::model::actor::Column::Updated.lt(chrono::Utc::now() - chrono::Duration::days(days)))
-			.stream(ctx.db())
-			.await?;
-
-
-		while let Some(user) = stream.try_next().await? {
-			if ctx.is_local(&user.id) { continue }
-			match ctx.pull(&user.id).await.map(|x| x.actor()) {
-				Err(e) => tracing::warn!("could not update user {}: {e}", user.id),
-				Ok(Err(e)) => tracing::warn!("could not update user {}: {e}", user.id),
-				Ok(Ok(doc)) => match crate::model::actor::ActiveModel::new(&doc) {
-					Ok(mut u) => {
-						u.internal = Set(user.internal);
-						u.updated = Set(chrono::Utc::now());
-						insertions.push((user.id, u));
-						count += 1;
-					},
-					Err(e) => tracing::warn!("failed deserializing user '{}': {e}", user.id),
-				},
-			}
-		}
-	}
-
-	for (uid, user_model) in insertions {
-		tracing::info!("updating user {}", uid);
-		crate::model::actor::Entity::update(user_model)
-			.exec(ctx.db())
-			.await?;
-	}
-
-	tracing::info!("updated {count} users");
-
-	Ok(())
-}

src/main.rs

@@ -1,145 +0,0 @@
-mod server;
-mod model;
-mod routes;
-
-pub mod errors;
-mod config;
-
-#[cfg(feature = "cli")]
-mod cli;
-
-#[cfg(feature = "migrations")]
-mod migrations;
-
-#[cfg(feature = "migrations")]
-use sea_orm_migration::MigratorTrait;
-use std::path::PathBuf;
-use config::Config;
-
-use clap::{Parser, Subcommand};
-use sea_orm::{ConnectOptions, Database};
-
-pub use errors::UpubResult as Result;
-use tower_http::{cors::CorsLayer, trace::TraceLayer};
-
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-
-#[derive(Parser)]
-/// all names were taken
-struct Args {
-	#[clap(subcommand)]
-	/// command to run
-	command: Mode,
-
-	/// path to config file, leave empty to not use any
-	#[arg(short, long)]
-	config: Option<PathBuf>,
-
-	#[arg(long = "db")]
-	/// database connection uri, overrides config value
-	database: Option<String>,
-
-	#[arg(long)]
-	/// instance base domain, for AP ids, overrides config value
-	domain: Option<String>,
-
-	#[arg(long, default_value_t=false)]
-	/// run with debug level tracing
-	debug: bool,
-}
-
-#[derive(Clone, Subcommand)]
-enum Mode {
-	/// run fediverse server
-	Serve {
-		#[arg(short, long, default_value="127.0.0.1:3000")]
-		/// addr to bind and serve onto
-		bind: String,
-	},
-
-	/// print current or default configuration
-	Config,
-
-	#[cfg(feature = "migrations")]
-	/// apply database migrations
-	Migrate,
-
-	#[cfg(feature = "cli")]
-	/// run maintenance CLI tasks
-	Cli {
-		#[clap(subcommand)]
-		/// task to run
-		command: cli::CliCommand,
-	},
-}
-
-#[tokio::main]
-async fn main() {
-
-	let args = Args::parse();
-
-	tracing_subscriber::fmt()
-		.compact()
-		.with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO })
-		.init();
-
-	let config = Config::load(args.config);
-
-	let database = args.database.unwrap_or(config.datasource.connection_string.clone());
-	let domain = args.domain.unwrap_or(config.instance.domain.clone());
-
-	// TODO can i do connectoptions.into() or .connect() and skip these ugly bindings?
-	let mut opts = ConnectOptions::new(&database);
-
-	opts
-		.sqlx_logging(true)
-		.sqlx_logging_level(tracing::log::LevelFilter::Debug)
-		.max_connections(config.datasource.max_connections)
-		.min_connections(config.datasource.min_connections)
-		.acquire_timeout(std::time::Duration::from_secs(config.datasource.acquire_timeout_seconds))
-		.connect_timeout(std::time::Duration::from_secs(config.datasource.connect_timeout_seconds))
-		.sqlx_slow_statements_logging_settings(
-			if config.datasource.slow_query_warn_enable { tracing::log::LevelFilter::Warn } else { tracing::log::LevelFilter::Debug },
-			std::time::Duration::from_secs(config.datasource.slow_query_warn_seconds)
-		);
-
-	let db = Database::connect(opts)
-		.await.expect("error connecting to db");
-
-	match args.command {
-		#[cfg(feature = "migrations")]
-		Mode::Migrate =>
-			migrations::Migrator::up(&db, None)
-				.await.expect("error applying migrations"),
-
-		#[cfg(feature = "cli")]
-		Mode::Cli { command } =>
-			cli::run(command, db, domain, config)
-				.await.expect("failed running cli task"),
-
-		Mode::Config => println!("{}", toml::to_string_pretty(&config).expect("failed serializing config")),
-
-		Mode::Serve { bind } => {
-			let ctx = server::Context::new(db, domain, config)
-				.await.expect("failed creating server context");
-
-			use routes::activitypub::ActivityPubRouter;
-			use routes::mastodon::MastodonRouter;
-
-			let router = axum::Router::new()
-				.ap_routes()
-				.mastodon_routes() // no-op if mastodon feature is disabled
-				.layer(CorsLayer::permissive())
-				.layer(TraceLayer::new_for_http())
-				.with_state(ctx);
-
-			// run our app with hyper, listening locally on port 3000
-			let listener = tokio::net::TcpListener::bind(bind)
-				.await.expect("could not bind tcp socket");
-
-			axum::serve(listener, router)
-				.await
-				.expect("failed serving application")
-		},
-	}
-}

src/migrations/README.md

@@ -1,3 +0,0 @@
-# migrations
-
-there are sea_orm migrations to apply to your database

src/migrations/mod.rs

@@ -1,24 +0,0 @@
-use sea_orm_migration::prelude::*;
-
-mod m20240524_000001_create_actor_activity_object_tables;
-mod m20240524_000002_create_relations_likes_shares;
-mod m20240524_000003_create_users_auth_and_config;
-mod m20240524_000004_create_addressing_deliveries;
-mod m20240524_000005_create_attachments_tags_mentions;
-mod m20240529_000001_add_relation_unique_index;
-
-pub struct Migrator;
-
-#[async_trait::async_trait]
-impl MigratorTrait for Migrator {
-	fn migrations() -> Vec<Box<dyn MigrationTrait>> {
-		vec![
-			Box::new(m20240524_000001_create_actor_activity_object_tables::Migration),
-			Box::new(m20240524_000002_create_relations_likes_shares::Migration),
-			Box::new(m20240524_000003_create_users_auth_and_config::Migration),
-			Box::new(m20240524_000004_create_addressing_deliveries::Migration),
-			Box::new(m20240524_000005_create_attachments_tags_mentions::Migration),
-			Box::new(m20240529_000001_add_relation_unique_index::Migration),
-		]
-	}
-}

src/model/addressing.rs

@@ -1,191 +0,0 @@
-use apb::{ActivityMut, ObjectMut};
-use sea_orm::{entity::prelude::*, sea_query::IntoCondition, Condition, FromQueryResult, Iterable, Order, QueryOrder, QuerySelect, SelectColumns};
-
-use crate::routes::activitypub::jsonld::LD;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "addressing")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub internal: i64,
-	pub actor: Option<i64>,
-	pub instance: Option<i64>,
-	pub activity: Option<i64>,
-	pub object: Option<i64>,
-	pub published: ChronoDateTimeUtc,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Activities,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Actor",
-		to = "super::actor::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Actors,
-	#[sea_orm(
-		belongs_to = "super::instance::Entity",
-		from = "Column::Instance",
-		to = "super::instance::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Instances,
-	#[sea_orm(
-		belongs_to = "super::object::Entity",
-		from = "Column::Object",
-		to = "super::object::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Objects,
-}
-
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Activities.def()
-	}
-}
-
-impl Related<super::actor::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Actors.def()
-	}
-}
-
-impl Related<super::instance::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Instances.def()
-	}
-}
-
-impl Related<super::object::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Objects.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-
-
-#[allow(clippy::large_enum_variant)] // tombstone is an outlier, not the norm! this is a beefy enum
-#[derive(Debug, Clone)]
-pub enum Event {
-	Tombstone,
-	Activity(crate::model::activity::Model),
-	StrayObject {
-		object: crate::model::object::Model,
-		liked: Option<String>,
-	},
-	DeepActivity {
-		activity: crate::model::activity::Model,
-		object: crate::model::object::Model,
-		liked: Option<String>,
-	}
-}
-
-
-impl Event {
-	pub fn internal(&self) -> i64 {
-		match self {
-			Event::Tombstone => 0,
-			Event::Activity(x) => x.internal,
-			Event::StrayObject { object, liked: _ } => object.internal,
-			Event::DeepActivity { activity: _, liked: _, object } => object.internal,
-		}
-	}
-
-	pub fn ap(self, attachment: Option<Vec<crate::model::attachment::Model>>) -> serde_json::Value {
-		let attachment = match attachment {
-			None => apb::Node::Empty,
-			Some(vec) => apb::Node::array(
-				vec.into_iter().map(|x| x.ap()).collect()
-			),
-		};
-		match self {
-			Event::Activity(x) => x.ap(),
-			Event::DeepActivity { activity, object, liked } =>
-				activity.ap().set_object(apb::Node::object(
-					object.ap()
-						.set_attachment(attachment)
-						.set_liked_by_me(if liked.is_some() { Some(true) } else { None })
-				)),
-			Event::StrayObject { object, liked } => serde_json::Value::new_object()
-				.set_activity_type(Some(apb::ActivityType::Activity))
-				.set_object(apb::Node::object(
-					object.ap()
-						.set_attachment(attachment)
-						.set_liked_by_me(if liked.is_some() { Some(true) } else { None })
-				)),
-			Event::Tombstone => serde_json::Value::new_object()
-				.set_activity_type(Some(apb::ActivityType::Activity))
-				.set_object(apb::Node::object(
-					serde_json::Value::new_object()
-						.set_object_type(Some(apb::ObjectType::Tombstone))
-				)),
-		}
-	}
-}
-
-impl FromQueryResult for Event {
-	fn from_query_result(res: &sea_orm::QueryResult, _pre: &str) -> Result<Self, sea_orm::DbErr> {
-		let activity = crate::model::activity::Model::from_query_result(res, crate::model::activity::Entity.table_name()).ok();
-		let object = crate::model::object::Model::from_query_result(res, crate::model::object::Entity.table_name()).ok();
-		let liked = res.try_get(crate::model::like::Entity.table_name(), &crate::model::like::Column::Actor.to_string()).ok();
-		match (activity, object) {
-			(Some(activity), Some(object)) => Ok(Self::DeepActivity { activity, object, liked }),
-			(Some(activity), None) => Ok(Self::Activity(activity)),
-			(None, Some(object)) => Ok(Self::StrayObject { object, liked }),
-			(None, None) => Ok(Self::Tombstone),
-		}
-	}
-}
-
-
-impl Entity {
-	pub fn find_addressed(uid: Option<i64>) -> Select<Entity> {
-		let mut select = Entity::find()
-			.distinct()
-			.select_only()
-			.join(sea_orm::JoinType::LeftJoin, Relation::Objects.def())
-			.join(sea_orm::JoinType::LeftJoin, Relation::Activities.def())
-			.filter(
-				// TODO ghetto double inner join because i want to filter out tombstones
-				Condition::any()
-					.add(crate::model::activity::Column::Id.is_not_null())
-					.add(crate::model::object::Column::Id.is_not_null())
-			)
-			.order_by(Column::Published, Order::Desc);
-
-		if let Some(uid) = uid {
-			select = select
-				.join(
-					sea_orm::JoinType::LeftJoin,
-					crate::model::object::Relation::Likes.def()
-						.on_condition(move |_l, _r| crate::model::like::Column::Actor.eq(uid).into_condition()),
-				)
-				.select_column_as(crate::model::like::Column::Actor, format!("{}{}", crate::model::like::Entity.table_name(), crate::model::like::Column::Actor.to_string()));
-		}
-
-		for col in crate::model::object::Column::iter() {
-			select = select.select_column_as(col, format!("{}{}", crate::model::object::Entity.table_name(), col.to_string()));
-		}
-
-		for col in crate::model::activity::Column::iter() {
-			select = select.select_column_as(col, format!("{}{}", crate::model::activity::Entity.table_name(), col.to_string()));
-		}
-
-		select
-	}
-}

src/model/attachment.rs

@@ -1,98 +0,0 @@
-use apb::{DocumentMut, DocumentType, ObjectMut};
-use sea_orm::entity::prelude::*;
-
-use crate::routes::activitypub::jsonld::LD;
-
-use super::addressing::Event;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "attachments")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub internal: i64,
-	#[sea_orm(unique)]
-	pub url: String,
-	pub object: i64,
-	pub document_type: DocumentType,
-	pub name: Option<String>,
-	pub media_type: String,
-	pub published: ChronoDateTimeUtc,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::object::Entity",
-		from = "Column::Object",
-		to = "super::object::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Objects,
-}
-
-impl Related<super::object::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Objects.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Model {
-	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
-			.set_url(apb::Node::link(self.url))
-			.set_document_type(Some(self.document_type))
-			.set_media_type(Some(&self.media_type))
-			.set_name(self.name.as_deref())
-			.set_published(Some(self.published))
-	}
-}
-
-#[axum::async_trait]
-pub trait BatchFillable {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<i64, Vec<Model>>, DbErr>;
-}
-
-#[axum::async_trait]
-impl BatchFillable for &[Event] {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<i64, Vec<Model>>, DbErr> {
-		let objects : Vec<crate::model::object::Model> = self
-			.iter()
-			.filter_map(|x| match x {
-				Event::Tombstone => None,
-				Event::Activity(_) => None,
-				Event::StrayObject { object, liked: _ } => Some(object.clone()),
-				Event::DeepActivity { activity: _, liked: _, object } => Some(object.clone()),
-			})
-			.collect();
-
-		let attachments = objects.load_many(Entity, db).await?;
-
-		let mut out : std::collections::BTreeMap<i64, Vec<Model>> = std::collections::BTreeMap::new();
-		for attach in attachments.into_iter().flatten() {
-			match out.entry(attach.object) {
-				std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); },
-				std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); },
-			}
-		}
-
-		Ok(out)
-	}
-}
-
-#[axum::async_trait]
-impl BatchFillable for Vec<Event> {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<i64, Vec<Model>>, DbErr> {
-		self.as_slice().load_attachments_batch(db).await
-	}
-}
-
-#[axum::async_trait]
-impl BatchFillable for Event {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<i64, Vec<Model>>, DbErr> {
-		let x = vec![self.clone()]; // TODO wasteful clone and vec![] but ehhh convenient
-		x.load_attachments_batch(db).await
-	}
-}

src/model/delivery.rs

@@ -1,66 +0,0 @@
-use sea_orm::entity::prelude::*;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "deliveries")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub internal: i64,
-	pub actor: String,
-	pub target: String,
-	pub activity: String,
-	pub published: ChronoDateTimeUtc,
-	pub not_before: ChronoDateTimeUtc,
-	pub attempt: i32,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Activities,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Actor",
-		to = "super::actor::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Actors,
-}
-
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Activities.def()
-	}
-}
-
-impl Related<super::actor::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Actors.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Model {
-	pub fn next_delivery(&self) -> ChronoDateTimeUtc {
-		match self.attempt {
-			0 => chrono::Utc::now() + std::time::Duration::from_secs(10),
-			1 => chrono::Utc::now() + std::time::Duration::from_secs(60),
-			2 => chrono::Utc::now() + std::time::Duration::from_secs(5 * 60),
-			3 => chrono::Utc::now() + std::time::Duration::from_secs(20 * 60),
-			4 => chrono::Utc::now() + std::time::Duration::from_secs(60 * 60),
-			5 => chrono::Utc::now() + std::time::Duration::from_secs(12 * 60 * 60),
-			_ => chrono::Utc::now() + std::time::Duration::from_secs(24 * 60 * 60),
-		}
-	}
-
-	pub fn expired(&self) -> bool {
-		chrono::Utc::now() - self.published > chrono::Duration::days(7)
-	}
-}

src/model/mod.rs

@@ -1,48 +0,0 @@
-pub mod actor;
-pub mod object;
-pub mod activity;
-
-pub mod config;
-pub mod credential;
-pub mod session;
-
-pub mod instance;
-pub mod delivery;
-
-pub mod relation;
-pub mod announce;
-pub mod like;
-
-pub mod hashtag;
-pub mod mention;
-pub mod attachment;
-
-pub mod addressing;
-
-#[derive(Debug, Clone, thiserror::Error)]
-#[error("missing required field: '{0}'")]
-pub struct FieldError(pub &'static str);
-
-impl From<FieldError> for axum::http::StatusCode {
-	fn from(value: FieldError) -> Self {
-		tracing::error!("bad request: {value}");
-		axum::http::StatusCode::BAD_REQUEST
-	}
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, sea_orm::FromJsonQueryResult)]
-pub struct Audience(pub Vec<String>);
-
-impl<T: apb::Base> From<apb::Node<T>> for Audience {
-	fn from(value: apb::Node<T>) -> Self {
-		Audience(
-			match value {
-				apb::Node::Empty => vec![],
-				apb::Node::Link(l) => vec![l.href().to_string()],
-				apb::Node::Object(o) => if let Some(id) = o.id() { vec![id.to_string()] } else { vec![] },
-				apb::Node::Array(arr) => arr.into_iter().filter_map(|l| Some(l.id()?.to_string())).collect(),
-			}
-		)
-	}
-}
-

src/model/relation.rs

@@ -1,119 +0,0 @@
-use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "relations")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub internal: i64,
-	pub follower: i64,
-	pub following: i64,
-	pub accept: Option<i64>,
-	pub activity: i64,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Accept",
-		to = "super::activity::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	ActivitiesAccept,
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	ActivitiesFollow,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Follower",
-		to = "super::actor::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	ActorsFollower,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Following",
-		to = "super::actor::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	ActorsFollowing,
-}
-
-impl Related<super::actor::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::ActorsFollowing.def()
-	}
-}
-
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::ActivitiesFollow.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Entity {
-	// TODO this is 2 queries!!! can it be optimized down to 1?
-	pub async fn followers(uid: &str, db: &DatabaseConnection) -> crate::Result<Vec<String>> {
-		let internal_id = super::actor::Entity::ap_to_internal(uid, db).await?;
-		let out = Entity::find()
-			.join(
-				sea_orm::JoinType::InnerJoin,
-				Entity::belongs_to(super::actor::Entity)
-					.from(Column::Follower)
-					.to(super::actor::Column::Internal)
-					.into()
-			)
-			.filter(Column::Accept.is_not_null())
-			.filter(Column::Following.eq(internal_id))
-			.select_only()
-			.select_column(super::actor::Column::Id)
-			.into_tuple::<String>()
-			.all(db)
-			.await?;
-
-		Ok(out)
-	}
-
-	// TODO this is 2 queries!!! can it be optimized down to 1?
-	pub async fn following(uid: &str, db: &DatabaseConnection) -> crate::Result<Vec<String>> {
-		let internal_id = super::actor::Entity::ap_to_internal(uid, db).await?;
-		let out = Entity::find()
-			.join(
-				sea_orm::JoinType::InnerJoin,
-				Entity::belongs_to(super::actor::Entity)
-					.from(Column::Following)
-					.to(super::actor::Column::Internal)
-					.into()
-			)
-			.filter(Column::Accept.is_not_null())
-			.filter(Column::Follower.eq(internal_id))
-			.select_only()
-			.select_column(super::actor::Column::Id)
-			.into_tuple::<String>()
-			.all(db)
-			.await?;
-
-		Ok(out)
-	}
-
-	// TODO this is 3 queries!!! can it be optimized down to 1?
-	pub fn is_following(follower: i64, following: i64) -> sea_orm::Selector<sea_orm::SelectGetableTuple<i64>> {
-		Entity::find()
-			.filter(Column::Accept.is_not_null())
-			.filter(Column::Follower.eq(follower))
-			.filter(Column::Following.eq(following))
-			.select_only()
-			.select_column(Column::Internal)
-			.into_tuple::<i64>()
-	}
-}

src/routes/activitypub/activity.rs

@@ -1,34 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, QueryFilter};
-use crate::{errors::UpubError, model::{self, addressing::Event, attachment::BatchFillable}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}};
-
-use super::{jsonld::LD, JsonLD, TryFetch};
-
-pub async fn view(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(query): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let aid = ctx.aid(&id);
-	if auth.is_local() && query.fetch && !ctx.is_local(&aid) {
-		let obj = ctx.fetch_activity(&aid).await?;
-		if obj.id != aid {
-			return Err(UpubError::Redirect(obj.id));
-		}
-	}
-
-	let row = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(model::activity::Column::Id.eq(&aid))
-		.filter(auth.filter_condition())
-		.into_model::<Event>()
-		.one(ctx.db())
-		.await?
-		.ok_or_else(UpubError::not_found)?;
-
-	let mut attachments = row.load_attachments_batch(ctx.db()).await?;
-	let attach = attachments.remove(&row.internal());
-
-	Ok(JsonLD(row.ap(attach).ld_context()))
-}
-

src/routes/activitypub/application.rs

@@ -1,93 +0,0 @@
-use apb::{ActorMut, BaseMut, ObjectMut, PublicKeyMut};
-use axum::{extract::{Query, State}, http::HeaderMap, response::{IntoResponse, Redirect, Response}, Form, Json};
-use reqwest::Method;
-
-use crate::{errors::UpubError, server::{auth::AuthIdentity, fetcher::Fetcher, Context}, url};
-
-use super::{jsonld::LD, JsonLD};
-
-
-pub async fn view(
-	headers: HeaderMap,
-	State(ctx): State<Context>,
-) -> crate::Result<Response> {
-	if let Some(accept) = headers.get("Accept") {
-		if let Ok(accept) = accept.to_str() {
-			if accept.contains("text/html") && !accept.contains("application/ld+json") {
-				return Ok(Redirect::to("/web").into_response());
-			}
-		}
-	}
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(&url!(ctx, "")))
-			.set_actor_type(Some(apb::ActorType::Application))
-			.set_name(Some(&ctx.cfg().instance.name))
-			.set_summary(Some(&ctx.cfg().instance.description))
-			.set_inbox(apb::Node::link(url!(ctx, "/inbox")))
-			.set_outbox(apb::Node::link(url!(ctx, "/outbox")))
-			.set_published(Some(ctx.actor().published))
-			.set_endpoints(apb::Node::Empty)
-			.set_preferred_username(Some(ctx.domain()))
-			.set_public_key(apb::Node::object(
-				serde_json::Value::new_object()
-					.set_id(Some(&url!(ctx, "#main-key")))
-					.set_owner(Some(&url!(ctx, "")))
-					.set_public_key_pem(&ctx.actor().public_key)
-			))
-			.ld_context()
-	).into_response())
-}
-
-#[derive(Debug, serde::Deserialize)]
-pub struct FetchPath {
-	id: String,
-}
-
-pub async fn proxy_get(
-	State(ctx): State<Context>,
-	Query(query): Query<FetchPath>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<Json<serde_json::Value>> {
-	// only local users can request fetches
-	if !ctx.cfg().security.allow_public_debugger && !auth.is_local() {
-		return Err(UpubError::unauthorized());
-	}
-	Ok(Json(
-		Context::request(
-			Method::GET,
-			&query.id,
-			None,
-			ctx.base(),
-			ctx.pkey(),
-			&format!("{}+proxy", ctx.domain()),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?
-	))
-}
-
-pub async fn proxy_form(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Form(query): Form<FetchPath>,
-) -> crate::Result<Json<serde_json::Value>> {
-	// only local users can request fetches
-	if !ctx.cfg().security.allow_public_debugger && auth.is_local() {
-		return Err(UpubError::unauthorized());
-	}
-	Ok(Json(
-		Context::request(
-			Method::GET,
-			&query.id,
-			None,
-			ctx.base(),
-			ctx.pkey(),
-			&format!("{}+proxy", ctx.domain()),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?
-	))
-}

src/routes/activitypub/context.rs

@@ -1,42 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, Condition, PaginatorTrait, QueryFilter};
-
-use crate::{model, routes::activitypub::{JsonLD, Pagination}, server::{auth::AuthIdentity, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let local_context_id = url!(ctx, "/context/{id}");
-	let context = ctx.oid(&id);
-
-	let count = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(auth.filter_condition())
-		.filter(model::object::Column::Context.eq(context))
-		.count(ctx.db())
-		.await?;
-
-	crate::server::builders::collection(&local_context_id, Some(count))
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let context = ctx.oid(&id);
-
-	crate::server::builders::paginate(
-		url!(ctx, "/context/{id}/page"),
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(model::object::Column::Context.eq(context)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		false,
-	)
-		.await
-}

src/routes/activitypub/inbox.rs

@@ -1,96 +0,0 @@
-use apb::{server::Inbox, Activity, ActivityType};
-use axum::{extract::{Query, State}, http::StatusCode, Json};
-use sea_orm::{sea_query::IntoCondition, ColumnTrait};
-
-use crate::{errors::UpubError, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-use super::{JsonLD, Pagination};
-
-
-pub async fn get(
-	State(ctx): State<Context>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/inbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::paginate(
-		url!(ctx, "/inbox/page"),
-		crate::model::addressing::Column::Actor.is_null()
-			.into_condition(),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		false,
-	)
-		.await
-}
-
-macro_rules! pretty_json {
-	($json:ident) => {
-		serde_json::to_string_pretty(&$json).expect("failed serializing to string serde_json::Value")
-	}
-}
-
-
-pub async fn post(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>
-) -> crate::Result<()> {
-	let Identity::Remote { domain: server, .. } = auth else {
-		if activity.activity_type() == Some(ActivityType::Delete) {
-			// this is spammy af, ignore them!
-			// we basically received a delete for a user we can't fetch and verify, meaning remote
-			// deleted someone we never saw. technically we deleted nothing so we should return error,
-			// but mastodon keeps hammering us trying to delete this user, so just make mastodon happy
-			// and return 200 without even bothering checking this stuff
-			// would be cool if mastodon played nicer with the network...
-			return Ok(());
-		}
-		tracing::warn!("refusing unauthorized activity: {}", pretty_json!(activity));
-		if matches!(auth, Identity::Anonymous) {
-			return Err(UpubError::unauthorized());
-		} else {
-			return Err(UpubError::forbidden());
-		}
-	};
-
-	let Some(actor) = activity.actor().id() else {
-		return Err(UpubError::bad_request());
-	};
-
-	if server != Context::server(&actor) {
-		return Err(UpubError::unauthorized());
-	}
-
-	tracing::debug!("processing federated activity: '{}'", serde_json::to_string(&activity).unwrap_or_default());
-
-	// TODO we could process Links and bare Objects maybe, but probably out of AP spec?
-	match activity.activity_type().ok_or_else(UpubError::bad_request)? {
-		ActivityType::Activity => {
-			tracing::warn!("skipping unprocessable base activity: {}", pretty_json!(activity));
-			Err(StatusCode::UNPROCESSABLE_ENTITY.into()) // won't ingest useless stuff
-		},
-
-		// TODO emojireacts are NOT likes, but let's process them like ones for now maybe?
-		ActivityType::Like | ActivityType::EmojiReact => Ok(ctx.like(server, activity).await?),
-		ActivityType::Create => Ok(ctx.create(server, activity).await?),
-		ActivityType::Follow => Ok(ctx.follow(server, activity).await?),
-		ActivityType::Announce => Ok(ctx.announce(server, activity).await?),
-		ActivityType::Accept(_) => Ok(ctx.accept(server, activity).await?),
-		ActivityType::Reject(_) => Ok(ctx.reject(server, activity).await?),
-		ActivityType::Undo => Ok(ctx.undo(server, activity).await?),
-		ActivityType::Delete => Ok(ctx.delete(server, activity).await?),
-		ActivityType::Update => Ok(ctx.update(server, activity).await?),
-
-		_x => {
-			tracing::info!("received unimplemented activity on inbox: {}", pretty_json!(activity));
-			Err(StatusCode::NOT_IMPLEMENTED.into())
-		},
-	}
-}

src/routes/activitypub/object/mod.rs

@@ -1,76 +0,0 @@
-pub mod replies;
-
-use apb::{CollectionMut, ObjectMut};
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, ModelTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{errors::UpubError, model::{self, addressing::Event}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}};
-
-use super::{jsonld::LD, JsonLD, TryFetch};
-
-pub async fn view(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(query): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let oid = ctx.oid(&id);
-	if auth.is_local() && query.fetch && !ctx.is_local(&oid) {
-		let obj = ctx.fetch_object(&oid).await?;
-		// some implementations serve statuses on different urls than their AP id
-		if obj.id != oid {
-			return Err(UpubError::Redirect(crate::url!(ctx, "/objects/{}", ctx.id(&obj.id))));
-		}
-	}
-
-	let item = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(model::object::Column::Id.eq(&oid))
-		.filter(auth.filter_condition())
-		.into_model::<Event>()
-		.one(ctx.db())
-		.await?
-		.ok_or_else(UpubError::not_found)?;
-
-	let object = match item {
-		Event::Tombstone => return Err(UpubError::not_found()),
-		Event::Activity(_) => return Err(UpubError::not_found()),
-		Event::StrayObject { liked: _, object } => object,
-		Event::DeepActivity { activity: _, liked: _, object } => object,
-	};
-
-	let attachments = object.find_related(model::attachment::Entity)
-		.all(ctx.db())
-		.await?
-		.into_iter()
-		.map(|x| x.ap())
-		.collect::<Vec<serde_json::Value>>();
-
-	let mut replies = apb::Node::Empty;
-	
-	if ctx.cfg().security.show_reply_ids {
-		let replies_ids = model::addressing::Entity::find_addressed(None)
-			.filter(model::object::Column::InReplyTo.eq(oid))
-			.filter(auth.filter_condition())
-			.select_only()
-			.select_column(model::object::Column::Id)
-			.into_tuple::<String>()
-			.all(ctx.db())
-			.await?;
-
-		replies = apb::Node::object(
-			serde_json::Value::new_object()
-				// .set_id(Some(&crate::url!(ctx, "/objects/{id}/replies")))
-				// .set_first(apb::Node::link(crate::url!(ctx, "/objects/{id}/replies/page")))
-				.set_collection_type(Some(apb::CollectionType::Collection))
-				.set_total_items(Some(object.replies as u64))
-				.set_items(apb::Node::links(replies_ids))
-		);
-	}
-	
-	Ok(JsonLD(
-		object.ap()
-			.set_attachment(apb::Node::array(attachments))
-			.set_replies(replies)
-			.ld_context()
-	))
-}

src/routes/activitypub/object/replies.rs

@@ -1,48 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, Condition, PaginatorTrait, QueryFilter};
-
-use crate::{model, routes::activitypub::{JsonLD, Pagination, TryFetch}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(q): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let replies_id = url!(ctx, "/objects/{id}/replies");
-	let oid = ctx.oid(&id);
-
-	if auth.is_local() && q.fetch {
-		ctx.fetch_thread(&oid).await?;
-	}
-
-	let count = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(auth.filter_condition())
-		.filter(model::object::Column::InReplyTo.eq(oid))
-		.count(ctx.db())
-		.await?;
-
-	crate::server::builders::collection(&replies_id, Some(count))
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let page_id = url!(ctx, "/objects/{id}/replies/page");
-	let oid = ctx.oid(&id);
-
-	crate::server::builders::paginate(
-		page_id,
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(model::object::Column::InReplyTo.eq(oid)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		false,
-	)
-		.await
-}

src/routes/activitypub/outbox.rs

@@ -1,35 +0,0 @@
-use axum::{extract::{Query, State}, http::StatusCode, Json};
-use sea_orm::{ColumnTrait, Condition};
-
-use crate::{errors::UpubError, routes::activitypub::{CreationResult, JsonLD, Pagination}, server::{auth::AuthIdentity, Context}, url};
-
-pub async fn get(State(ctx): State<Context>) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/outbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::paginate(
-		url!(ctx, "/outbox/page"),
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(crate::model::actor::Column::Domain.eq(ctx.domain().to_string())),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		true,
-	)
-		.await
-}
-
-pub async fn post(
-	State(_ctx): State<Context>,
-	AuthIdentity(_auth): AuthIdentity,
-	Json(_activity): Json<serde_json::Value>,
-) -> Result<CreationResult, UpubError> {
-	// TODO administrative actions may be carried out against this outbox?
-	Err(StatusCode::NOT_IMPLEMENTED.into())
-}

src/routes/activitypub/user/following.rs

@@ -1,47 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{routes::activitypub::{JsonLD, Pagination}, model, server::Context, url};
-
-use model::relation::Column::{Following, Follower};
-
-pub async fn get<const OUTGOING: bool>(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let follow___ = if OUTGOING { "following" } else { "followers" };
-	let count = model::relation::Entity::find()
-		.filter(if OUTGOING { Follower } else { Following }.eq(ctx.uid(&id)))
-		.count(ctx.db()).await.unwrap_or_else(|e| {
-			tracing::error!("failed counting {follow___} for {id}: {e}");
-			0
-		});
-
-	crate::server::builders::collection(&url!(ctx, "/actors/{id}/{follow___}"), Some(count))
-}
-
-pub async fn page<const OUTGOING: bool>(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let follow___ = if OUTGOING { "following" } else { "followers" };
-	let limit = page.batch.unwrap_or(20).min(50);
-	let offset = page.offset.unwrap_or(0);
-
-	let following = model::relation::Entity::find()
-		.filter(if OUTGOING { Follower } else { Following }.eq(ctx.uid(&id)))
-		.select_only()
-		.select_column(if OUTGOING { Following } else { Follower })
-		.limit(limit)
-		.offset(page.offset.unwrap_or(0))
-		.into_tuple::<String>()
-		.all(ctx.db())
-		.await?;
-
-	crate::server::builders::collection_page(
-		&url!(ctx, "/actors/{id}/{follow___}/page"),
-		offset, limit,
-		following.into_iter().map(serde_json::Value::String).collect()
-	)
-}

src/routes/activitypub/user/inbox.rs

@@ -1,58 +0,0 @@
-use axum::{extract::{Path, Query, State}, http::StatusCode, Json};
-
-use sea_orm::{ColumnTrait, Condition};
-use crate::{errors::UpubError, model, routes::activitypub::{JsonLD, Pagination}, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	match auth {
-		Identity::Anonymous => Err(StatusCode::FORBIDDEN.into()),
-		Identity::Remote { .. } => Err(StatusCode::FORBIDDEN.into()),
-		Identity::Local { id: user, .. } => if ctx.uid(&id) == user {
-			crate::server::builders::collection(&url!(ctx, "/actors/{id}/inbox"), None)
-		} else {
-			Err(StatusCode::FORBIDDEN.into())
-		},
-	}
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let Identity::Local { id: uid, internal } = &auth else {
-		// local inbox is only for local users
-		return Err(UpubError::forbidden());
-	};
-	if uid != &ctx.uid(&id) {
-		return Err(UpubError::forbidden());
-	}
-
-	crate::server::builders::paginate(
-		url!(ctx, "/actors/{id}/inbox/page"),
-		Condition::any()
-			.add(model::addressing::Column::Actor.eq(*internal))
-			.add(model::object::Column::AttributedTo.eq(uid))
-			.add(model::activity::Column::Actor.eq(uid)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		false,
-	)
-		.await
-}
-
-pub async fn post(
-	State(ctx): State<Context>,
-	Path(_id): Path<String>,
-	AuthIdentity(_auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>,
-) -> Result<(), UpubError> {
-	// POSTing to user inboxes is effectively the same as POSTing to the main inbox
-	super::super::inbox::post(State(ctx), AuthIdentity(_auth), Json(activity)).await
-}

src/routes/activitypub/user/outbox.rs

@@ -1,90 +0,0 @@
-use axum::{extract::{Path, Query, State}, http::StatusCode, Json};
-use sea_orm::{ColumnTrait, Condition};
-
-use apb::{server::Outbox, AcceptType, ActivityType, Base, BaseType, ObjectType, RejectType};
-use crate::{errors::UpubError, model, routes::activitypub::{CreationResult, JsonLD, Pagination}, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/actors/{id}/outbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let uid = ctx.uid(&id);
-	crate::server::builders::paginate(
-		url!(ctx, "/actors/{id}/outbox/page"),
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(
-				Condition::any()
-					.add(model::activity::Column::Actor.eq(&uid))
-					.add(model::object::Column::AttributedTo.eq(&uid))
-				),
-		ctx.db(),
-		page,
-		auth.my_id(),
-		false,
-	)
-		.await
-}
-
-pub async fn post(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>,
-) -> Result<CreationResult, UpubError> {
-	match auth {
-		Identity::Anonymous => Err(StatusCode::UNAUTHORIZED.into()),
-		Identity::Remote { .. } => Err(StatusCode::NOT_IMPLEMENTED.into()),
-		Identity::Local { id: uid, .. } => if ctx.uid(&id) == uid {
-			tracing::debug!("processing new local activity: {}", serde_json::to_string(&activity).unwrap_or_default());
-			match activity.base_type() {
-				None => Err(StatusCode::BAD_REQUEST.into()),
-
-				Some(BaseType::Link(_)) => Err(StatusCode::UNPROCESSABLE_ENTITY.into()),
-
-				Some(BaseType::Object(ObjectType::Note)) =>
-					Ok(CreationResult(ctx.create_note(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Create))) =>
-					Ok(CreationResult(ctx.create(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Like))) =>
-					Ok(CreationResult(ctx.like(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Follow))) =>
-					Ok(CreationResult(ctx.follow(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Announce))) =>
-					Ok(CreationResult(ctx.announce(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Accept(AcceptType::Accept)))) =>
-					Ok(CreationResult(ctx.accept(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Reject(RejectType::Reject)))) =>
-					Ok(CreationResult(ctx.reject(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Undo))) =>
-					Ok(CreationResult(ctx.undo(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Delete))) =>
-					Ok(CreationResult(ctx.delete(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Update))) =>
-					Ok(CreationResult(ctx.update(uid, activity).await?)),
-
-				Some(_) => Err(StatusCode::NOT_IMPLEMENTED.into()),
-			}
-		} else {
-			Err(StatusCode::FORBIDDEN.into())
-		}
-	}
-}

src/routes/mod.rs

@@ -1,16 +0,0 @@
-pub mod activitypub;
-
-#[cfg(feature = "web")]
-pub mod web;
-
-#[cfg(feature = "mastodon")]
-pub mod mastodon;
-
-#[cfg(not(feature = "mastodon"))]
-pub mod mastodon {
-	pub trait MastodonRouter {
-		fn mastodon_routes(self) -> Self where Self: Sized { self }
-	}
-	
-	impl MastodonRouter for axum::Router<crate::server::Context> {}
-}

src/server/addresser.rs

@@ -1,130 +0,0 @@
-use sea_orm::{ActiveValue::{NotSet, Set}, EntityTrait};
-
-use crate::model;
-
-use super::{fetcher::Fetcher, Context};
-
-
-#[axum::async_trait]
-pub trait Addresser {
-	async fn expand_addressing(&self, targets: Vec<String>) -> crate::Result<Vec<String>>;
-	async fn address_to(&self, aid: Option<i64>, oid: Option<i64>, targets: &[String]) -> crate::Result<()>;
-	async fn deliver_to(&self, aid: &str, from: &str, targets: &[String]) -> crate::Result<()>;
-	//#[deprecated = "should probably directly invoke address_to() since we most likely have internal ids at this point"]
-	async fn dispatch(&self, uid: &str, activity_targets: Vec<String>, aid: &str, oid: Option<&str>) -> crate::Result<()>;
-}
-
-#[axum::async_trait]
-impl Addresser for super::Context {
-	async fn expand_addressing(&self, targets: Vec<String>) -> crate::Result<Vec<String>> {
-		let mut out = Vec::new();
-		for target in targets {
-			if target.ends_with("/followers") {
-				let target_id = target.replace("/followers", "");
-				let mut followers = model::relation::Entity::followers(&target_id, self.db()).await?;
-				if followers.is_empty() { // stuff with zero addressing will never be seen again!!! TODO
-					followers.push(target_id);
-				}
-				for follower in followers {
-					out.push(follower);
-				}
-			} else {
-				out.push(target);
-			}
-		}
-		Ok(out)
-	}
-
-	async fn address_to(&self, aid: Option<i64>, oid: Option<i64>, targets: &[String]) -> crate::Result<()> {
-		// TODO address_to became kind of expensive, with these two selects right away and then another
-		//      select for each target we're addressing to... can this be improved??
-		let local_activity = if let Some(x) = aid { self.is_local_internal_activity(x).await.unwrap_or(false) } else { false };
-		let local_object = if let Some(x) = oid { self.is_local_internal_object(x).await.unwrap_or(false) } else { false };
-		let mut addressing = Vec::new();
-		for target in targets
-			.iter()
-			.filter(|to| !to.is_empty())
-			.filter(|to| !to.ends_with("/followers"))
-			.filter(|to| local_activity || local_object || to.as_str() == apb::target::PUBLIC || self.is_local(to))
-		{
-			let (server, actor) = if target == apb::target::PUBLIC { (None, None) } else {
-				match (
-					model::instance::Entity::domain_to_internal(&Context::server(target), self.db()).await,
-					model::actor::Entity::ap_to_internal(target, self.db()).await,
-				) {
-					(Ok(server), Ok(actor)) => (Some(server), Some(actor)),
-					(Err(e), Ok(_)) => { tracing::error!("failed resolving domain: {e}"); continue; },
-					(Ok(_), Err(e)) => { tracing::error!("failed resolving actor: {e}"); continue; },
-					(Err(es), Err(ea)) => { tracing::error!("failed resolving domain ({es}) and actor ({ea})"); continue; },
-				}
-			};
-			addressing.push(
-				model::addressing::ActiveModel {
-					internal: NotSet,
-					instance: Set(server),
-					actor: Set(actor),
-					activity: Set(aid),
-					object: Set(oid),
-					published: Set(chrono::Utc::now()),
-				}
-			);
-		}
-
-		if !addressing.is_empty() {
-			model::addressing::Entity::insert_many(addressing)
-				.exec(self.db())
-				.await?;
-		}
-
-		Ok(())
-	}
-
-	async fn deliver_to(&self, aid: &str, from: &str, targets: &[String]) -> crate::Result<()> {
-		let mut deliveries = Vec::new();
-		for target in targets.iter()
-			.filter(|to| !to.is_empty())
-			.filter(|to| Context::server(to) != self.domain())
-			.filter(|to| to != &apb::target::PUBLIC)
-		{
-			// TODO fetch concurrently
-			match self.fetch_user(target).await {
-				Ok(model::actor::Model { inbox: Some(inbox), .. }) => deliveries.push(
-					model::delivery::ActiveModel {
-						internal: sea_orm::ActiveValue::NotSet,
-						actor: Set(from.to_string()),
-						// TODO we should resolve each user by id and check its inbox because we can't assume
-						// it's /actors/{id}/inbox for every software, but oh well it's waaaaay easier now
-						target: Set(inbox),
-						activity: Set(aid.to_string()),
-						published: Set(chrono::Utc::now()),
-						not_before: Set(chrono::Utc::now()),
-						attempt: Set(0),
-					}
-				),
-				Ok(_) => tracing::error!("resolved target but missing inbox: '{target}', skipping delivery"),
-				Err(e) => tracing::error!("failed resolving target inbox: {e}, skipping delivery to '{target}'"),
-			}
-		}
-
-		if !deliveries.is_empty() {
-			model::delivery::Entity::insert_many(deliveries)
-				.exec(self.db())
-				.await?;
-		}
-
-		self.dispatcher().wakeup();
-
-		Ok(())
-	}
-
-	//#[deprecated = "should probably directly invoke address_to() since we most likely have internal ids at this point"]
-	async fn dispatch(&self, uid: &str, activity_targets: Vec<String>, aid: &str, oid: Option<&str>) -> crate::Result<()> {
-		let addressed = self.expand_addressing(activity_targets).await?;
-		let internal_aid = model::activity::Entity::ap_to_internal(aid, self.db()).await?;
-		let internal_oid = if let Some(o) = oid { Some(model::object::Entity::ap_to_internal(o, self.db()).await?) } else { None };
-		self.address_to(Some(internal_aid), internal_oid, &addressed).await?;
-		self.deliver_to(aid, uid, &addressed).await?;
-		Ok(())
-	}
-
-}

src/server/auth.rs

@@ -1,152 +0,0 @@
-use axum::{extract::{FromRef, FromRequestParts}, http::{header, request::Parts}};
-use reqwest::StatusCode;
-use sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter};
-
-use crate::{errors::UpubError, model, server::Context};
-
-use super::{fetcher::Fetcher, httpsign::HttpSignature};
-
-#[derive(Debug, Clone)]
-pub enum Identity {
-	Anonymous,
-	Remote {
-		domain: String,
-		internal: i64,
-	},
-	Local {
-		id: String,
-		internal: i64,
-	},
-}
-
-impl Identity {
-	pub fn filter_condition(&self) -> Condition {
-		let base_cond = Condition::any().add(model::addressing::Column::Actor.is_null());
-		match self {
-			Identity::Anonymous => base_cond,
-			Identity::Remote { internal, .. } => base_cond.add(model::addressing::Column::Instance.eq(*internal)),
-			// TODO should we allow all users on same server to see? or just specific user??
-			Identity::Local { id, internal } => base_cond
-				.add(model::addressing::Column::Actor.eq(*internal))
-				.add(model::activity::Column::Actor.eq(id))
-				.add(model::object::Column::AttributedTo.eq(id)),
-		}
-	}
-
-	pub fn my_id(&self) -> Option<i64> {
-		match self {
-			Identity::Local { internal, .. } => Some(*internal),
-			_ => None,
-		}
-	}
-
-	pub fn is(&self, uid: &str) -> bool {
-		match self {
-			Identity::Anonymous => false,
-			Identity::Remote { .. } => false, // TODO per-actor server auth should check this
-			Identity::Local { id, .. } => id.as_str() == uid
-		}
-	}
-
-	#[allow(unused)]
-	pub fn is_anon(&self) -> bool {
-		matches!(self, Self::Anonymous)
-	}
-
-	#[allow(unused)]
-	pub fn is_local(&self) -> bool {
-		matches!(self, Self::Local { .. })
-	}
-
-	#[allow(unused)]
-	pub fn is_remote(&self) -> bool {
-		matches!(self, Self::Remote { .. })
-	}
-}
-
-pub struct AuthIdentity(pub Identity);
-
-#[axum::async_trait]
-impl<S> FromRequestParts<S> for AuthIdentity
-where
-	Context: FromRef<S>,
-	S: Send + Sync,
-{
-	type Rejection = UpubError;
-
-	async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
-		let ctx = Context::from_ref(state);
-		let mut identity = Identity::Anonymous;
-
-		let auth_header = parts
-			.headers
-			.get(header::AUTHORIZATION)
-			.map(|v| v.to_str().unwrap_or(""))
-			.unwrap_or("");
-
-		if auth_header.starts_with("Bearer ") {
-			match model::session::Entity::find()
-				.filter(model::session::Column::Secret.eq(auth_header.replace("Bearer ", "")))
-				.filter(model::session::Column::Expires.gt(chrono::Utc::now()))
-				.one(ctx.db())
-				.await
-			{
-				Ok(None) => return Err(UpubError::unauthorized()),
-				Ok(Some(x)) => {
-					// TODO could we store both actor ap id and internal id in session? to avoid this extra
-					// lookup on *every* local authed request we receive...
-					let internal = model::actor::Entity::ap_to_internal(&x.actor, ctx.db()).await?;
-					identity = Identity::Local { id: x.actor, internal };
-				},
-				Err(e) => {
-					tracing::error!("failed querying user session: {e}");
-					return Err(UpubError::internal_server_error())
-				},
-			}
-		}
-
-		if let Some(sig) = parts
-			.headers
-			.get("Signature")
-			.map(|v| v.to_str().unwrap_or(""))
-		{
-			let mut http_signature = HttpSignature::parse(sig);
-
-			// TODO assert payload's digest is equal to signature's
-			let user_id = http_signature.key_id
-				.split('#')
-				.next().ok_or(UpubError::bad_request())?
-				.to_string();
-
-			match ctx.fetch_user(&user_id).await {
-				Ok(user) => match http_signature
-						.build_from_parts(parts)
-						.verify(&user.public_key)
-					{
-						Ok(true) => {
-							// TODO can we avoid this extra db rountrip made on each server fetch?
-							let domain = Context::server(&user_id); 
-							// TODO this will fail because we never fetch and insert into instance oops
-							let internal = model::instance::Entity::domain_to_internal(&domain, ctx.db()).await?;
-							identity = Identity::Remote { domain, internal };
-						},
-						Ok(false) => tracing::warn!("invalid signature: {http_signature:?}"),
-						Err(e) => tracing::error!("error verifying signature: {e}"),
-					},
-				Err(e) => {
-					// since most activities are deletions for users we never saw, let's handle this case
-					// if while fetching we receive a GONE, it means we didn't have this user and it doesn't
-					// exist anymore, so it must be a deletion we can ignore
-					if let UpubError::Reqwest(ref x) = e {
-						if let Some(StatusCode::GONE) = x.status() {
-							return Err(UpubError::Status(StatusCode::OK)); // 200 so mastodon will shut uppp
-						}
-					}
-					tracing::warn!("could not fetch user (won't verify): {e}");
-				}
-			}
-		}
-
-		Ok(AuthIdentity(identity))
-	}
-}

src/server/builders.rs

@@ -1,92 +0,0 @@
-use apb::{BaseMut, CollectionMut, CollectionPageMut};
-use sea_orm::{Condition, DatabaseConnection, QueryFilter, QuerySelect, RelationTrait};
-
-use crate::{model::{addressing::Event, attachment::BatchFillable}, routes::activitypub::{jsonld::LD, JsonLD, Pagination}};
-
-pub async fn paginate(
-	id: String,
-	filter: Condition,
-	db: &DatabaseConnection,
-	page: Pagination,
-	my_id: Option<i64>,
-	with_users: bool, // TODO ewww too many arguments for this weird function...
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let limit = page.batch.unwrap_or(20).min(50);
-	let offset = page.offset.unwrap_or(0);
-
-	let mut select = crate::model::addressing::Entity::find_addressed(my_id);
-
-	if with_users {
-		select = select
-			.join(sea_orm::JoinType::InnerJoin, crate::model::activity::Relation::Actors.def());
-	}
-
-	let items = select
-		.filter(filter)
-		// TODO also limit to only local activities
-		.limit(limit)
-		.offset(offset)
-		.into_model::<Event>()
-		.all(db)
-		.await?;
-
-	let mut attachments = items.load_attachments_batch(db).await?;
-
-	let items : Vec<serde_json::Value> = items
-		.into_iter()
-		.map(|item| {
-			let attach = attachments.remove(&item.internal());
-			item.ap(attach)
-		})
-		.collect();
-
-	collection_page(&id, offset, limit, items)
-}
-
-pub fn collection_page(id: &str, offset: u64, limit: u64, items: Vec<serde_json::Value>) -> crate::Result<JsonLD<serde_json::Value>> {
-	let next = if items.len() < limit as usize {
-		apb::Node::Empty
-	} else {
-		apb::Node::link(format!("{id}?offset={}", offset+limit))
-	};
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(&format!("{id}?offset={offset}")))
-			.set_collection_type(Some(apb::CollectionType::OrderedCollectionPage))
-			.set_part_of(apb::Node::link(id.replace("/page", "")))
-			.set_ordered_items(apb::Node::array(items))
-			.set_next(next)
-			.ld_context()
-	))
-}
-
-
-pub fn collection(id: &str, total_items: Option<u64>) -> crate::Result<JsonLD<serde_json::Value>> {
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(id))
-			.set_collection_type(Some(apb::CollectionType::OrderedCollection))
-			.set_first(apb::Node::link(format!("{id}/page")))
-			.set_total_items(total_items)
-			.ld_context()
-	))
-}
-
-#[axum::async_trait]
-pub trait AnyQuery {
-	async fn any(self, db: &sea_orm::DatabaseConnection) -> crate::Result<bool>;
-}
-
-#[axum::async_trait]
-impl<T : sea_orm::EntityTrait> AnyQuery for sea_orm::Select<T> {
-	async fn any(self, db: &sea_orm::DatabaseConnection) ->	crate::Result<bool> {
-		Ok(self.one(db).await?.is_some())
-	}
-}
-
-#[axum::async_trait]
-impl<T : sea_orm::SelectorTrait + Send> AnyQuery for sea_orm::Selector<T> {
-	async fn any(self, db: &sea_orm::DatabaseConnection) ->	crate::Result<bool> {
-		Ok(self.one(db).await?.is_some())
-	}
-}

src/server/dispatcher.rs

@@ -1,134 +0,0 @@
-use reqwest::Method;
-use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, Order, QueryFilter, QueryOrder};
-use tokio::{sync::broadcast, task::JoinHandle};
-
-use apb::{ActivityMut, Node};
-use crate::{model, routes::activitypub::jsonld::LD, server::{fetcher::Fetcher, Context}};
-
-pub struct Dispatcher {
-	waker: broadcast::Sender<()>,
-}
-
-impl Default for Dispatcher {
-	fn default() -> Self {
-		let (waker, _) = broadcast::channel(1);
-		Dispatcher { waker }
-	}
-}
-
-impl Dispatcher {
-	pub fn spawn(&self, db: DatabaseConnection, domain: String, poll_interval: u64) -> JoinHandle<()> {
-		let mut waker = self.waker.subscribe();
-		tokio::spawn(async move {
-			loop {
-				if let Err(e) = worker(&db, &domain, poll_interval, &mut waker).await {
-					tracing::error!("delivery worker exited with error: {e}");
-				}
-				tokio::time::sleep(std::time::Duration::from_secs(poll_interval * 10)).await;
-			}
-		})
-	}
-
-	pub fn wakeup(&self) {
-		match self.waker.send(()) {
-			Err(_) => tracing::error!("no worker to wakeup"), 
-			Ok(n) => tracing::debug!("woken {n} workers"),
-		}
-	}
-}
-
-async fn worker(db: &DatabaseConnection, domain: &str, poll_interval: u64, waker: &mut broadcast::Receiver<()>) -> crate::Result<()> {
-	loop {
-		let Some(delivery) = model::delivery::Entity::find()
-			.filter(model::delivery::Column::NotBefore.lte(chrono::Utc::now()))
-			.order_by(model::delivery::Column::NotBefore, Order::Asc)
-			.one(db)
-			.await?
-		else {
-			tokio::select! {
-				biased;
-				_ = waker.recv() => {},
-				_ = tokio::time::sleep(std::time::Duration::from_secs(poll_interval)) => {},
-			}
-			continue
-		};
-
-		let del_row = model::delivery::ActiveModel {
-			internal: sea_orm::ActiveValue::Set(delivery.internal),
-			..Default::default()
-		};
-		let del = model::delivery::Entity::delete(del_row)
-			.exec(db)
-			.await?;
-
-		if del.rows_affected == 0 {
-			// another worker claimed this delivery
-			continue; // go back to the top
-		}
-		if delivery.expired() {
-			// try polling for another one
-			continue; // go back to top
-		}
-
-		tracing::info!("delivering {} to {}", delivery.activity, delivery.target);
-
-		let payload = match model::activity::Entity::find_by_ap_id(&delivery.activity)
-			.find_also_related(model::object::Entity)
-			.one(db)
-			.await? // TODO probably should not fail here and at least re-insert the delivery
-		{
-			Some((activity, None)) => activity.ap().ld_context(),
-			Some((activity, Some(object))) => {
-				let always_embed = matches!(
-					activity.activity_type,
-					apb::ActivityType::Create
-					| apb::ActivityType::Undo
-					| apb::ActivityType::Update
-					| apb::ActivityType::Accept(_)
-					| apb::ActivityType::Reject(_)
-				);
-				if always_embed {
-					activity.ap().set_object(Node::object(object.ap())).ld_context()
-				} else {
-					activity.ap().ld_context()
-				}
-			},
-			None => {
-				tracing::warn!("skipping dispatch for deleted object {}", delivery.activity);
-				continue;
-			},
-		};
-
-		let Some(actor) = model::actor::Entity::find_by_ap_id(&delivery.actor)
-			.one(db)
-			.await?
-		else {
-			tracing::error!("abandoning delivery of {} from non existant actor: {}", delivery.activity, delivery.actor);
-			continue;
-		};
-
-		let Some(key) = actor.private_key
-		else {
-			tracing::error!("abandoning delivery of {} from actor without private key: {}", delivery.activity, delivery.actor);
-			continue;
-		};
-
-		if let Err(e) = Context::request(
-			Method::POST, &delivery.target,
-			Some(&serde_json::to_string(&payload).unwrap()),
-			&delivery.actor, &key, domain
-		).await {
-			tracing::warn!("failed delivery of {} to {} : {e}", delivery.activity, delivery.target);
-			let new_delivery = model::delivery::ActiveModel {
-				internal: sea_orm::ActiveValue::NotSet,
-				not_before: sea_orm::ActiveValue::Set(delivery.next_delivery()),
-				actor: sea_orm::ActiveValue::Set(delivery.actor),
-				target: sea_orm::ActiveValue::Set(delivery.target),
-				activity: sea_orm::ActiveValue::Set(delivery.activity),
-				published: sea_orm::ActiveValue::Set(delivery.published),
-				attempt: sea_orm::ActiveValue::Set(delivery.attempt + 1),
-			};
-			model::delivery::Entity::insert(new_delivery).exec(db).await?;
-		}
-	}
-}

src/server/fetcher.rs

@@ -1,453 +0,0 @@
-use std::collections::BTreeMap;
-
-use apb::{target::Addressed, Activity, Actor, ActorMut, Base, Collection, Object};
-use base64::Engine;
-use reqwest::{header::{ACCEPT, CONTENT_TYPE, USER_AGENT}, Method, Response};
-use sea_orm::{EntityTrait, IntoActiveModel, NotSet};
-
-use crate::{errors::UpubError, model, VERSION};
-
-use super::{addresser::Addresser, httpsign::HttpSignature, normalizer::Normalizer, Context};
-
-#[derive(Debug, Clone)]
-pub enum PullResult<T> {
-	Actor(T),
-	Activity(T),
-	Object(T),
-}
-
-impl PullResult<serde_json::Value> {
-	pub fn actor(self) -> crate::Result<serde_json::Value> {
-		match self {
-			Self::Actor(x) => Ok(x),
-			Self::Activity(x) => Err(UpubError::Mismatch(apb::ObjectType::Actor(apb::ActorType::Person), x.object_type().unwrap_or(apb::ObjectType::Activity(apb::ActivityType::Activity)))),
-			Self::Object(x) => Err(UpubError::Mismatch(apb::ObjectType::Actor(apb::ActorType::Person), x.object_type().unwrap_or(apb::ObjectType::Object))),
-		}
-	}
-
-	pub fn activity(self) -> crate::Result<serde_json::Value> {
-		match self {
-			Self::Actor(x) => Err(UpubError::Mismatch(apb::ObjectType::Activity(apb::ActivityType::Activity), x.object_type().unwrap_or(apb::ObjectType::Actor(apb::ActorType::Person)))),
-			Self::Activity(x) => Ok(x),
-			Self::Object(x) => Err(UpubError::Mismatch(apb::ObjectType::Activity(apb::ActivityType::Activity), x.object_type().unwrap_or(apb::ObjectType::Object))),
-		}
-	}
-
-	pub fn object(self) -> crate::Result<serde_json::Value> {
-		match self {
-			Self::Actor(x) => Err(UpubError::Mismatch(apb::ObjectType::Object, x.object_type().unwrap_or(apb::ObjectType::Actor(apb::ActorType::Person)))),
-			Self::Activity(x) => Err(UpubError::Mismatch(apb::ObjectType::Object, x.object_type().unwrap_or(apb::ObjectType::Activity(apb::ActivityType::Activity)))),
-			Self::Object(x) => Ok(x),
-		}
-	}
-}
-
-#[axum::async_trait]
-pub trait Fetcher {
-	async fn pull(&self, id: &str) -> crate::Result<PullResult<serde_json::Value>> { self.pull_r(id, 0).await }
-	async fn pull_r(&self, id: &str, depth: u32) -> crate::Result<PullResult<serde_json::Value>>;
-
-
-	async fn webfinger(&self, user: &str, host: &str) -> crate::Result<String>;
-
-	async fn fetch_domain(&self, domain: &str) -> crate::Result<model::instance::Model>;
-
-	async fn fetch_user(&self, id: &str) -> crate::Result<model::actor::Model>;
-	async fn resolve_user(&self, actor: serde_json::Value) -> crate::Result<model::actor::Model>;
-
-	async fn fetch_activity(&self, id: &str) -> crate::Result<model::activity::Model>;
-	async fn resolve_activity(&self, activity: serde_json::Value) -> crate::Result<model::activity::Model>;
-
-	async fn fetch_object(&self, id: &str) -> crate::Result<model::object::Model> { self.fetch_object_r(id, 0).await }
-	#[allow(unused)] async fn resolve_object(&self, object: serde_json::Value) -> crate::Result<model::object::Model> { self.resolve_object_r(object, 0).await }
-
-	async fn fetch_object_r(&self, id: &str, depth: u32) -> crate::Result<model::object::Model>;
-	async fn resolve_object_r(&self, object: serde_json::Value, depth: u32) -> crate::Result<model::object::Model>;
-
-
-	async fn fetch_thread(&self, id: &str) -> crate::Result<()>;
-
-	async fn request(
-		method: reqwest::Method,
-		url: &str,
-		payload: Option<&str>,
-		from: &str,
-		key: &str,
-		domain: &str,
-	) -> crate::Result<Response> {
-		let host = Context::server(url);
-		let date = chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string(); // lmao @ "GMT"
-		let path = url.replace("https://", "").replace("http://", "").replace(&host, "");
-		let digest = format!("sha-256={}",
-			base64::prelude::BASE64_STANDARD.encode(
-				openssl::sha::sha256(payload.unwrap_or("").as_bytes())
-			)
-		);
-
-		let headers = vec!["(request-target)", "host", "date", "digest"];
-		let headers_map : BTreeMap<String, String> = [
-			("host".to_string(), host.clone()),
-			("date".to_string(), date.clone()),
-			("digest".to_string(), digest.clone()),
-		].into();
-
-		let mut signer = HttpSignature::new(
-			format!("{from}#main-key"), // TODO don't hardcode #main-key
-			//"hs2019".to_string(), // pixelfeed/iceshrimp made me go back
-			"rsa-sha256".to_string(),
-			&headers,
-		);
-		
-		signer
-			.build_manually(&method.to_string().to_lowercase(), &path, headers_map)
-			.sign(key)?;
-
-		let response = reqwest::Client::new()
-			.request(method.clone(), url)
-			.header(ACCEPT, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")
-			.header(CONTENT_TYPE, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")
-			.header(USER_AGENT, format!("upub+{VERSION} ({domain})"))
-			.header("Host", host.clone())
-			.header("Date", date.clone())
-			.header("Digest", digest)
-			.header("Signature", signer.header())
-			.body(payload.unwrap_or("").to_string())
-			.send()
-			.await?;
-
-		// TODO this is ugly but i want to see the raw response text when it's a failure
-		match response.error_for_status_ref() {
-			Ok(_) => Ok(response),
-			Err(e) => Err(UpubError::FetchError(e, response.text().await?)),
-		}
-	}
-}
-
-
-#[axum::async_trait]
-impl Fetcher for Context {
-	async fn pull_r(&self, id: &str, depth: u32) -> crate::Result<PullResult<serde_json::Value>> {
-		let _domain = self.fetch_domain(&Context::server(id)).await?;
-
-		let document = Self::request(
-			Method::GET, id, None,
-			&format!("https://{}", self.domain()), self.pkey(), self.domain(),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?;
-
-		let doc_id = document.id().ok_or_else(|| UpubError::field("id"))?;
-		if id != doc_id {
-			if depth >= self.cfg().security.max_id_redirects {
-				return Err(UpubError::unprocessable());
-			}
-			return self.pull(doc_id).await;
-		}
-
-		match document.object_type() {
-			None => Err(UpubError::bad_request()),
-			Some(apb::ObjectType::Collection(_)) => Err(UpubError::unprocessable()),
-			Some(apb::ObjectType::Tombstone) => Err(UpubError::not_found()),
-			Some(apb::ObjectType::Activity(_)) => Ok(PullResult::Activity(document)),
-			Some(apb::ObjectType::Actor(_)) => Ok(PullResult::Actor(document)),
-			_ => Ok(PullResult::Object(document)),
-		}
-	}
-
-
-	async fn webfinger(&self, user: &str, host: &str) -> crate::Result<String> {
-		let subject = format!("acct:{user}@{host}");
-		let webfinger_uri = format!("https://{host}/.well-known/webfinger?resource={subject}");
-		let resource = reqwest::Client::new()
-			.get(webfinger_uri)
-			.header(ACCEPT, "application/jrd+json")
-			.header(USER_AGENT, format!("upub+{VERSION} ({})", self.domain()))
-			.send()
-			.await?
-			.json::<jrd::JsonResourceDescriptor>()
-			.await?;
-
-		if resource.subject != subject {
-			return Err(UpubError::unprocessable());
-		}
-
-		for link in resource.links {
-			if link.rel == "self" {
-				if let Some(href) = link.href {
-					return Ok(href);
-				}
-			}
-		}
-
-		if let Some(alias) = resource.aliases.into_iter().next() {
-			return Ok(alias);
-		}
-
-		Err(UpubError::not_found())
-	}
-
-	async fn fetch_domain(&self, domain: &str) -> crate::Result<model::instance::Model> {
-		if let Some(x) = model::instance::Entity::find_by_domain(domain).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		let mut instance_model = model::instance::Model {
-			internal: 0,
-			domain: domain.to_string(),
-			name: None,
-			software: None,
-			down_since: None,
-			icon: None,
-			version: None,
-			users: None,
-			posts: None,
-			published: chrono::Utc::now(),
-			updated: chrono::Utc::now(),
-		};
-
-		if let Ok(res) = Self::request(
-			Method::GET, &format!("https://{domain}"), None, &format!("https://{}", self.domain()), self.pkey(), self.domain(),
-		).await {
-			if let Ok(actor) = res.json::<serde_json::Value>().await {
-				if let Some(name) = actor.name() {
-					instance_model.name = Some(name.to_string());
-				}
-				if let Some(icon) = actor.icon().id() {
-					instance_model.icon = Some(icon);
-				}
-			}
-		}
-
-		if let Ok(nodeinfo) = model::instance::Entity::nodeinfo(domain).await {
-			instance_model.software = Some(nodeinfo.software.name);
-			instance_model.version = nodeinfo.software.version;
-			instance_model.users = nodeinfo.usage.users.and_then(|x| x.total);
-			instance_model.posts = nodeinfo.usage.local_posts;
-		}
-
-		let mut active_model = instance_model.clone().into_active_model();
-		active_model.internal = NotSet;
-		model::instance::Entity::insert(active_model).exec(self.db()).await?;
-
-		let internal = model::instance::Entity::domain_to_internal(domain, self.db()).await?;
-		instance_model.internal = internal;
-
-		Ok(instance_model)
-	}
-
-	async fn resolve_user(&self, mut document: serde_json::Value) -> crate::Result<model::actor::Model> {
-		let id = document.id().ok_or_else(|| UpubError::field("id"))?.to_string();
-
-		// TODO try fetching these numbers from audience/generator fields to avoid making 2 more GETs every time
-		if let Some(followers_url) = &document.followers().id() {
-			let req = Self::request(
-				Method::GET, followers_url, None,
-				&format!("https://{}", self.domain()), self.pkey(), self.domain(),
-			).await;
-			if let Ok(res) = req {
-				if let Ok(user_followers) = res.json::<serde_json::Value>().await {
-					if let Some(total) = user_followers.total_items() {
-						document = document.set_followers_count(Some(total));
-					}
-				}
-			}
-		}
-
-		if let Some(following_url) = &document.following().id() {
-			let req =  Self::request(
-				Method::GET, following_url, None,
-				&format!("https://{}", self.domain()), self.pkey(), self.domain(),
-			).await;
-			if let Ok(res) = req {
-				if let Ok(user_following) = res.json::<serde_json::Value>().await {
-					if let Some(total) = user_following.total_items() {
-						document = document.set_following_count(Some(total));
-					}
-				}
-			}
-		}
-
-		let user_model = model::actor::ActiveModel::new(&document)?;
-
-		// TODO this may fail: while fetching, remote server may fetch our service actor.
-		//      if it does so with http signature, we will fetch that actor in background
-		//      meaning that, once we reach here, it's already inserted and returns an UNIQUE error
-		model::actor::Entity::insert(user_model).exec(self.db()).await?;
-		
-		// TODO fetch it back to get the internal id
-		Ok(
-			model::actor::Entity::find_by_ap_id(&id)
-				.one(self.db())
-				.await?
-				.ok_or_else(UpubError::internal_server_error)?
-		)
-	}
-
-	async fn fetch_user(&self, id: &str) -> crate::Result<model::actor::Model> {
-		if let Some(x) = model::actor::Entity::find_by_ap_id(id).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		let document = self.pull(id).await?.actor()?;
-
-		self.resolve_user(document).await
-	}
-
-	async fn fetch_activity(&self, id: &str) -> crate::Result<model::activity::Model> {
-		if let Some(x) = model::activity::Entity::find_by_ap_id(id).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		let activity = self.pull(id).await?.activity()?;
-
-		self.resolve_activity(activity).await
-	}
-
-	async fn resolve_activity(&self, activity: serde_json::Value) -> crate::Result<model::activity::Model> {
-		let id = activity.id().ok_or_else(|| UpubError::field("id"))?.to_string();
-
-		if let Some(activity_actor) = activity.actor().id() {
-			if let Err(e) = self.fetch_user(&activity_actor).await {
-				tracing::warn!("could not get actor of fetched activity: {e}");
-			}
-		}
-
-		if let Some(activity_object) = activity.object().id() {
-			if let Err(e) = self.fetch_object(&activity_object).await {
-				tracing::warn!("could not get object of fetched activity: {e}");
-			}
-		}
-
-		let activity_model = self.insert_activity(activity, Some(Context::server(&id))).await?;
-
-		let addressed = activity_model.addressed();
-		let expanded_addresses = self.expand_addressing(addressed).await?;
-		self.address_to(Some(activity_model.internal), None, &expanded_addresses).await?;
-
-		Ok(activity_model)
-	}
-
-	async fn fetch_thread(&self, _id: &str) -> crate::Result<()> {
-		// crawl_replies(self, id, 0).await
-		todo!()
-	}
-
-	async fn fetch_object_r(&self, id: &str, depth: u32) -> crate::Result<model::object::Model> {
-		if let Some(x) = model::object::Entity::find_by_ap_id(id).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		let object = self.pull(id).await?.object()?;
-
-		self.resolve_object_r(object, depth).await
-	}
-
-	async fn resolve_object_r(&self, object: serde_json::Value, depth: u32) -> crate::Result<model::object::Model> {
-		let id = object.id().ok_or_else(|| UpubError::field("id"))?.to_string();
-
-		if let Some(oid) = object.id() {
-			if oid != id {
-				if let Some(x) = model::object::Entity::find_by_ap_id(oid).one(self.db()).await? {
-					return Ok(x); // already in db, but with id different that given url
-				}
-			}
-		}
-
-		if let Some(attributed_to) = object.attributed_to().id() {
-			if let Err(e) = self.fetch_user(&attributed_to).await {
-				tracing::warn!("could not get actor of fetched object: {e}");
-			}
-		}
-
-		let addressed = object.addressed();
-
-		if let Some(reply) = object.in_reply_to().id() {
-			if depth <= self.cfg().security.thread_crawl_depth {
-				self.fetch_object_r(&reply, depth + 1).await?;
-			} else {
-				tracing::warn!("thread deeper than {}, giving up fetching more replies", self.cfg().security.thread_crawl_depth);
-			}
-		}
-
-		let object_model = self.insert_object(object, None).await?;
-
-		let expanded_addresses = self.expand_addressing(addressed).await?;
-		self.address_to(None, Some(object_model.internal), &expanded_addresses).await?;
-
-		Ok(object_model)
-	}
-}
-
-#[axum::async_trait]
-pub trait Fetchable : Sync + Send {
-	async fn fetch(&mut self, ctx: &crate::server::Context) -> crate::Result<&mut Self>;
-}
-
-#[axum::async_trait]
-impl Fetchable for apb::Node<serde_json::Value> {
-	async fn fetch(&mut self, ctx: &crate::server::Context) -> crate::Result<&mut Self> {
-		if let apb::Node::Link(uri) = self {
-			*self = Context::request(Method::GET, uri.href(), None, ctx.base(), ctx.pkey(), ctx.domain())
-				.await?
-				.json::<serde_json::Value>()
-				.await?
-				.into();
-		}
-
-		Ok(self)
-	}
-}
-
-// #[async_recursion::async_recursion]
-// async fn crawl_replies(ctx: &Context, id: &str, depth: usize) -> crate::Result<()> {
-// 	tracing::info!("crawling replies of '{id}'");
-// 	let object = Context::request(
-// 		Method::GET, id, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 	).await?.json::<serde_json::Value>().await?;
-// 
-// 	let object_model = model::object::Model::new(&object)?;
-// 	match model::object::Entity::insert(object_model.into_active_model())
-// 		.exec(ctx.db()).await
-// 	{
-// 		Ok(_) => {},
-// 		Err(sea_orm::DbErr::RecordNotInserted) => {},
-// 		Err(sea_orm::DbErr::Exec(_)) => {}, // ughhh bad fix for sqlite
-// 		Err(e) => return Err(e.into()),
-// 	}
-// 
-// 	if depth > 16 {
-// 		tracing::warn!("stopping thread crawling: too deep!");
-// 		return Ok(());
-// 	}
-// 
-// 	let mut page_url = match object.replies().get() {
-// 		Some(serde_json::Value::String(x)) => {
-// 			let replies = Context::request(
-// 				Method::GET, x, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 			).await?.json::<serde_json::Value>().await?;
-// 			replies.first().id()
-// 		},
-// 		Some(serde_json::Value::Object(x)) => {
-// 			let obj = serde_json::Value::Object(x.clone()); // lol putting it back, TODO!
-// 			obj.first().id()
-// 		},
-// 		_ => return Ok(()),
-// 	};
-// 
-// 	while let Some(ref url) = page_url {
-// 		let replies = Context::request(
-// 			Method::GET, url, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 		).await?.json::<serde_json::Value>().await?;
-// 
-// 		for reply in replies.items() {
-// 			// TODO right now it crawls one by one, could be made in parallel but would be quite more
-// 			// abusive, so i'll keep it like this while i try it out
-// 			crawl_replies(ctx, reply.href(), depth + 1).await?;
-// 		}
-// 
-// 		page_url = replies.next().id();
-// 	}
-// 
-// 	Ok(())
-// }

src/server/inbox.rs

@@ -1,316 +0,0 @@
-use apb::{target::Addressed, Activity, Base, Object};
-use reqwest::StatusCode;
-use sea_orm::{sea_query::Expr, ActiveValue::{Set, NotSet}, ColumnTrait, EntityTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{errors::{LoggableError, UpubError}, model, server::{addresser::Addresser, builders::AnyQuery, normalizer::Normalizer}};
-
-use super::{fetcher::{Fetcher, PullResult}, side_effects::SideEffects, Context};
-
-
-#[axum::async_trait]
-impl apb::server::Inbox for Context {
-	type Error = UpubError;
-	type Activity = serde_json::Value;
-
-	async fn create(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let Some(object_node) = activity.object().extract() else {
-			// TODO we could process non-embedded activities or arrays but im lazy rn
-			tracing::error!("refusing to process activity without embedded object: {}", serde_json::to_string_pretty(&activity).unwrap());
-			return Err(UpubError::unprocessable());
-		};
-		if let Some(reply) = object_node.in_reply_to().id() {
-			if let Err(e) = self.fetch_object(&reply).await {
-				tracing::warn!("failed fetching replies for received object: {e}");
-			}
-		}
-		let activity_model = self.insert_activity(activity, Some(server.clone())).await?;
-		let object_model = self.insert_object(object_node, Some(server)).await?;
-		let expanded_addressing = self.expand_addressing(activity_model.addressed()).await?;
-		self.address_to(Some(activity_model.internal), Some(object_model.internal), &expanded_addressing).await?;
-		tracing::info!("{} posted {}", activity_model.actor, object_model.id);
-		Ok(())
-	}
-
-	async fn like(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let uid = activity.actor().id().ok_or(UpubError::bad_request())?;
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-		let object_uri = activity.object().id().ok_or(UpubError::bad_request())?;
-		let obj = self.fetch_object(&object_uri).await?;
-		if model::like::Entity::find_by_uid_oid(internal_uid, obj.internal)
-			.any(self.db())
-			.await?
-		{
-			return Err(UpubError::not_modified());
-		}
-
-		let activity_model = self.insert_activity(activity, Some(server)).await?;
-		self.process_like(internal_uid, obj.internal, activity_model.internal, activity_model.published).await?;
-		let mut expanded_addressing = self.expand_addressing(activity_model.addressed()).await?;
-		if expanded_addressing.is_empty() { // WHY MASTODON!!!!!!!
-			expanded_addressing.push(
-				model::object::Entity::find_by_id(obj.internal)
-					.select_only()
-					.select_column(model::object::Column::AttributedTo)
-					.into_tuple::<String>()
-					.one(self.db())
-					.await?
-					.ok_or_else(UpubError::not_found)?
-				);
-		}
-		self.address_to(Some(activity_model.internal), None, &expanded_addressing).await?;
-		tracing::info!("{} liked {}", uid, obj.id);
-		Ok(())
-	}
-
-	async fn follow(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		let aid = activity.id().ok_or_else(UpubError::bad_request)?.to_string();
-		let source_actor = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		let source_actor_internal = model::actor::Entity::ap_to_internal(&source_actor, self.db()).await?;
-		let target_actor = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let usr = self.fetch_user(&target_actor).await?;
-		let activity_model = model::activity::ActiveModel::new(&activity)?;
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db()).await?;
-		let internal_aid = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-		let relation_model = model::relation::ActiveModel {
-			internal: NotSet,
-			accept: Set(None),
-			activity: Set(internal_aid),
-			follower: Set(source_actor_internal),
-			following: Set(usr.internal),
-		};
-		model::relation::Entity::insert(relation_model)
-			.exec(self.db()).await?;
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&target_actor) {
-			expanded_addressing.push(target_actor);
-		}
-		self.address_to(Some(internal_aid), None, &expanded_addressing).await?;
-		tracing::info!("{} wants to follow {}", source_actor, usr.id);
-		Ok(())
-	}
-
-	async fn accept(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		// TODO what about TentativeAccept
-		let aid = activity.id().ok_or_else(UpubError::bad_request)?.to_string();
-		let target_actor = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		let follow_request_id = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let follow_activity = model::activity::Entity::find_by_ap_id(&follow_request_id)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-
-		if follow_activity.object.unwrap_or("".into()) != target_actor {
-			return Err(UpubError::forbidden());
-		}
-
-		let activity_model = model::activity::ActiveModel::new(&activity)?;
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db())
-			.await?;
-		let accept_internal_id = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-
-		model::actor::Entity::update_many()
-			.col_expr(
-				model::actor::Column::FollowingCount,
-				Expr::col(model::actor::Column::FollowingCount).add(1)
-			)
-			.filter(model::actor::Column::Id.eq(&follow_activity.actor))
-			.exec(self.db())
-			.await?;
-		model::actor::Entity::update_many()
-			.col_expr(
-				model::actor::Column::FollowersCount,
-				Expr::col(model::actor::Column::FollowersCount).add(1)
-			)
-			.filter(model::actor::Column::Id.eq(&follow_activity.actor))
-			.exec(self.db())
-			.await?;
-
-		model::relation::Entity::update_many()
-			.col_expr(model::relation::Column::Accept, Expr::value(Some(accept_internal_id)))
-			.filter(model::relation::Column::Activity.eq(follow_activity.internal))
-			.exec(self.db()).await?;
-
-		tracing::info!("{} accepted follow request by {}", target_actor, follow_activity.actor);
-
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&follow_activity.actor) {
-			expanded_addressing.push(follow_activity.actor);
-		}
-		self.address_to(Some(accept_internal_id), None, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn reject(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		// TODO what about TentativeReject?
-		let aid = activity.id().ok_or_else(UpubError::bad_request)?.to_string();
-		let uid = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		let follow_request_id = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let follow_activity = model::activity::Entity::find_by_ap_id(&follow_request_id)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-
-		if follow_activity.object.unwrap_or("".into()) != uid {
-			return Err(UpubError::forbidden());
-		}
-
-		let activity_model = model::activity::ActiveModel::new(&activity)?;
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db())
-			.await?;
-		let internal_aid = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-
-		model::relation::Entity::delete_many()
-			.filter(model::relation::Column::Activity.eq(internal_aid))
-			.exec(self.db())
-			.await?;
-
-		tracing::info!("{} rejected follow request by {}", uid, follow_activity.actor);
-
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&follow_activity.actor) {
-			expanded_addressing.push(follow_activity.actor);
-		}
-
-		self.address_to(Some(internal_aid), None, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn delete(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		model::actor::Entity::delete_by_ap_id(&oid).exec(self.db()).await.info_failed("failed deleting from users");
-		model::object::Entity::delete_by_ap_id(&oid).exec(self.db()).await.info_failed("failed deleting from objects");
-		tracing::debug!("deleted '{oid}'");
-		Ok(())
-	}
-
-	async fn update(&self, _server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let uid = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		let aid = activity.id().ok_or_else(UpubError::bad_request)?;
-		let Some(object_node) = activity.object().extract() else {
-			// TODO we could process non-embedded activities or arrays but im lazy rn
-			tracing::error!("refusing to process activity without embedded object: {}", serde_json::to_string_pretty(&activity).unwrap());
-			return Err(UpubError::unprocessable());
-		};
-		let oid = object_node.id().ok_or_else(UpubError::bad_request)?.to_string();
-
-		let activity_model = model::activity::ActiveModel::new(&activity)?;
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db())
-			.await?;
-		let internal_aid = model::activity::Entity::ap_to_internal(aid, self.db()).await?;
-
-		let internal_oid = match object_node.object_type().ok_or_else(UpubError::bad_request)? {
-			apb::ObjectType::Actor(_) => {
-				let internal_uid = model::actor::Entity::ap_to_internal(&oid, self.db()).await?;
-				let mut actor_model = model::actor::ActiveModel::new(&object_node)?;
-				actor_model.internal = Set(internal_uid);
-				actor_model.updated = Set(chrono::Utc::now());
-				model::actor::Entity::update(actor_model)
-					.exec(self.db())
-					.await?;
-				Some(internal_uid)
-			},
-			apb::ObjectType::Note => {
-				let internal_oid = model::object::Entity::ap_to_internal(&oid, self.db()).await?;
-				let mut object_model = model::object::ActiveModel::new(&object_node)?;
-				object_model.internal = Set(internal_oid);
-				object_model.updated = Set(chrono::Utc::now());
-				model::object::Entity::update(object_model)
-					.exec(self.db())
-					.await?;
-				Some(internal_oid)
-			},
-			t => {
-				tracing::warn!("no side effects implemented for update type {t:?}");
-				None
-			},
-		};
-
-		tracing::info!("{} updated {}", uid, oid);
-		let expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		self.address_to(Some(internal_aid), internal_oid, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn undo(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let uid = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-		// TODO in theory we could work with just object_id but right now only accept embedded
-		let undone_activity = activity.object().extract().ok_or_else(UpubError::bad_request)?;
-		let undone_activity_author = undone_activity.actor().id().ok_or_else(UpubError::bad_request)?;
-
-		// can't undo activities from remote actors!
-		if server != Context::server(&undone_activity_author) {
-			return Err(UpubError::forbidden());
-		};
-
-		let activity_model = self.insert_activity(activity.clone(), Some(server)).await?;
-
-		let targets = self.expand_addressing(activity.addressed()).await?;
-		self.process_undo(internal_uid, activity).await?;
-
-		self.address_to(Some(activity_model.internal), None, &targets).await?;
-
-		Ok(())
-	}
-	
-	async fn announce(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let uid = activity.actor().id().ok_or_else(|| UpubError::field("actor"))?;
-		let actor = self.fetch_user(&uid).await?;
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-		let announced_id = activity.object().id().ok_or_else(|| UpubError::field("object"))?;
-		
-		match self.pull(&announced_id).await? {
-			PullResult::Actor(_) => Err(UpubError::unprocessable()),
-			PullResult::Object(object) => {
-				let object_model = self.resolve_object(object).await?;
-				let activity_model = self.insert_activity(activity.clone(), Some(server.clone())).await?;
-
-				// relays send us objects as Announce, but we don't really want to count those towards the
-				// total shares count of an object, so just fetch the object and be done with it
-				if !matches!(actor.actor_type, apb::ActorType::Person) {
-					tracing::info!("relay {} broadcasted {}", activity_model.actor, announced_id);
-					return Ok(())
-				}
-
-				let share = model::announce::ActiveModel {
-					internal: NotSet,
-					actor: Set(internal_uid),
-					object: Set(object_model.internal),
-					published: Set(activity.published().unwrap_or(chrono::Utc::now())),
-				};
-
-				let expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-				self.address_to(Some(activity_model.internal), None, &expanded_addressing).await?;
-				model::announce::Entity::insert(share)
-					.exec(self.db()).await?;
-				model::object::Entity::update_many()
-					.col_expr(model::object::Column::Announces, Expr::col(model::object::Column::Announces).add(1))
-					.filter(model::object::Column::Internal.eq(object_model.internal))
-					.exec(self.db())
-					.await?;
-
-				tracing::info!("{} shared {}", activity_model.actor, announced_id);
-				Ok(())
-			},
-			PullResult::Activity(activity) => {
-				// groups update all members of other things that happen inside, process those
-				let server = Context::server(activity.id().unwrap_or_default());
-				match activity.activity_type().ok_or_else(UpubError::bad_request)? {
-					apb::ActivityType::Like | apb::ActivityType::EmojiReact => Ok(self.like(server, activity).await?),
-					apb::ActivityType::Create => Ok(self.create(server, activity).await?),
-					apb::ActivityType::Undo => Ok(self.undo(server, activity).await?),
-					apb::ActivityType::Delete => Ok(self.delete(server, activity).await?),
-					apb::ActivityType::Update => Ok(self.update(server, activity).await?),
-					x => {
-						tracing::warn!("ignoring unhandled announced activity of type {x:?}");
-						Err(StatusCode::NOT_IMPLEMENTED.into())
-					},
-				}
-			},
-		}
-	}
-}

src/server/mod.rs

@@ -1,15 +0,0 @@
-pub mod addresser;
-pub mod admin;
-pub mod context;
-pub mod dispatcher;
-pub mod fetcher;
-pub mod inbox;
-pub mod init;
-pub mod outbox;
-pub mod auth;
-pub mod builders;
-pub mod httpsign;
-pub mod normalizer;
-pub mod side_effects;
-
-pub use context::Context;

src/server/normalizer.rs

@@ -1,160 +0,0 @@
-use apb::{Node, Base, Object, Document};
-use sea_orm::{sea_query::Expr, ActiveValue::{NotSet, Set}, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter};
-use crate::{errors::UpubError, model, server::Context};
-
-#[axum::async_trait]
-pub trait Normalizer {
-	async fn insert_object(&self, obj: impl apb::Object, server: Option<String>) -> crate::Result<model::object::Model>;
-	async fn insert_activity(&self, act: impl apb::Activity, server: Option<String>) -> crate::Result<model::activity::Model>;
-}
-
-#[axum::async_trait]
-impl Normalizer for super::Context {
-	async fn insert_object(&self, object_node: impl apb::Object, server: Option<String>) -> crate::Result<model::object::Model> {
-		let oid = object_node.id().ok_or_else(UpubError::bad_request)?.to_string();
-		let uid = object_node.attributed_to().id();
-		let mut object_model = model::object::ActiveModel::new(&object_node)?;
-		if let Some(server) = server {
-			// make sure we're allowed to create this object
-			if let Set(Some(object_author)) = &object_model.attributed_to {
-				if server != Context::server(object_author) {
-					return Err(UpubError::forbidden());
-				}
-			} else if server != Context::server(&oid) {
-				return Err(UpubError::forbidden());
-			};
-		}
-
-		// make sure content only contains a safe subset of html
-		if let Set(Some(content)) = object_model.content {
-			object_model.content = Set(Some(mdhtml::safe_html(&content)));
-		}
-
-		// fix context for remote posts
-		// > note that this will effectively recursively try to fetch the parent object, in order to find
-		// > the context (which is id of topmost object). there's a recursion limit of 16 hidden inside
-		// > btw! also if any link is broken or we get rate limited, the whole insertion fails which is
-		// > kind of dumb. there should be a job system so this can be done in waves. or maybe there's
-		// > some whole other way to do this?? im thinking but misskey aaaa!! TODO
-		if let Set(Some(ref reply)) = object_model.in_reply_to {
-			if let Some(o) = model::object::Entity::find_by_ap_id(reply).one(self.db()).await? {
-				object_model.context = Set(o.context);
-			} else {
-				object_model.context = Set(None); // TODO to be filled by some other task
-			}
-		} else {
-			object_model.context = Set(Some(oid.clone()));
-		}
-
-		model::object::Entity::insert(object_model.clone().into_active_model()).exec(self.db()).await?;
-		let object = model::object::Entity::find_by_ap_id(&oid).one(self.db()).await?.ok_or_else(UpubError::internal_server_error)?;
-
-		// update replies counter
-		if let Set(Some(ref in_reply_to)) = object_model.in_reply_to {
-			model::object::Entity::update_many()
-				.filter(model::object::Column::Id.eq(in_reply_to))
-				.col_expr(model::object::Column::Replies, Expr::col(model::object::Column::Replies).add(1))
-				.exec(self.db())
-				.await?;
-		}
-		// update statuses counter
-		if let Some(object_author) = uid {
-			model::actor::Entity::update_many()
-				.col_expr(model::actor::Column::StatusesCount, Expr::col(model::actor::Column::StatusesCount).add(1))
-				.filter(model::actor::Column::Id.eq(&object_author))
-				.exec(self.db())
-				.await?;
-		}
-
-		for attachment in object_node.attachment().flat() {
-			let attachment_model = match attachment {
-				Node::Empty => continue,
-				Node::Array(_) => {
-					tracing::warn!("ignoring array-in-array while processing attachments");
-					continue
-				},
-				Node::Link(l) => model::attachment::ActiveModel {
-					internal: sea_orm::ActiveValue::NotSet,
-					url: Set(l.href().to_string()),
-					object: Set(object.internal),
-					document_type: Set(apb::DocumentType::Page),
-					name: Set(l.link_name().map(|x| x.to_string())),
-					media_type: Set(l.link_media_type().unwrap_or("link").to_string()),
-					published: Set(chrono::Utc::now()),
-				},
-				Node::Object(o) => model::attachment::ActiveModel {
-					internal: sea_orm::ActiveValue::NotSet,
-					url: Set(o.url().id().unwrap_or_else(|| o.id().map(|x| x.to_string()).unwrap_or_default())),
-					object: Set(object.internal),
-					document_type: Set(o.as_document().map_or(apb::DocumentType::Document, |x| x.document_type().unwrap_or(apb::DocumentType::Page))),
-					name: Set(o.name().map(|x| x.to_string())),
-					media_type: Set(o.media_type().unwrap_or("link").to_string()),
-					published: Set(o.published().unwrap_or_else(chrono::Utc::now)),
-				},
-			};
-			model::attachment::Entity::insert(attachment_model)
-				.exec(self.db())
-				.await?;
-		}
-		// lemmy sends us an image field in posts, treat it like an attachment i'd say
-		if let Some(img) = object_node.image().get() {
-			// TODO lemmy doesnt tell us the media type but we use it to display the thing...
-			let img_url = img.url().id().unwrap_or_default();
-			let media_type = if img_url.ends_with("png") {
-				Some("image/png".to_string())
-			} else if img_url.ends_with("webp") {
-				Some("image/webp".to_string())
-			} else if img_url.ends_with("jpeg") || img_url.ends_with("jpg") {
-				Some("image/jpeg".to_string())
-			} else {
-				None
-			};
-
-			let attachment_model = model::attachment::ActiveModel {
-				internal: sea_orm::ActiveValue::NotSet,
-				url: Set(img.url().id().unwrap_or_else(|| img.id().map(|x| x.to_string()).unwrap_or_default())),
-				object: Set(object.internal),
-				document_type: Set(img.as_document().map_or(apb::DocumentType::Document, |x| x.document_type().unwrap_or(apb::DocumentType::Page))),
-				name: Set(img.name().map(|x| x.to_string())),
-				media_type: Set(img.media_type().unwrap_or(media_type.as_deref().unwrap_or("link")).to_string()),
-				published: Set(img.published().unwrap_or_else(chrono::Utc::now)),
-			};
-			model::attachment::Entity::insert(attachment_model)
-				.exec(self.db())
-				.await?;
-		}
-
-		Ok(object)
-	}
-
-	async fn insert_activity(&self, activity: impl apb::Activity, server: Option<String>) -> crate::Result<model::activity::Model> {
-		let mut activity_model = model::activity::Model {
-			internal: 0,
-			id: activity.id().ok_or_else(|| UpubError::field("id"))?.to_string(),
-			activity_type: activity.activity_type().ok_or_else(|| UpubError::field("type"))?,
-			actor: activity.actor().id().ok_or_else(|| UpubError::field("actor"))?,
-			object: activity.object().id(),
-			target: activity.target().id(),
-			published: activity.published().unwrap_or(chrono::Utc::now()),
-			to: activity.to().into(),
-			bto: activity.bto().into(),
-			cc: activity.cc().into(),
-			bcc: activity.bcc().into(),
-		};
-		if let Some(server) = server {
-			if Context::server(&activity_model.actor) != server
-				|| Context::server(&activity_model.id) != server {
-				return Err(UpubError::forbidden());
-			}
-		}
-		let mut active_model = activity_model.clone().into_active_model();
-		active_model.internal = NotSet;
-		model::activity::Entity::insert(active_model)
-			.exec(self.db())
-			.await?;
-
-		let internal = model::activity::Entity::ap_to_internal(&activity_model.id, self.db()).await?;
-		activity_model.internal = internal;
-		Ok(activity_model)
-	}
-}

src/server/outbox.rs

@@ -1,423 +0,0 @@
-use apb::{target::Addressed, Activity, ActivityMut, Base, BaseMut, Node, Object, ObjectMut};
-use reqwest::StatusCode;
-use sea_orm::{sea_query::Expr, ActiveValue::{Set, NotSet, Unchanged}, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{errors::UpubError, model, routes::activitypub::jsonld::LD};
-
-use super::{addresser::Addresser, builders::AnyQuery, fetcher::Fetcher, normalizer::Normalizer, side_effects::SideEffects, Context};
-
-
-#[axum::async_trait]
-impl apb::server::Outbox for Context {
-	type Error = UpubError;
-	type Object = serde_json::Value;
-	type Activity = serde_json::Value;
-
-	async fn create_note(&self, uid: String, object: serde_json::Value) -> crate::Result<String> {
-		self.create(
-			uid,
-			serde_json::Value::new_object()
-				.set_activity_type(Some(apb::ActivityType::Create))
-				.set_to(object.to())
-				.set_bto(object.bto())
-				.set_cc(object.cc())
-				.set_bcc(object.bcc())
-				.set_object(Node::object(object))
-		).await
-	}
-
-	async fn create(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let Some(object) = activity.object().extract() else {
-			return Err(UpubError::bad_request());
-		};
-
-		let raw_oid = uuid::Uuid::new_v4().to_string();
-		let oid = self.oid(&raw_oid);
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-
-		if let Some(reply) = object.in_reply_to().id() {
-			self.fetch_object(&reply).await?;
-		}
-
-		// TODO regex hell here i come...
-		let re = regex::Regex::new(r"@(.+)@([^ ]+)").expect("failed compiling regex pattern");
-		let mut content = object.content().map(|x| x.to_string());
-		if let Some(c) = content {
-			let mut tmp = mdhtml::safe_markdown(&c);
-			for (full, [user, domain]) in re.captures_iter(&tmp.clone()).map(|x| x.extract()) {
-				if let Ok(Some(uid)) = model::actor::Entity::find()
-					.filter(model::actor::Column::PreferredUsername.eq(user))
-					.filter(model::actor::Column::Domain.eq(domain))
-					.select_only()
-					.select_column(model::actor::Column::Id)
-					.into_tuple::<String>()
-					.one(self.db())
-					.await
-				{
-					tmp = tmp.replacen(full, &format!("<a href=\"{uid}\" class=\"u-url mention\">@{user}</a>"), 1);
-				}
-			}
-			content = Some(tmp);
-		}
-
-		self.insert_object(
-			object
-				.set_id(Some(&oid))
-				.set_attributed_to(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-				.set_content(content.as_deref())
-				.set_url(Node::maybe_link(self.cfg().instance.frontend.as_ref().map(|x| format!("{x}/objects/{raw_oid}")))),
-			Some(self.domain().to_string()),
-		).await?;
-
-		self.insert_activity(
-			activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_object(Node::link(oid.clone()))
-				.set_published(Some(chrono::Utc::now())),
-			Some(self.domain().to_string()),
-		).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, Some(&oid)).await?;
-		Ok(aid)
-	}
-
-	async fn like(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let obj_model = self.fetch_object(&oid).await?;
-
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-
-		if model::like::Entity::find_by_uid_oid(internal_uid, obj_model.internal)
-			.any(self.db())
-			.await?
-		{
-			return Err(UpubError::not_modified());
-		}
-
-		let activity_model = self.insert_activity(
-			activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now())),
-			Some(self.domain().to_string()),
-		).await?;
-
-		self.process_like(internal_uid, obj_model.internal, activity_model.internal, chrono::Utc::now()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn follow(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let target = activity.object().id().ok_or_else(UpubError::bad_request)?;
-
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		let follower_internal = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-		let following_internal = model::actor::Entity::ap_to_internal(&target, self.db()).await?;
-
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db()).await?;
-
-		let internal_aid = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-
-		let relation_model = model::relation::ActiveModel {
-			internal: NotSet,
-			follower: Set(follower_internal),
-			following: Set(following_internal),
-			activity: Set(internal_aid),
-			accept: Set(None),
-		};
-
-		model::relation::Entity::insert(relation_model)
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn accept(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let accepted_id = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let accepted_activity = model::activity::Entity::find_by_ap_id(&accepted_id)
-			.one(self.db()).await?
-			.ok_or_else(UpubError::not_found)?;
-
-		if accepted_activity.activity_type != apb::ActivityType::Follow {
-			return Err(UpubError::bad_request());
-		}
-		if uid != accepted_activity.object.ok_or_else(UpubError::bad_request)? {
-			return Err(UpubError::forbidden());
-		}
-
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		let internal_aid = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-
-		match accepted_activity.activity_type {
-			apb::ActivityType::Follow => {
-				model::actor::Entity::update_many()
-					.col_expr(
-						model::actor::Column::FollowersCount,
-						Expr::col(model::actor::Column::FollowersCount).add(1)
-					)
-					.filter(model::actor::Column::Id.eq(&uid))
-					.exec(self.db())
-					.await?;
-				model::relation::Entity::update_many()
-					.filter(model::relation::Column::Activity.eq(accepted_activity.internal))
-					.col_expr(model::relation::Column::Accept, Expr::value(Some(internal_aid)))
-					.exec(self.db()).await?;
-			},
-			t => tracing::error!("no side effects implemented for accepting {t:?}"),
-		}
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn reject(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let rejected_id = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let rejected_activity = model::activity::Entity::find_by_ap_id(&rejected_id)
-			.one(self.db()).await?
-			.ok_or_else(UpubError::not_found)?;
-
-		if rejected_activity.activity_type != apb::ActivityType::Follow {
-			return Err(UpubError::bad_request());
-		}
-		if uid != rejected_activity.object.ok_or_else(UpubError::bad_request)? {
-			return Err(UpubError::forbidden());
-		}
-
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db()).await?;
-
-		let internal_aid = model::activity::Entity::ap_to_internal(&aid, self.db()).await?;
-
-		model::relation::Entity::delete_many()
-			.filter(model::relation::Column::Activity.eq(internal_aid))
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn undo(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-		let old_aid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let old_activity = model::activity::Entity::find_by_ap_id(&old_aid)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-		if old_activity.actor != uid {
-			return Err(UpubError::forbidden());
-		}
-
-		let activity_model = self.insert_activity(
-			activity.clone()
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now())),
-			Some(self.domain().to_string())
-		).await?;
-
-		let targets = self.expand_addressing(activity.addressed()).await?;
-		self.process_undo(internal_uid, activity).await?;
-
-		self.address_to(Some(activity_model.internal), None, &targets).await?;
-		self.deliver_to(&activity_model.id, &uid, &targets).await?;
-
-		Ok(aid)
-	}
-
-	async fn delete(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-
-		let object = model::object::Entity::find_by_ap_id(&oid)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-
-		if uid != object.attributed_to.ok_or_else(UpubError::forbidden)? {
-			// can't change objects of others, and objects from noone count as others
-			return Err(UpubError::forbidden());
-		}
-
-		let addressed = activity.addressed();
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db())
-			.await?;
-
-		model::object::Entity::delete_by_ap_id(&oid)
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, addressed, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn update(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let object_node = activity.object().extract().ok_or_else(UpubError::bad_request)?;
-		let addressed = activity.addressed();
-		let target = object_node.id().ok_or_else(UpubError::bad_request)?.to_string();
-
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db()).await?;
-
-		match object_node.object_type() {
-			Some(apb::ObjectType::Actor(_)) => {
-				let old_actor_model = model::actor::Entity::find_by_ap_id(&target)
-					.one(self.db())
-					.await?
-					.ok_or_else(UpubError::not_found)?;
-
-				if old_actor_model.id != uid {
-					// can't change user fields of others
-					return Err(UpubError::forbidden());
-				}
-
-				let mut new_actor_model = model::actor::ActiveModel {
-					internal: Unchanged(old_actor_model.internal),
-					..Default::default()
-				};
-
-				if let Some(name) = object_node.name() {
-					new_actor_model.name = Set(Some(name.to_string()));
-				}
-				if let Some(summary) = object_node.summary() {
-					new_actor_model.summary = Set(Some(summary.to_string()));
-				}
-				if let Some(image) = object_node.image().id() {
-					new_actor_model.image = Set(Some(image));
-				}
-				if let Some(icon) = object_node.icon().id() {
-					new_actor_model.icon = Set(Some(icon));
-				}
-				new_actor_model.updated = Set(chrono::Utc::now());
-
-				model::actor::Entity::update(new_actor_model)
-					.exec(self.db()).await?;
-			},
-			Some(apb::ObjectType::Note) => {
-				let old_object_model = model::object::Entity::find_by_ap_id(&target)
-					.one(self.db())
-					.await?
-					.ok_or_else(UpubError::not_found)?;
-
-				if uid != old_object_model.attributed_to.ok_or_else(UpubError::forbidden)? {
-					// can't change objects of others
-					return Err(UpubError::forbidden());
-				}
-
-				let mut new_object_model = model::object::ActiveModel {
-					internal: Unchanged(old_object_model.internal),
-					..Default::default()
-				};
-
-				if let Some(name) = object_node.name() {
-					new_object_model.name = Set(Some(name.to_string()));
-				}
-				if let Some(summary) = object_node.summary() {
-					new_object_model.summary = Set(Some(summary.to_string()));
-				}
-				if let Some(content) = object_node.content() {
-					new_object_model.content = Set(Some(content.to_string()));
-				}
-				new_object_model.updated = Set(chrono::Utc::now());
-
-				model::object::Entity::update(new_object_model)
-					.exec(self.db()).await?;
-			},
-			_ => return Err(UpubError::Status(StatusCode::NOT_IMPLEMENTED)),
-		}
-
-		self.dispatch(&uid, addressed, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn announce(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let obj = self.fetch_object(&oid).await?;
-		let internal_uid = model::actor::Entity::ap_to_internal(&uid, self.db()).await?;
-
-		let activity_model = model::activity::ActiveModel::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		let share_model = model::announce::ActiveModel {
-			internal: NotSet,
-			actor: Set(internal_uid),
-			object: Set(obj.internal),
-			published: Set(chrono::Utc::now()),
-		};
-		model::activity::Entity::insert(activity_model)
-			.exec(self.db()).await?;
-		model::announce::Entity::insert(share_model).exec(self.db()).await?;
-		model::object::Entity::update_many()
-			.col_expr(model::object::Column::Announces, Expr::col(model::object::Column::Announces).add(1))
-			.filter(model::object::Column::Internal.eq(obj.internal))
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-}

src/server/side_effects.rs

@@ -1,79 +0,0 @@
-use reqwest::StatusCode;
-use sea_orm::{sea_query::Expr, ActiveValue::{NotSet, Set}, ColumnTrait, Condition, EntityTrait, QueryFilter};
-
-use crate::{errors::UpubError, model};
-
-#[axum::async_trait]
-pub trait SideEffects {
-	async fn process_like(&self, who: i64, what: i64, with: i64, when: chrono::DateTime<chrono::Utc>) -> crate::Result<()>;
-	async fn process_undo(&self, who: i64, activity: impl apb::Activity) -> crate::Result<()>;
-}
-
-#[axum::async_trait]
-impl SideEffects for super::Context {
-	async fn process_like(&self, who: i64, what: i64, with: i64, when: chrono::DateTime<chrono::Utc>) -> crate::Result<()> {
-		let like = model::like::ActiveModel {
-			internal: NotSet,
-			actor: Set(who),
-			object: Set(what),
-			activity: Set(with),
-			published: Set(when),
-		};
-		model::like::Entity::insert(like).exec(self.db()).await?;
-		model::object::Entity::update_many()
-			.col_expr(model::object::Column::Likes, Expr::col(model::object::Column::Likes).add(1))
-			.filter(model::object::Column::Internal.eq(what))
-			.exec(self.db())
-			.await?;
-
-		Ok(())
-	}
-
-	async fn process_undo(&self, who: i64, activity: impl apb::Activity) -> crate::Result<()> {
-		let undone_object_id = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		match activity.activity_type() {
-			Some(apb::ActivityType::Like) => {
-				let internal_oid = model::object::Entity::ap_to_internal(&undone_object_id, self.db()).await?;
-				model::like::Entity::delete_many()
-					.filter(
-						Condition::all()
-							.add(model::like::Column::Actor.eq(who))
-							.add(model::like::Column::Object.eq(internal_oid))
-					)
-					.exec(self.db())
-					.await?;
-				model::object::Entity::update_many()
-					.filter(model::object::Column::Internal.eq(internal_oid))
-					.col_expr(model::object::Column::Likes, Expr::col(model::object::Column::Likes).sub(1))
-					.exec(self.db())
-					.await?;
-			},
-			Some(apb::ActivityType::Follow) => {
-				let undone_aid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-				let internal_aid = model::activity::Entity::ap_to_internal(&undone_aid, self.db()).await?;
-				model::relation::Entity::delete_many()
-					.filter(model::relation::Column::Activity.eq(internal_aid))
-					.exec(self.db())
-					.await?;
-				model::actor::Entity::update_many()
-					.filter(model::actor::Column::Internal.eq(who))
-					.col_expr(model::actor::Column::FollowingCount, Expr::col(model::actor::Column::FollowingCount).sub(1))
-					.exec(self.db())
-					.await?;
-				model::actor::Entity::update_many()
-					.filter(model::actor::Column::Id.eq(&undone_object_id))
-					.col_expr(model::actor::Column::FollowersCount, Expr::col(model::actor::Column::FollowersCount).sub(1))
-					.exec(self.db())
-					.await?;
-			},
-			t => {
-				tracing::error!("received 'Undo' for unimplemented activity type: {t:?}");
-				return Err(StatusCode::NOT_IMPLEMENTED.into());
-			},
-		}
-		
-
-		Ok(())
-	}
-
-}

upub/cli/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "upub-cli"
+version = "0.2.0"
+edition = "2021"
+authors = [ "alemi <me@alemi.dev>" ]
+description = "cli maintenance tasks for upub"
+license = "AGPL-3.0"
+repository = "https://git.alemi.dev/upub.git"
+readme = "README.md"
+
+[lib]
+
+[dependencies]
+apb = { path = "../../apb/" }
+upub = { path = "../core" }
+tracing = "0.1"
+serde_json = "1"
+sha256 = "1.5"
+uuid = { version = "1.10", features = ["v4"] }
+chrono = { version = "0.4", features = ["serde"] }
+openssl = "0.10" # TODO handle pubkeys with a smaller crate
+clap = { version = "4.5", features = ["derive"] }
+sea-orm = "1.0"
+futures = "0.3"
+mdhtml = { path = "../../utils/mdhtml/" }
+reqwest = { version = "0.12", features = ["json"] }

upub/cli/README.md

@@ -0,0 +1 @@
+# upub cli

upub/cli/src/cloak.rs

@@ -0,0 +1,115 @@
+use futures::TryStreamExt;
+use sea_orm::{ActiveModelTrait, ActiveValue::{NotSet, Set, Unchanged}, ColumnTrait, Condition, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns};
+use upub::traits::{fetch::RequestError, Cloaker};
+
+pub async fn cloak(ctx: upub::Context, post_contents: bool, objects: bool, actors: bool) -> Result<(), RequestError> {
+	let local_base = format!("{}%", ctx.base());
+	{
+		let mut stream = upub::model::attachment::Entity::find()
+			.filter(upub::model::attachment::Column::Url.not_like(&local_base))
+			.stream(ctx.db())
+			.await?;
+
+		while let Some(attachment) = stream.try_next().await? {
+			tracing::info!("cloaking {}", attachment.url);
+			let (sig, url) = ctx.cloak(&attachment.url);
+			let mut model = attachment.into_active_model();
+			model.url = Set(upub::url!(ctx, "/proxy/{sig}/{url}"));
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if objects {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Image.is_not_null())
+			.filter(upub::model::object::Column::Image.not_like(&local_base))
+			.select_only()
+			.select_column(upub::model::object::Column::Internal)
+			.select_column(upub::model::object::Column::Image)
+			.into_tuple::<(i64, String)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, image)) = stream.try_next().await? {
+			tracing::info!("cloaking object image {image}");
+			let model = upub::model::object::ActiveModel {
+				internal: Unchanged(internal),
+				image: Set(Some(ctx.cloaked(&image))),
+				..Default::default()
+			};
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if actors {
+		let mut stream = upub::model::actor::Entity::find()
+			.filter(
+				Condition::any()
+					.add(upub::model::actor::Column::Image.not_like(&local_base))
+					.add(upub::model::actor::Column::Icon.not_like(&local_base))
+			)
+			.select_only()
+			.select_column(upub::model::actor::Column::Internal)
+			.select_column(upub::model::actor::Column::Image)
+			.select_column(upub::model::actor::Column::Icon)
+			.into_tuple::<(i64, Option<String>, Option<String>)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, image, icon)) = stream.try_next().await? {
+			tracing::info!("cloaking user #{internal}");
+			if image.is_none() && icon.is_none() { continue }
+			// TODO can this if/else/else be made nicer??
+			let image = if let Some(img) = image {
+				if !img.starts_with(ctx.base()) {
+					Set(Some(ctx.cloaked(&img)))
+				} else {
+					NotSet
+				}
+			} else {
+				NotSet
+			};
+			let icon = if let Some(icn) = icon {
+				if !icn.starts_with(ctx.base()) {
+					Set(Some(ctx.cloaked(&icn)))
+				} else {
+					NotSet
+				}
+			} else {
+				NotSet
+			};
+			let model = upub::model::actor::ActiveModel {
+				internal: Unchanged(internal),
+				image, icon,
+				..Default::default()
+			};
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if post_contents {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Content.like("%<img%"))
+			.select_only()
+			.select_column(upub::model::object::Column::Internal)
+			.select_column(upub::model::object::Column::Content)
+			.into_tuple::<(i64, String)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, content)) = stream.try_next().await? {
+			let sanitized = ctx.sanitize(&content);
+			if sanitized != content {
+				tracing::info!("sanitizing object #{internal}");
+				let model = upub::model::object::ActiveModel {
+					internal: Unchanged(internal),
+					content: Set(Some(sanitized)),
+					..Default::default()
+				};
+				model.update(ctx.db()).await?;
+			}
+		}
+	}
+
+	Ok(())
+}

upub/cli/src/faker.rs

@@ -1,8 +1,8 @@
-use crate::model::{addressing, config, credential, activity, object, actor, Audience};
+use upub::{ext::JsonVec, model::{activity, actor, addressing, config, credential, object}};
 use openssl::rsa::Rsa;
 use sea_orm::{ActiveValue::NotSet, IntoActiveModel};
 
-pub async fn faker(ctx: crate::server::Context, count: i64) -> Result<(), sea_orm::DbErr> {
+pub async fn faker(ctx: upub::Context, count: i64) -> Result<(), sea_orm::DbErr> {
 	use sea_orm::{EntityTrait, Set};
 
 	let domain = ctx.domain();
@@ -21,6 +21,9 @@ pub async fn faker(ctx: crate::server::Context, count: i64) -> Result<(), sea_or
 		followers: None,
 		followers_count: 0,
 		statuses_count: count as i32,
+		fields: JsonVec::default(),
+		also_known_as: JsonVec::default(),
+		moved_to: None,
 		icon: Some("https://cdn.alemi.dev/social/circle-square.png".to_string()),
 		image: Some("https://cdn.alemi.dev/social/someriver-xs.jpg".to_string()),
 		inbox: None,
@@ -51,6 +54,7 @@ pub async fn faker(ctx: crate::server::Context, count: i64) -> Result<(), sea_or
 		actor: Set(test_user.id.clone()),
 		login: Set("mail@example.net".to_string()),
 		password: Set(sha256::digest("very-strong-password")),
+		active: Set(true),
 	}).exec(db).await?;
 
 	let context = uuid::Uuid::new_v4().to_string();
@@ -77,16 +81,19 @@ pub async fn faker(ctx: crate::server::Context, count: i64) -> Result<(), sea_or
 			summary: Set(None),
 			context: Set(Some(context.clone())),
 			in_reply_to: Set(None),
+			quote: Set(None),
 			content: Set(Some(format!("[{i}] Tic(k). Quasiparticle of intensive multiplicity. Tics (or ticks) are intrinsically several components of autonomously numbering anorganic populations, propagating by contagion between segmentary divisions in the order of nature. Ticks - as nonqualitative differentially-decomposable counting marks - each designate a multitude comprehended as a singular variation in tic(k)-density."))),
+			image: Set(None),
 			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i as u64)),
 			updated: Set(chrono::Utc::now()),
 			replies: Set(0),
 			likes: Set(0),
 			announces: Set(0),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
+			audience: Set(None),
-			bto: Set(Audience::default()),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
-			cc: Set(Audience(vec![])),
+			bto: Set(JsonVec::default()),
-			bcc: Set(Audience::default()),
+			cc: Set(JsonVec(vec![])),
+			bcc: Set(JsonVec::default()),
 			url: Set(None),
 			sensitive: Set(false),
 		}).exec(db).await?;
@@ -99,10 +106,10 @@ pub async fn faker(ctx: crate::server::Context, count: i64) -> Result<(), sea_or
 			object: Set(Some(format!("{domain}/objects/{oid}"))),
 			target: Set(None),
 			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i as u64)),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
-			bto: Set(Audience::default()),
+			bto: Set(JsonVec::default()),
-			cc: Set(Audience(vec![])),
+			cc: Set(JsonVec(vec![])),
-			bcc: Set(Audience::default()),
+			bcc: Set(JsonVec::default()),
 		}).exec(db).await?;
 	}
 

upub/cli/src/fetch.rs

@@ -0,0 +1,65 @@
+use sea_orm::{EntityTrait, TransactionTrait};
+use upub::traits::{fetch::RequestError, Addresser, Fetcher, Normalizer};
+
+pub async fn fetch(ctx: upub::Context, uri: String, save: bool, actor: Option<String>) -> Result<(), RequestError> {
+	use apb::Base;
+
+	let mut pkey = ctx.pkey().to_string();
+	let mut from = ctx.base().to_string();
+	
+	if let Some(actor) = actor {
+		let actor_model = upub::model::actor::Entity::find_by_ap_id(&actor)
+			.one(ctx.db())
+			.await?
+			.ok_or_else(|| sea_orm::DbErr::RecordNotFound(actor.clone()))?;
+
+		match actor_model.private_key {
+			None => tracing::error!("requested actor lacks a private key, fetching with server key instead"),
+			Some(x) => {
+				pkey = x;
+				from = actor.to_string();
+			},
+		}
+	}
+
+	let mut node = apb::Node::link(uri.to_string());
+	if let apb::Node::Link(ref uri) = node {
+		if let Ok(href) = uri.href() {
+			node = upub::Context::request(reqwest::Method::GET, href, None, &from, &pkey, ctx.domain())
+				.await?
+				.json::<serde_json::Value>()
+				.await?
+				.into();
+		}
+	}
+
+
+	let obj = node.extract().expect("node still empty after fetch?");
+
+	println!("{}", serde_json::to_string_pretty(&obj).unwrap());
+
+	if save {
+		let tx = ctx.db().begin().await?;
+		match obj.base_type() {
+			Ok(apb::BaseType::Object(apb::ObjectType::Actor(_))) => {
+				upub::model::actor::Entity::insert(upub::AP::actor_q(&obj, None)?)
+					.exec(&tx)
+					.await?;
+			},
+			Ok(apb::BaseType::Object(apb::ObjectType::Activity(_))) => {
+				let act = ctx.insert_activity(obj, &tx).await?;
+				ctx.address(Some(&act), None, &tx).await?;
+			},
+			Ok(apb::BaseType::Object(apb::ObjectType::Note)) => {
+				let obj = ctx.insert_object(obj, &tx).await?;
+				ctx.address(None, Some(&obj), &tx).await?;
+			},
+			Ok(apb::BaseType::Object(t)) => tracing::warn!("not implemented: {:?}", t),
+			Ok(apb::BaseType::Link(_)) => tracing::error!("fetched another link?"),
+			Err(_) => tracing::error!("no type on object"),
+		}
+		tx.commit().await?;
+	}
+
+	Ok(())
+}

upub/cli/src/fix.rs

@@ -1,7 +1,6 @@
-use sea_orm::EntityTrait;
+use sea_orm::{ActiveModelTrait, EntityTrait};
 
-
+pub async fn fix(ctx: upub::Context, likes: bool, shares: bool, replies: bool) -> Result<(), sea_orm::DbErr> {
-pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies: bool) -> crate::Result<()> {
 	use futures::TryStreamExt;
 	let db = ctx.db();
 
@@ -9,22 +8,19 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing likes...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::like::Entity::find().stream(db).await?;
+			let mut stream = upub::model::like::Entity::find().stream(db).await?;
 			while let Some(like) = stream.try_next().await? {
 				store.insert(like.object, store.get(&like.object).unwrap_or(&0) + 1);
 			}
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
+			let m = upub::model::object::ActiveModel {
-				internal: sea_orm::Set(k),
+				internal: sea_orm::Unchanged(k),
 				likes: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
+			if let Err(e) = m.update(db).await {
-				.exec(db)
-				.await
-			{
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}
@@ -34,22 +30,19 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing shares...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::announce::Entity::find().stream(db).await?;
+			let mut stream = upub::model::announce::Entity::find().stream(db).await?;
 			while let Some(share) = stream.try_next().await? {
 				store.insert(share.object, store.get(&share.object).unwrap_or(&0) + 1);
 			}
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
+			let m = upub::model::object::ActiveModel {
-				internal: sea_orm::Set(k),
+				internal: sea_orm::Unchanged(k),
 				announces: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
+			if let Err(e) = m.update(db).await {
-				.exec(db)
-				.await
-			{
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}
@@ -59,7 +52,7 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing replies...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::object::Entity::find().stream(db).await?;
+			let mut stream = upub::model::object::Entity::find().stream(db).await?;
 			while let Some(object) = stream.try_next().await? {
 				if let Some(reply) = object.in_reply_to {
 					let before = store.get(&reply).unwrap_or(&0);
@@ -69,15 +62,13 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
+			let m = upub::model::object::ActiveModel {
-				id: sea_orm::Set(k.clone()),
+				id: sea_orm::Unchanged(k.clone()),
 				replies: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
+			// TODO will update work with non-primary-key field??
-				.exec(db)
+			if let Err(e) = m.update(db).await {
-				.await
-			{
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}

upub/cli/src/lib.rs

@@ -16,6 +16,15 @@ pub use register::*;
 mod update;
 pub use update::*;
 
+mod nuke;
+pub use nuke::*;
+
+mod thread;
+pub use thread::*;
+
+mod cloak;
+pub use cloak::*;
+
 #[derive(Debug, Clone, clap::Subcommand)]
 pub enum CliCommand {
 	/// generate fake user, note and activity
@@ -32,16 +41,17 @@ pub enum CliCommand {
 		#[arg(long, default_value_t = false)]
 		/// store fetched object in local db
 		save: bool,
+
+		#[arg(long)]
+		/// use this actor's private key to fetch
+		fetch_as: Option<String>,
 	},
 
-	/// follow a remote relay
+	/// act on remote relay actors at instance level
 	Relay {
-		/// actor url, same as with pleroma
+		#[clap(subcommand)]
-		actor: String,
+		/// action to take against this relay
-
+		action: RelayCommand,
-		#[arg(long, default_value_t = false)]
-		/// instead of sending a follow request, send an accept
-		accept: bool
 	},
 
 	/// run db maintenance tasks
@@ -59,11 +69,15 @@ pub enum CliCommand {
 		replies: bool,
 	},
 
-	/// update remote users
+	/// update remote actors
 	Update {
-		#[arg(long, short, default_value_t = 7)]
+		#[arg(long, short, default_value_t = 10)]
-		/// number of days after which users should get updated
+		/// number of days after which actors should get updated
 		days: i64,
+
+		#[arg(long)]
+		/// stop after updating this many actors
+		limit: Option<u64>,
 	},
 
 	/// register a new local user
@@ -90,30 +104,60 @@ pub enum CliCommand {
 		/// url for banner image of new user
 		#[arg(long = "banner")]
 		banner_url: Option<String>,
-	}
+	},
+
+	/// break all user relations so that instance can be shut down
+	Nuke {
+		/// unless this is set, nuke will be a dry run
+		#[arg(long, default_value_t = false)]
+		for_real: bool,
+
+		/// also send Delete activities for all local objects
+		#[arg(long, default_value_t = false)]
+		delete_objects: bool,
+	},
+
+	/// attempt to fix broken threads and completely gather their context
+	Thread {
+		
+	},
+
+	/// replaces all attachment urls with proxied local versions (only useful for old instances)
+	Cloak {
+		/// also cloak objects image urls
+		#[arg(long, default_value_t = false)]
+		objects: bool,
+
+		/// also cloak actor images
+		#[arg(long, default_value_t = false)]
+		actors: bool,
+
+		/// also replace urls inside post contents
+		#[arg(long, default_value_t = false)]
+		contents: bool,
+	},
 }
 
-pub async fn run(
+pub async fn run(ctx: upub::Context, command: CliCommand) -> Result<(), Box<dyn std::error::Error>> {
-	command: CliCommand,
+	tracing::info!("running cli task: {command:?}");
-	db: sea_orm::DatabaseConnection,
-	domain: String,
-	config: crate::config::Config,
-) -> crate::Result<()> {
-	let ctx = crate::server::Context::new(
-		db, domain, config,
-	).await?;
 	match command {
 		CliCommand::Faker { count } =>
 			Ok(faker(ctx, count as i64).await?),
-		CliCommand::Fetch { uri, save } =>
+		CliCommand::Fetch { uri, save, fetch_as } =>
-			Ok(fetch(ctx, uri, save).await?),
+			Ok(fetch(ctx, uri, save, fetch_as).await?),
-		CliCommand::Relay { actor, accept } =>
+		CliCommand::Relay { action } =>
-			Ok(relay(ctx, actor, accept).await?),
+			Ok(relay(ctx, action).await?),
 		CliCommand::Fix { likes, shares, replies } =>
 			Ok(fix(ctx, likes, shares, replies).await?),
-		CliCommand::Update { days } =>
+		CliCommand::Update { days, limit } =>
-			Ok(update_users(ctx, days).await?),
+			Ok(update_users(ctx, days, limit).await?),
 		CliCommand::Register { username, password, display_name, summary, avatar_url, banner_url } =>
 			Ok(register(ctx, username, password, display_name, summary, avatar_url, banner_url).await?),
+		CliCommand::Nuke { for_real, delete_objects } =>
+			Ok(nuke(ctx, for_real, delete_objects).await?),
+		CliCommand::Thread { } =>
+			Ok(thread(ctx).await?),
+		CliCommand::Cloak { objects, actors, contents } =>
+			Ok(cloak(ctx, contents, objects, actors).await?),
 	}
 }

upub/cli/src/nuke.rs

@@ -0,0 +1,137 @@
+use std::collections::HashSet;
+
+use apb::{ActivityMut, BaseMut, ObjectMut};
+use futures::TryStreamExt;
+use sea_orm::{ActiveValue::{Set, NotSet}, ColumnTrait, EntityTrait, QueryFilter, QuerySelect, SelectColumns};
+
+
+pub async fn nuke(ctx: upub::Context, for_real: bool, delete_posts: bool) -> Result<(), sea_orm::DbErr> {
+	if !for_real {
+		tracing::warn!("THIS IS A DRY RUN! pass --for-real to actually nuke this instance");
+	}
+
+	let mut to_undo = Vec::new();
+
+	// TODO rather expensive to find all local users with a LIKE query, should add an isLocal flag
+	let local_users_vec = upub::model::actor::Entity::find()
+		.filter(upub::model::actor::Column::Id.like(format!("{}%", ctx.base())))
+		.select_only()
+		.select_column(upub::model::actor::Column::Internal)
+		.into_tuple::<i64>()
+		.all(ctx.db())
+		.await?;
+	
+	let local_users : HashSet<i64> = HashSet::from_iter(local_users_vec);
+
+	{
+		let mut stream = upub::model::relation::Entity::find().stream(ctx.db()).await?;
+		while let Some(like) = stream.try_next().await? {
+			if local_users.contains(&like.follower) {
+				to_undo.push(like.activity);
+			} else if local_users.contains(&like.following) {
+				if let Some(accept) = like.accept {
+					to_undo.push(accept);
+				}
+			}
+		}
+	}
+
+	for internal in to_undo {
+		let Some(activity) = upub::model::activity::Entity::find_by_id(internal)
+			.one(ctx.db())
+			.await?
+		else {
+			tracing::error!("could not load activity #{internal}");
+			continue;
+		};
+
+		let Some(ref oid) = activity.object
+		else {
+			tracing::error!("can't undo activity without object");
+			continue;
+		};
+
+		let (target, undone) = if matches!(activity.activity_type, apb::ActivityType::Follow) {
+			(oid.clone(), activity.clone().ap())
+		} else {
+			let follow_activity = upub::model::activity::Entity::find_by_ap_id(oid)
+				.one(ctx.db())
+				.await?
+				.ok_or(sea_orm::DbErr::RecordNotFound(oid.clone()))?;
+			(follow_activity.clone().object.unwrap_or_default(), follow_activity.ap())
+		};
+
+		let aid = ctx.aid(&upub::Context::new_id());
+		let undo_activity = apb::new()
+			.set_id(Some(&aid))
+			.set_activity_type(Some(apb::ActivityType::Undo))
+			.set_actor(apb::Node::link(activity.actor.clone()))
+			.set_object(apb::Node::object(undone))
+			.set_to(apb::Node::links(vec![target]))
+			.set_published(Some(chrono::Utc::now()));
+
+
+		let job = upub::model::job::ActiveModel {
+			internal: NotSet,
+			activity: Set(aid.clone()),
+			job_type: Set(upub::model::job::JobType::Outbound),
+			actor: Set(activity.actor),
+			target: Set(None),
+			published: Set(chrono::Utc::now()),
+			not_before: Set(chrono::Utc::now()),
+			attempt: Set(0),
+			payload: Set(Some(undo_activity)),
+			error: Set(None),
+		};
+
+		tracing::info!("undoing {}", activity.id);
+
+		if for_real {
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		}
+	}
+
+	if delete_posts {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Id.like(format!("{}%", ctx.base())))
+			.stream(ctx.db())
+			.await?;
+
+		while let Some(object) = stream.try_next().await? {
+			let aid = ctx.aid(&upub::Context::new_id());
+			let actor = object.attributed_to.unwrap_or_else(|| ctx.domain().to_string());
+			let undo_activity = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Delete))
+				.set_actor(apb::Node::link(actor.clone()))
+				.set_object(apb::Node::link(object.id.clone()))
+				.set_to(apb::Node::links(object.to.0))
+				.set_cc(apb::Node::links(object.cc.0))
+				.set_bto(apb::Node::links(object.bto.0))
+				.set_bcc(apb::Node::links(object.bcc.0))
+				.set_published(Some(chrono::Utc::now()));
+
+
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(actor),
+				target: Set(None),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				attempt: Set(0),
+				payload: Set(Some(undo_activity)),
+				error: Set(None),
+			};
+
+			tracing::info!("deleting {}", object.id);
+
+			if for_real {
+				upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+			}
+		}
+	}
+
+	Ok(())
+}

upub/cli/src/register.rs

@@ -1,14 +1,14 @@
-use crate::server::admin::Administrable;
+use upub::traits::Administrable;
 
 pub async fn register(
-	ctx: crate::server::Context,
+	ctx: upub::Context,
 	username: String,
 	password: String,
 	display_name: Option<String>,
 	summary: Option<String>,
 	avatar_url: Option<String>,
 	banner_url: Option<String>,
-) -> crate::Result<()> {
+) -> Result<(), sea_orm::DbErr> {
 	ctx.register_user(
 		username.clone(),
 		password,

upub/cli/src/relay.rs

@@ -0,0 +1,204 @@
+use apb::{ActivityMut, BaseMut, ObjectMut};
+use sea_orm::{ActiveValue::{NotSet, Set}, DbErr, EntityTrait, QueryFilter, ColumnTrait};
+use upub::traits::{fetch::RequestError, Fetcher};
+
+#[derive(Debug, Clone, clap::Subcommand)]
+/// available actions to take on relays
+pub enum RelayCommand {
+	/// get all current pending and accepted relays
+	Status,
+	/// request to follow a specific relay
+	Follow {
+		/// relay actor to follow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// accept a pending relay request
+	Accept {
+		/// relay actor to accept (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// retract a follow relation to a relay, stopping receiving content
+	Unfollow {
+		/// relay actor to unfollow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// remove a follow relation from a relay, stopping sending content
+	Remove {
+		/// relay actor to unfollow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+}
+
+pub async fn relay(ctx: upub::Context, action: RelayCommand) -> Result<(), RequestError> {
+	let my_internal = upub::model::actor::Entity::ap_to_internal(ctx.base(), ctx.db())
+		.await?
+		.ok_or_else(|| DbErr::RecordNotFound(ctx.base().to_string()))?;
+
+	let their_internal = match &action {
+		RelayCommand::Status => 0,
+		RelayCommand::Follow { actor }
+		| RelayCommand::Accept { actor }
+		| RelayCommand::Unfollow { actor }
+		| RelayCommand::Remove { actor }
+		=> ctx.fetch_user(actor, ctx.db()).await?.internal,
+	};
+
+	match action {
+		RelayCommand::Status => {
+			tracing::info!("active sinks:");
+			for sink in upub::Query::related(None, Some(my_internal), false)
+				.into_model::<upub::model::actor::Model>()
+				.all(ctx.db())
+				.await?
+			{
+				tracing::info!("[>>] {} {}", sink.name.unwrap_or_default(), sink.id);
+			}
+
+			tracing::info!("active sources:");
+			for source in  upub::Query::related(Some(my_internal), None, false)
+				.into_model::<upub::model::actor::Model>()
+				.all(ctx.db())
+				.await?
+			{
+				tracing::info!("[<<] {} {}", source.name.unwrap_or_default(), source.id);
+			}
+		},
+
+		RelayCommand::Follow { actor } => {
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Follow))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::link(actor.clone()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("following relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Accept { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(their_internal))
+				.filter(upub::model::relation::Column::Following.eq(my_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{their_internal}-{my_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(relation.activity)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", relation.activity)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Accept(apb::AcceptType::Accept)))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::link(activity.id))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("accepting relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Remove { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(their_internal))
+				.filter(upub::model::relation::Column::Following.eq(my_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{their_internal}-{my_internal}")))?;
+			let accept_activity_id = relation.accept.ok_or(DbErr::RecordNotFound(format!("accept-{their_internal}-{my_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(accept_activity_id)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", accept_activity_id)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Undo))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::object(activity.ap()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("unfollowing relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Unfollow { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(my_internal))
+				.filter(upub::model::relation::Column::Following.eq(their_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{my_internal}-{their_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(relation.activity)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", relation.activity)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Undo))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::object(activity.ap()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("unfollowing relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+	}
+
+	Ok(())
+}

upub/cli/src/thread.rs

@@ -0,0 +1,34 @@
+use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter};
+use upub::traits::{fetch::RequestError, Fetcher};
+
+pub async fn thread(ctx: upub::Context) -> Result<(), RequestError> {
+	use futures::TryStreamExt;
+	let db = ctx.db();
+
+	tracing::info!("fixing contexts...");
+	let mut stream = upub::model::object::Entity::find()
+		.filter(upub::model::object::Column::Context.is_null())
+		.stream(db)
+		.await?;
+
+	while let Some(mut object) = stream.try_next().await? {
+		match object.in_reply_to {
+			None => object.context = Some(object.id.clone()),
+			Some(ref in_reply_to) => {
+				let reply = ctx.fetch_object(in_reply_to, ctx.db()).await?;
+				if let Some(context) = reply.context {
+					object.context = Some(context);
+				} else {
+					continue;
+				}
+			},
+		}
+		tracing::info!("updating context of {}", object.id);
+		upub::model::object::Entity::update(object.into_active_model())
+			.exec(ctx.db())
+			.await?;
+	}
+
+	tracing::info!("done fixing contexts");
+	Ok(())
+}

upub/cli/src/update.rs

@@ -0,0 +1,48 @@
+use futures::TryStreamExt;
+use sea_orm::{ActiveModelTrait, ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter, ModelTrait};
+use upub::traits::Fetcher;
+
+pub async fn update_users(ctx: upub::Context, days: i64, limit: Option<u64>) -> Result<(), sea_orm::DbErr> {
+	let mut count = 0;
+	let mut stream = upub::model::actor::Entity::find()
+		.filter(upub::model::actor::Column::Updated.lt(chrono::Utc::now() - chrono::Duration::days(days)))
+		.stream(ctx.db())
+		.await?;
+
+
+	while let Some(user) = stream.try_next().await? {
+		if ctx.is_local(&user.id) { continue }
+		if let Some(limit) = limit {
+			if count >= limit { break }
+		}
+		match ctx.pull(&user.id).await.and_then(|x| x.actor()) {
+			Err(upub::traits::fetch::RequestError::Fetch(status, msg)) => {
+				if status.as_u16() == 410 {
+					tracing::info!("user {} has been deleted", user.id);
+					user.delete(ctx.db()).await?;
+				}
+				else if status.as_u16() == 404 {
+					tracing::info!("user {} does not exist anymore", user.id);
+					user.delete(ctx.db()).await?;
+				}
+				else {
+					tracing::warn!("could not fetch user {}: failed with status {status} -- {msg}", user.id);
+				}
+			},
+			Err(e) => tracing::warn!("could not fetch user {}: {e}", user.id),
+			Ok(doc) => match upub::AP::actor_q(&doc, Some(user.internal)) {
+				Ok(mut u) => {
+					tracing::info!("updating user {}", user.id);
+					u.updated = Set(chrono::Utc::now());
+					u.update(ctx.db()).await?;
+					count += 1;
+				},
+				Err(e) => tracing::warn!("failed deserializing user '{}': {e}", user.id),
+			},
+		}
+	}
+
+	tracing::info!("updated {count} users");
+
+	Ok(())
+}

upub/core/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "upub"
+version = "0.3.0"
+edition = "2021"
+authors = [ "alemi <me@alemi.dev>" ]
+description = "core inner workings of upub"
+license = "AGPL-3.0"
+repository = "https://git.alemi.dev/upub.git"
+readme = "README.md"
+
+[lib]
+
+[dependencies]
+thiserror = "1"
+async-recursion = "1.1"
+async-trait = "0.1"
+sha256 = "1.5" # TODO get rid of this and use directly sha2!!
+sha2 = "0.10"
+hmac = "0.12"
+openssl = "0.10" # TODO handle pubkeys with a smaller crate
+base64 = "0.22"
+chrono = { version = "0.4", features = ["serde"] }
+uuid = { version = "1.10", features = ["v4"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_default = "0.2"
+serde-inline-default = "0.2"
+toml = "0.8"
+uriproxy = { path = "../../utils/uriproxy" }
+httpsign = { path = "../../utils/httpsign/" }
+mdhtml = { path = "../../utils/mdhtml/" }
+jrd = "0.1"
+tracing = "0.1"
+sea-orm = { version = "1.0", features = ["macros"] }
+reqwest = { version = "0.12", features = ["json"] }
+apb = { path = "../../apb", features = ["unstructured", "orm", "did-core", "activitypub-miscellaneous-terms", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
+# nodeinfo = "0.0.2" # the version on crates.io doesn't re-export necessary types to build the struct!!!
+nodeinfo = { git = "https://codeberg.org/thefederationinfo/nodeinfo-rs", rev = "e865094804" }

upub/core/README.md

@@ -0,0 +1 @@
+# upub core

upub/core/src/config.rs

@@ -12,6 +12,9 @@ pub struct Config {
 	#[serde(default)]
 	pub security: SecurityConfig,
 
+	#[serde(default)]
+	pub compat: CompatibilityConfig,
+
 	// TODO should i move app keys here?
 }
 
@@ -40,19 +43,19 @@ pub struct DatasourceConfig {
 	#[serde_inline_default("sqlite://./upub.db".into())]
 	pub connection_string: String,
 
-	#[serde_inline_default(4)]
+	#[serde_inline_default(32)]
 	pub max_connections: u32,
 
 	#[serde_inline_default(1)]
 	pub min_connections: u32,
 
-	#[serde_inline_default(300u64)]
+	#[serde_inline_default(90u64)]
 	pub connect_timeout_seconds: u64,
 
-	#[serde_inline_default(300u64)]
+	#[serde_inline_default(30u64)]
 	pub acquire_timeout_seconds: u64,
 
-	#[serde_inline_default(1u64)]
+	#[serde_inline_default(10u64)]
 	pub slow_query_warn_seconds: u64,
 
 	#[serde_inline_default(true)]
@@ -65,25 +68,55 @@ pub struct SecurityConfig {
 	#[serde(default)]
 	pub allow_registration: bool,
 
+	#[serde(default)] // TODO i don't like the name of this
+	pub require_user_approval: bool,
+
 	#[serde(default)]
 	pub allow_public_debugger: bool,
 
+	#[serde(default)]
+	pub allow_public_search: bool,
+
+	#[serde_inline_default("changeme".to_string())]
+	pub proxy_secret: String,
+
 	#[serde_inline_default(true)]
 	pub show_reply_ids: bool,
 
-	#[serde(default)]
+	#[serde_inline_default(true)]
 	pub allow_login_refresh: bool,
 
+	#[serde_inline_default(7 * 24)]
+	pub session_duration_hours: i64,
+
 	#[serde_inline_default(2)]
-	pub max_id_redirects: u32,
+	pub max_id_redirects: u32, // TODO not sure it fits here
 
 	#[serde_inline_default(20)]
-	pub thread_crawl_depth: u32,
+	pub thread_crawl_depth: u32, // TODO doesn't really fit here
+
+	#[serde_inline_default(30)]
+	pub job_expiration_days: u32, // TODO doesn't really fit here
+
+	#[serde_inline_default(100)]
+	pub reinsertion_attempt_limit: u32, // TODO doesn't really fit here
 }
 
+#[serde_inline_default::serde_inline_default]
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, serde_default::DefaultFromSerde)]
+pub struct CompatibilityConfig {
+	#[serde(default)]
+	pub fix_attachment_images_media_type: bool,
+
+	#[serde(default)]
+	pub add_explicit_target_to_likes_if_local: bool,
+
+	#[serde(default)]
+	pub skip_single_attachment_if_image_is_set: bool,
+}
 
 impl Config {
-	pub fn load(path: Option<std::path::PathBuf>) -> Self {
+	pub fn load(path: Option<&std::path::PathBuf>) -> Self {
 		let Some(cfg_path) = path else { return Config::default() };
 		match std::fs::read_to_string(cfg_path) {
 			Ok(x) => match toml::from_str(&x) {
@@ -94,4 +127,8 @@ impl Config {
 		}
 		Config::default()
 	}
+
+	pub fn frontend_url(&self, url: &str) -> Option<String> {
+		Some(format!("{}{}", self.instance.frontend.as_deref()?, url))
+	}
 }

upub/core/src/context.rs

@@ -1,13 +1,10 @@
 use std::{collections::BTreeSet, sync::Arc};
 
-use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QuerySelect, SelectColumns};
+use sea_orm::{DatabaseConnection, DbErr, QuerySelect, SelectColumns};
 
-use crate::{config::Config, errors::UpubError, model};
+use crate::{config::Config, model};
 use uriproxy::UriClass;
 
-use super::{builders::AnyQuery, dispatcher::Dispatcher};
-
-
 #[derive(Clone)]
 pub struct Context(Arc<ContextInner>);
 struct ContextInner {
@@ -16,11 +13,11 @@ struct ContextInner {
 	domain: String,
 	protocol: String,
 	base_url: String,
-	dispatcher: Dispatcher,
 	// TODO keep these pre-parsed
 	actor: model::actor::Model,
 	instance: model::instance::Model,
 	pkey: String,
+	waker: Option<Box<dyn WakerToken>>,
 	#[allow(unused)] relay: Relays,
 }
 
@@ -37,10 +34,14 @@ macro_rules! url {
 	};
 }
 
+pub trait WakerToken: Sync + Send {
+	fn wake(&self);
+}
+
 impl Context {
 
 	// TODO slim constructor down, maybe make a builder?
-	pub async fn new(db: DatabaseConnection, mut domain: String, config: Config) -> crate::Result<Self> {
+	pub async fn new(db: DatabaseConnection, mut domain: String, config: Config, waker: Option<Box<dyn WakerToken>>) -> Result<Self, crate::init::InitError> {
 		let protocol = if domain.starts_with("http://")
 		{ "http://" } else { "https://" }.to_string();
 		if domain.ends_with('/') {
@@ -49,19 +50,26 @@ impl Context {
 		if domain.starts_with("http") {
 			domain = domain.replace("https://", "").replace("http://", "");
 		}
-		let dispatcher = Dispatcher::default();
-		for _ in 0..1 { // TODO customize delivery workers amount
-			dispatcher.spawn(db.clone(), domain.clone(), 30); // TODO ew don't do it this deep and secretly!!
-		}
 		let base_url = format!("{}{}", protocol, domain);
 
 		let (actor, instance) = super::init::application(domain.clone(), base_url.clone(), &db).await?;
 
 		// TODO maybe we could provide a more descriptive error...
-		let pkey = actor.private_key.as_deref().ok_or_else(UpubError::internal_server_error)?.to_string();
+		let pkey = actor.private_key.as_deref().ok_or_else(|| DbErr::RecordNotFound("application private key".into()))?.to_string();
 
-		let relay_sinks = model::relation::Entity::followers(&actor.id, &db).await?;
+		let relay_sinks = crate::Query::related(None, Some(actor.internal), false)
-		let relay_sources = model::relation::Entity::following(&actor.id, &db).await?;
+			.select_only()
+			.select_column(crate::model::actor::Column::Id)
+			.into_tuple::<String>()
+			.all(&db)
+			.await?;
+
+		let relay_sources = crate::Query::related(Some(actor.internal), None, false)
+			.select_only()
+			.select_column(crate::model::actor::Column::Id)
+			.into_tuple::<String>()
+			.all(&db)
+			.await?;
 
 		let relay = Relays {
 			sources: BTreeSet::from_iter(relay_sources),
@@ -69,7 +77,7 @@ impl Context {
 		};
 
 		Ok(Context(Arc::new(ContextInner {
-			base_url, db, domain, protocol, actor, instance, dispatcher, config, pkey, relay,
+			base_url, db, domain, protocol, actor, instance, config, pkey, relay, waker,
 		})))
 	}
 
@@ -106,8 +114,8 @@ impl Context {
 		&self.0.base_url
 	}
 
-	pub fn dispatcher(&self) -> &Dispatcher {
+	pub fn new_id() -> String {
-		&self.0.dispatcher
+		uuid::Uuid::new_v4().to_string()
 	}
 
 	/// get full user id uri
@@ -128,9 +136,9 @@ impl Context {
 	/// get bare id, which is uuid for local stuff and +{uri|base64} for remote stuff
 	pub fn id(&self, full_id: &str) -> String {
 		if self.is_local(full_id) {
-			uriproxy::decompose_id(full_id)
+			uriproxy::decompose(full_id)
 		} else {
-			uriproxy::compact_id(full_id)
+			uriproxy::compact(full_id)
 		}
 	}
 
@@ -148,35 +156,26 @@ impl Context {
 		id.starts_with(self.base())
 	}
 
-	pub async fn is_local_internal_object(&self, internal: i64) -> crate::Result<bool> {
+	pub async fn find_internal(&self, id: &str) -> Result<Option<Internal>, DbErr> {
-		model::object::Entity::find()
+		if let Some(internal) = model::object::Entity::ap_to_internal(id, self.db()).await? {
-			.filter(model::object::Column::Internal.eq(internal))
+			return Ok(Some(Internal::Object(internal)));
-			.select_only()
+		}
-			.select_column(model::object::Column::Internal)
+
-			.into_tuple::<i64>()
+		if let Some(internal) = model::activity::Entity::ap_to_internal(id, self.db()).await? {
-			.any(self.db())
+			return Ok(Some(Internal::Activity(internal)));
-			.await
+		}
+
+		if let Some(internal) = model::actor::Entity::ap_to_internal(id, self.db()).await? {
+			return Ok(Some(Internal::Actor(internal)));
+		}
+
+		Ok(None)
 	}
 
-	pub async fn is_local_internal_activity(&self, internal: i64) -> crate::Result<bool> {
+	pub fn wake_workers(&self) {
-		model::activity::Entity::find()
+		if let Some(ref waker) = self.0.waker {
-			.filter(model::activity::Column::Internal.eq(internal))
+			waker.wake();
-			.select_only()
+		}
-			.select_column(model::activity::Column::Internal)
-			.into_tuple::<i64>()
-			.any(self.db())
-			.await
-	}
-
-	#[allow(unused)]
-	pub async fn is_local_internal_actor(&self, internal: i64) -> crate::Result<bool> {
-		model::actor::Entity::find()
-			.filter(model::actor::Column::Internal.eq(internal))
-			.select_only()
-			.select_column(model::actor::Column::Internal)
-			.into_tuple::<i64>()
-			.any(self.db())
-			.await
 	}
 
 	#[allow(unused)]
@@ -184,3 +183,9 @@ impl Context {
 		self.0.relay.sources.contains(id) || self.0.relay.sinks.contains(id)
 	}
 }
+
+pub enum Internal {
+	Object(i64),
+	Activity(i64),
+	Actor(i64),
+}

upub/core/src/ext.rs

@@ -0,0 +1,134 @@
+use sea_orm::{ConnectionTrait, PaginatorTrait};
+
+#[allow(async_fn_in_trait)]
+pub trait AnyQuery {
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr>;
+}
+
+impl<T : sea_orm::EntityTrait> AnyQuery for sea_orm::Select<T>
+where
+	T::Model : Sync,
+{
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr> {
+		// TODO ConnectionTrait became an iterator?? self.count(db) gives error now
+		Ok(PaginatorTrait::count(self, db).await? > 0)
+	}
+}
+
+impl<T : sea_orm::SelectorTrait + Send + Sync> AnyQuery for sea_orm::Selector<T> {
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr> {
+		Ok(self.count(db).await? > 0)
+	}
+}
+
+pub trait LoggableError {
+	fn info_failed(self, msg: &str);
+	fn warn_failed(self, msg: &str);
+	fn err_failed(self, msg: &str);
+}
+
+impl<T, E: std::error::Error> LoggableError for Result<T, E> {
+	fn info_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::info!("{} : {}", msg, e);
+		}
+	}
+
+	fn warn_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::warn!("{} : {}", msg, e);
+		}
+	}
+
+	fn err_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::error!("{} : {}", msg, e);
+		}
+	}
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct JsonVec<T>(pub Vec<T>);
+
+impl<T> From<Vec<T>> for JsonVec<T> {
+	fn from(value: Vec<T>) -> Self {
+		JsonVec(value)
+	}
+}
+
+impl<T> Default for JsonVec<T> {
+	fn default() -> Self {
+		JsonVec(Vec::new())
+	}
+}
+
+// TODO we need this dummy to access the default implementation, which needs to be wrapped to catch
+//      nulls. is there a way to directly call super::try_get_from_json ?? i think this gets
+//      compiled into a lot of variants...
+#[derive(serde::Deserialize)]
+struct DummyVec<T>(pub Vec<T>);
+impl<T: serde::de::DeserializeOwned> sea_orm::TryGetableFromJson for DummyVec<T> {}
+
+impl<T: serde::de::DeserializeOwned> sea_orm::TryGetableFromJson for JsonVec<T> {
+	fn try_get_from_json<I: sea_orm::ColIdx>(res: &sea_orm::QueryResult, idx: I) -> Result<Self, sea_orm::TryGetError> {
+		match DummyVec::try_get_from_json(res, idx) {
+			Ok(DummyVec(x)) => Ok(Self(x)),
+			Err(sea_orm::TryGetError::Null(_)) => Ok(Self::default()),
+			Err(e) => Err(e),
+		}
+	}
+
+	fn from_json_vec(value: serde_json::Value) -> Result<Vec<Self>, sea_orm::TryGetError> {
+		match DummyVec::from_json_vec(value) {
+			Ok(x) => Ok(x.into_iter().map(|x| JsonVec(x.0)).collect()),
+			Err(sea_orm::TryGetError::Null(_)) => Ok(vec![]),
+			Err(e) => Err(e),
+		}
+	}
+}
+
+impl<T: serde::ser::Serialize> std::convert::From<JsonVec<T>> for sea_orm::Value {
+	fn from(source: JsonVec<T>) -> Self {
+		sea_orm::Value::Json(serde_json::to_value(&source).ok().map(std::boxed::Box::new))
+	}
+}
+
+impl<T: serde::de::DeserializeOwned + TypeName> sea_orm::sea_query::ValueType for JsonVec<T> {
+	fn try_from(v: sea_orm::Value) -> Result<Self, sea_orm::sea_query::ValueTypeErr> {
+		match v {
+			sea_orm::Value::Json(Some(json)) => Ok(
+				serde_json::from_value(*json).map_err(|_| sea_orm::sea_query::ValueTypeErr)?,
+			),
+			sea_orm::Value::Json(None) => Ok(JsonVec::default()),
+			_ => Err(sea_orm::sea_query::ValueTypeErr),
+		}
+	}
+
+	fn type_name() -> String {
+		format!("JsonVec_{}", T::type_name())
+	}
+
+	fn array_type() -> sea_orm::sea_query::ArrayType {
+		sea_orm::sea_query::ArrayType::Json
+	}
+
+	fn column_type() -> sea_orm::sea_query::ColumnType {
+		sea_orm::sea_query::ColumnType::Json
+	}
+}
+
+impl<T> sea_orm::sea_query::Nullable for JsonVec<T> {
+	fn null() -> sea_orm::Value {
+		sea_orm::Value::Json(None)
+	}
+}
+
+pub trait TypeName {
+	fn type_name() -> String;
+}
+
+impl TypeName for String {
+	fn type_name() -> String {
+		"String".to_string()
+	}
+}

upub/core/src/init.rs

@@ -1,13 +1,25 @@
 use openssl::rsa::Rsa;
 use sea_orm::{ActiveValue::{NotSet, Set}, DatabaseConnection, EntityTrait};
 
-use crate::model;
+use crate::{ext::JsonVec, model};
+
+#[derive(Debug, thiserror::Error)]
+pub enum InitError {
+	#[error("database error: {0:?}")]
+	Database(#[from] sea_orm::DbErr),
+
+	#[error("openssl error: {0:?}")]
+	OpenSSL(#[from] openssl::error::ErrorStack),
+
+	#[error("pem format error: {0:?}")]
+	KeyError(#[from] std::str::Utf8Error),
+}
 
 pub async fn application(
 	domain: String,
 	base_url: String,
 	db: &DatabaseConnection
-) -> crate::Result<(model::actor::Model, model::instance::Model)> {
+) -> Result<(model::actor::Model, model::instance::Model), InitError> {
 	Ok((
 		match model::actor::Entity::find_by_ap_id(&base_url).one(db).await? {
 			Some(model) => model,
@@ -22,6 +34,9 @@ pub async fn application(
 					domain: Set(domain.clone()),
 					preferred_username: Set(domain.clone()),
 					actor_type: Set(apb::ActorType::Application),
+					also_known_as: Set(JsonVec::default()),
+					moved_to: Set(None),
+					fields: Set(JsonVec::default()), // TODO we could put some useful things here actually
 					private_key: Set(Some(privk)),
 					public_key: Set(pubk),
 					following: Set(None),

upub/core/src/lib.rs

@@ -0,0 +1,18 @@
+pub mod model;
+pub mod traits;
+
+pub mod context;
+pub use context::Context;
+
+pub mod config;
+pub use config::Config;
+
+pub mod init;
+pub mod ext;
+
+pub mod selector;
+pub use selector::Query;
+
+pub use traits::normalize::AP;
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");

upub/core/src/model/activity.rs

@@ -1,7 +1,7 @@
 use apb::{ActivityMut, ActivityType, BaseMut, ObjectMut};
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::{model::Audience, errors::UpubError, routes::activitypub::jsonld::LD};
+use crate::ext::JsonVec;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "activities")]
@@ -14,10 +14,10 @@ pub struct Model {
 	pub actor: String,
 	pub object: Option<String>,
 	pub target: Option<String>,
-	pub to: Audience,
+	pub to: JsonVec<String>,
-	pub bto: Audience,
+	pub bto: JsonVec<String>,
-	pub cc: Audience,
+	pub cc: JsonVec<String>,
-	pub bcc: Audience,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 }
 
@@ -33,8 +33,8 @@ pub enum Relation {
 	Actors,
 	#[sea_orm(has_many = "super::addressing::Entity")]
 	Addressing,
-	#[sea_orm(has_many = "super::delivery::Entity")]
+	#[sea_orm(has_many = "super::notification::Entity")]
-	Deliveries,
+	Notifications,
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
 		from = "Column::Object",
@@ -57,9 +57,9 @@ impl Related<super::addressing::Entity> for Entity {
 	}
 }
 
-impl Related<super::delivery::Entity> for Entity {
+impl Related<super::notification::Entity> for Entity {
 	fn to() -> RelationDef {
-		Relation::Deliveries.def()
+		Relation::Notifications.def()
 	}
 }
 
@@ -76,40 +76,20 @@ impl Entity {
 		Entity::find().filter(Column::Id.eq(id))
 	}
 
-	pub async fn ap_to_internal(id: &str, db: &DatabaseConnection) -> crate::Result<i64> {
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
 		Entity::find()
 			.filter(Column::Id.eq(id))
 			.select_only()
 			.select_column(Column::Internal)
 			.into_tuple::<i64>()
 			.one(db)
-			.await?
+			.await
-			.ok_or_else(UpubError::not_found)
-	}
-}
-
-impl ActiveModel {
-	//#[deprecated = "should remove this, get models thru normalizer"]
-	pub fn new(activity: &impl apb::Activity) -> Result<Self, super::FieldError> {
-		Ok(ActiveModel {
-			internal: sea_orm::ActiveValue::NotSet,
-			id: sea_orm::ActiveValue::Set(activity.id().ok_or(super::FieldError("id"))?.to_string()),
-			activity_type: sea_orm::ActiveValue::Set(activity.activity_type().ok_or(super::FieldError("type"))?),
-			actor: sea_orm::ActiveValue::Set(activity.actor().id().ok_or(super::FieldError("actor"))?),
-			object: sea_orm::ActiveValue::Set(activity.object().id()),
-			target: sea_orm::ActiveValue::Set(activity.target().id()),
-			published: sea_orm::ActiveValue::Set(activity.published().unwrap_or(chrono::Utc::now())),
-			to: sea_orm::ActiveValue::Set(activity.to().into()),
-			bto: sea_orm::ActiveValue::Set(activity.bto().into()),
-			cc: sea_orm::ActiveValue::Set(activity.cc().into()),
-			bcc: sea_orm::ActiveValue::Set(activity.bcc().into()),
-		})
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_activity_type(Some(self.activity_type))
 			.set_actor(apb::Node::link(self.actor))
@@ -131,4 +111,10 @@ impl apb::target::Addressed for Model {
 		to.append(&mut self.bcc.0.clone());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to = self.to.0.clone();
+		to.append(&mut self.bto.0.clone());
+		to
+	}
 }

upub/core/src/model/actor.rs

@@ -1,8 +1,37 @@
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use apb::{Actor, ActorMut, ActorType, BaseMut, DocumentMut, Endpoints, EndpointsMut, Object, ObjectMut, PublicKey, PublicKeyMut};
+use apb::{field::OptionalString, ActorMut, ActorType, BaseMut, DocumentMut, EndpointsMut, ObjectMut, PublicKeyMut};
 
-use crate::{errors::UpubError, routes::activitypub::jsonld::LD};
+use crate::ext::{JsonVec, TypeName};
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct Field {
+	#[serde(default)]
+	pub name: String,
+	#[serde(default)]
+	pub value: String,
+	#[serde(default, skip_serializing_if = "Option::is_none")]
+	pub verified_at: Option<ChronoDateTimeUtc>,
+	#[serde(default, rename = "type")]
+	pub field_type: String,
+}
+
+impl TypeName for Field {
+	fn type_name() -> String {
+		"Field".to_string()
+	}
+}
+
+impl<T: apb::Object> From<T> for Field {
+	fn from(value: T) -> Self {
+		Field {
+			name: value.name().str().unwrap_or_default(),
+			value: mdhtml::safe_html(value.value().unwrap_or_default()),
+			field_type: "PropertyValue".to_string(), // TODO can we try parsing this instead??
+			verified_at: None, // TODO where does verified_at come from? extend apb maybe
+		}
+	}
+}
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "actors")]
@@ -18,6 +47,7 @@ pub struct Model {
 	pub image: Option<String>,
 	pub icon: Option<String>,
 	pub preferred_username: String,
+	pub fields: JsonVec<Field>,
 	pub inbox: Option<String>,
 	pub shared_inbox: Option<String>,
 	pub outbox: Option<String>,
@@ -30,6 +60,8 @@ pub struct Model {
 	pub private_key: Option<String>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
+	pub also_known_as: JsonVec<String>,
+	pub moved_to: Option<String>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -44,8 +76,6 @@ pub enum Relation {
 	Configs,
 	#[sea_orm(has_many = "super::credential::Entity")]
 	Credentials,
-	#[sea_orm(has_many = "super::delivery::Entity")]
-	Deliveries,
 	#[sea_orm(
 		belongs_to = "super::instance::Entity",
 		from = "Column::Domain",
@@ -54,10 +84,14 @@ pub enum Relation {
 		on_delete = "NoAction"
 	)]
 	Instances,
+	#[sea_orm(has_many = "super::dislike::Entity")]
+	Dislikes,
 	#[sea_orm(has_many = "super::like::Entity")]
 	Likes,
 	#[sea_orm(has_many = "super::mention::Entity")]
 	Mentions,
+	#[sea_orm(has_many = "super::notification::Entity")]
+	Notifications,
 	#[sea_orm(has_many = "super::object::Entity")]
 	Objects,
 	#[sea_orm(has_many = "super::relation::Entity")]
@@ -96,18 +130,18 @@ impl Related<super::credential::Entity> for Entity {
 	}
 }
 
-impl Related<super::delivery::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Deliveries.def()
-	}
-}
-
 impl Related<super::instance::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Instances.def()
 	}
 }
 
+impl Related<super::dislike::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Dislikes.def()
+	}
+}
+
 impl Related<super::like::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Likes.def()
@@ -120,6 +154,12 @@ impl Related<super::mention::Entity> for Entity {
 	}
 }
 
+impl Related<super::notification::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Notifications.def()
+	}
+}
+
 impl Related<super::object::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Objects.def()
@@ -149,66 +189,42 @@ impl Entity {
 		Entity::delete_many().filter(Column::Id.eq(id))
 	}
 
-	pub async fn ap_to_internal(id: &str, db: &DatabaseConnection) -> crate::Result<i64> {
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
 		Entity::find()
 			.filter(Column::Id.eq(id))
 			.select_only()
 			.select_column(Column::Internal)
 			.into_tuple::<i64>()
 			.one(db)
-			.await?
+			.await
-			.ok_or_else(UpubError::not_found)
-	}
-}
-
-impl ActiveModel {
-	pub fn new(object: &impl Actor) -> Result<Self, super::FieldError> {
-		let ap_id = object.id().ok_or(super::FieldError("id"))?.to_string();
-		let (domain, fallback_preferred_username) = split_user_id(&ap_id);
-		Ok(ActiveModel {
-			internal: sea_orm::ActiveValue::NotSet,
-			domain: sea_orm::ActiveValue::Set(domain),
-			id: sea_orm::ActiveValue::Set(ap_id),
-			preferred_username: sea_orm::ActiveValue::Set(object.preferred_username().unwrap_or(&fallback_preferred_username).to_string()),
-			actor_type: sea_orm::ActiveValue::Set(object.actor_type().ok_or(super::FieldError("type"))?),
-			name: sea_orm::ActiveValue::Set(object.name().map(|x| x.to_string())),
-			summary: sea_orm::ActiveValue::Set(object.summary().map(|x| x.to_string())),
-			icon: sea_orm::ActiveValue::Set(object.icon().get().and_then(|x| x.url().id())),
-			image: sea_orm::ActiveValue::Set(object.image().get().and_then(|x| x.url().id())),
-			inbox: sea_orm::ActiveValue::Set(object.inbox().id()),
-			outbox: sea_orm::ActiveValue::Set(object.outbox().id()),
-			shared_inbox: sea_orm::ActiveValue::Set(object.endpoints().get().and_then(|x| Some(x.shared_inbox()?.to_string()))),
-			followers: sea_orm::ActiveValue::Set(object.followers().id()),
-			following: sea_orm::ActiveValue::Set(object.following().id()),
-			published: sea_orm::ActiveValue::Set(object.published().unwrap_or(chrono::Utc::now())),
-			updated: sea_orm::ActiveValue::Set(chrono::Utc::now()),
-			following_count: sea_orm::ActiveValue::Set(object.following_count().unwrap_or(0) as i32),
-			followers_count: sea_orm::ActiveValue::Set(object.followers_count().unwrap_or(0) as i32),
-			statuses_count: sea_orm::ActiveValue::Set(object.statuses_count().unwrap_or(0) as i32),
-			public_key: sea_orm::ActiveValue::Set(object.public_key().get().ok_or(super::FieldError("publicKey"))?.public_key_pem().to_string()),
-			private_key: sea_orm::ActiveValue::Set(None), // there's no way to transport privkey over AP json, must come from DB
-		})
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_actor_type(Some(self.actor_type))
 			.set_name(self.name.as_deref())
 			.set_summary(self.summary.as_deref())
 			.set_icon(apb::Node::maybe_object(self.icon.map(|i|
-				serde_json::Value::new_object()
+				apb::new()
 					.set_document_type(Some(apb::DocumentType::Image))
 					.set_url(apb::Node::link(i.clone()))
 			)))
 			.set_image(apb::Node::maybe_object(self.image.map(|i|
-				serde_json::Value::new_object()
+				apb::new()
 					.set_document_type(Some(apb::DocumentType::Image))
 					.set_url(apb::Node::link(i.clone()))
 			)))
+			.set_attachment(apb::Node::array(
+				self.fields.0
+					.into_iter()
+					.filter_map(|x| serde_json::to_value(x).ok())
+					.collect()
+			))
 			.set_published(Some(self.published))
+			.set_updated(if self.updated != self.published { Some(self.updated) } else { None })
 			.set_preferred_username(Some(&self.preferred_username))
 			.set_statuses_count(Some(self.statuses_count as u64))
 			.set_followers_count(Some(self.followers_count as u64))
@@ -218,25 +234,17 @@ impl Model {
 			.set_following(apb::Node::maybe_link(self.following))
 			.set_followers(apb::Node::maybe_link(self.followers))
 			.set_public_key(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_id(Some(&format!("{}#main-key", self.id)))
 					.set_owner(Some(&self.id))
 					.set_public_key_pem(&self.public_key)
 			))
 			.set_endpoints(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_shared_inbox(self.shared_inbox.as_deref())
 			))
+			.set_also_known_as(apb::Node::links(self.also_known_as.0))
+			.set_moved_to(apb::Node::maybe_link(self.moved_to))
 			.set_discoverable(Some(true))
 	}
 }
-
-fn split_user_id(id: &str) -> (String, String) {
-	let clean = id
-		.replace("http://", "")
-		.replace("https://", "");
-	let mut splits = clean.split('/');
-	let first = splits.next().unwrap_or("");
-	let last = splits.last().unwrap_or(first);
-	(first.to_string(), last.to_string())
-}

upub/core/src/model/addressing.rs

@@ -0,0 +1,75 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "addressing")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub actor: Option<i64>,
+	pub instance: Option<i64>,
+	pub activity: Option<i64>,
+	pub object: Option<i64>,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Activity",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Activities,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+	#[sea_orm(
+		belongs_to = "super::instance::Entity",
+		from = "Column::Instance",
+		to = "super::instance::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	Instances,
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::activity::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Activities.def()
+	}
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::instance::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Instances.def()
+	}
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}

upub/core/src/model/announce.rs

@@ -43,3 +43,9 @@ impl Related<super::object::Entity> for Entity {
 }
 
 impl ActiveModelBehavior for ActiveModel {}
+
+impl Entity {
+	pub fn find_by_uid_oid(uid: i64, oid: i64) -> Select<Entity> {
+		Entity::find().filter(Column::Actor.eq(uid)).filter(Column::Object.eq(oid))
+	}
+}

upub/core/src/model/attachment.rs

@@ -0,0 +1,45 @@
+use apb::{DocumentMut, DocumentType, ObjectMut};
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "attachments")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	#[sea_orm(unique)]
+	pub url: String,
+	pub object: i64,
+	pub document_type: DocumentType,
+	pub name: Option<String>,
+	pub media_type: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Model {
+	pub fn ap(self) -> serde_json::Value {
+		apb::new()
+			.set_url(apb::Node::link(self.url))
+			.set_document_type(Some(self.document_type))
+			.set_media_type(Some(&self.media_type))
+			.set_name(self.name.as_deref())
+	}
+}

upub/core/src/model/credential.rs

@@ -9,6 +9,7 @@ pub struct Model {
 	pub actor: String,
 	pub login: String,
 	pub password: String,
+	pub active: bool,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

upub/core/src/model/dislike.rs

@@ -0,0 +1,51 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "dislikes")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub actor: i64,
+	pub object: i64,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Entity {
+	pub fn find_by_uid_oid(uid: i64, oid: i64) -> Select<Entity> {
+		Entity::find().filter(Column::Actor.eq(uid)).filter(Column::Object.eq(oid))
+	}
+}

upub/core/src/model/hashtag.rs

@@ -7,7 +7,6 @@ pub struct Model {
 	pub internal: i64,
 	pub object: i64,
 	pub name: String,
-	pub published: ChronoDateTimeUtc,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

upub/core/src/model/instance.rs

@@ -1,8 +1,6 @@
 use nodeinfo::NodeInfoOwned;
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::errors::UpubError;
-
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "instances")]
 pub struct Model {
@@ -48,23 +46,24 @@ impl Entity {
 		Entity::find().filter(Column::Domain.eq(domain))
 	}
 
-	pub async fn domain_to_internal(domain: &str, db: &DatabaseConnection) -> crate::Result<i64> {
+	pub async fn domain_to_internal(domain: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
 		Entity::find()
 			.filter(Column::Domain.eq(domain))
 			.select_only()
 			.select_column(Column::Internal)
 			.into_tuple::<i64>()
 			.one(db)
-			.await?
+			.await
-			.ok_or_else(UpubError::not_found)
 	}
 
-	pub async fn nodeinfo(domain: &str) -> crate::Result<NodeInfoOwned> {
+	pub async fn nodeinfo(domain: &str) -> reqwest::Result<NodeInfoOwned> {
-		Ok(
+		match reqwest::get(format!("https://{domain}/nodeinfo/2.0.json")).await {
-			reqwest::get(format!("https://{domain}/nodeinfo/2.0.json"))
+			Ok(res) => res.json().await,
+			// ughhh pleroma wants with json, key without
+			Err(_) => reqwest::get(format!("https://{domain}/nodeinfo/2.0.json"))
 				.await?
 				.json()
-				.await?
+				.await,
-		)
+		}
 	}
 }

upub/core/src/model/job.rs

@@ -0,0 +1,59 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[sea_orm(rs_type = "i16", db_type = "SmallInteger")]
+pub enum JobType {
+	Inbound = 1,
+	Outbound = 2,
+	Delivery = 3,
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "jobs")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub job_type: JobType,
+	pub actor: String,
+	pub target: Option<String>,
+	pub activity: String,
+	pub payload: Option<serde_json::Value>,
+	pub published: ChronoDateTimeUtc,
+	pub not_before: ChronoDateTimeUtc,
+	pub attempt: i16,
+	pub error: Option<String>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Model {
+	pub fn next_attempt(&self) -> ChronoDateTimeUtc {
+		match self.attempt {
+			0 => chrono::Utc::now() + std::time::Duration::from_secs(10),
+			1 => chrono::Utc::now() + std::time::Duration::from_secs(60),
+			2 => chrono::Utc::now() + std::time::Duration::from_secs(5 * 60),
+			3 => chrono::Utc::now() + std::time::Duration::from_secs(20 * 60),
+			4 => chrono::Utc::now() + std::time::Duration::from_secs(60 * 60),
+			5 => chrono::Utc::now() + std::time::Duration::from_secs(12 * 60 * 60),
+			_ => chrono::Utc::now() + std::time::Duration::from_secs(24 * 60 * 60),
+		}
+	}
+
+	pub fn repeat(self, error: Option<String>) -> ActiveModel {
+		ActiveModel {
+			internal: sea_orm::ActiveValue::NotSet,
+			job_type: sea_orm::ActiveValue::Set(self.job_type),
+			not_before: sea_orm::ActiveValue::Set(self.next_attempt()),
+			actor: sea_orm::ActiveValue::Set(self.actor),
+			target: sea_orm::ActiveValue::Set(self.target),
+			payload: sea_orm::ActiveValue::Set(self.payload),
+			activity: sea_orm::ActiveValue::Set(self.activity),
+			published: sea_orm::ActiveValue::Set(self.published),
+			attempt: sea_orm::ActiveValue::Set(self.attempt + 1),
+			error: sea_orm::ActiveValue::Set(error),
+		}
+	}
+}

upub/core/src/model/like.rs

@@ -7,20 +7,11 @@ pub struct Model {
 	pub internal: i64,
 	pub actor: i64,
 	pub object: i64,
-	pub activity: i64,
 	pub published: ChronoDateTimeUtc,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
 pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Internal",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Activities,
 	#[sea_orm(
 		belongs_to = "super::actor::Entity",
 		from = "Column::Actor",
@@ -39,12 +30,6 @@ pub enum Relation {
 	Objects,
 }
 
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Activities.def()
-	}
-}
-
 impl Related<super::actor::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Actors.def()

upub/core/src/model/mention.rs

@@ -6,8 +6,7 @@ pub struct Model {
 	#[sea_orm(primary_key)]
 	pub internal: i64,
 	pub object: i64,
-	pub actor: String,
+	pub actor: i64,
-	pub published: ChronoDateTimeUtc,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

upub/core/src/model/mod.rs

@@ -0,0 +1,21 @@
+pub mod actor;
+pub mod object;
+pub mod activity;
+
+pub mod config;
+pub mod credential;
+pub mod session;
+
+pub mod instance;
+pub mod job;
+pub mod addressing;
+pub mod notification;
+pub mod relation;
+
+pub mod announce;
+pub mod like;
+pub mod dislike;
+
+pub mod hashtag;
+pub mod mention;
+pub mod attachment;

upub/core/src/model/notification.rs

@@ -0,0 +1,46 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "notifications")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub activity: i64,
+	pub actor: i64,
+	pub seen: bool,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Activity",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Activities,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::activity::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Activities.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}

upub/core/src/model/object.rs

@@ -1,9 +1,7 @@
-use apb::{BaseMut, Collection, CollectionMut, ObjectMut, ObjectType};
+use apb::{BaseMut, CollectionMut, DocumentMut, ObjectMut, ObjectType};
 use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::{errors::UpubError, routes::activitypub::jsonld::LD};
+use crate::ext::JsonVec;
-
-use super::Audience;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "objects")]
@@ -17,6 +15,8 @@ pub struct Model {
 	pub name: Option<String>,
 	pub summary: Option<String>,
 	pub content: Option<String>,
+	pub image: Option<String>,
+	pub quote: Option<String>,
 	pub sensitive: bool,
 	pub in_reply_to: Option<String>,
 	pub url: Option<String>,
@@ -24,12 +24,14 @@ pub struct Model {
 	pub announces: i32,
 	pub replies: i32,
 	pub context: Option<String>,
-	pub to: Audience,
+	pub to: JsonVec<String>,
-	pub bto: Audience,
+	pub bto: JsonVec<String>,
-	pub cc: Audience,
+	pub cc: JsonVec<String>,
-	pub bcc: Audience,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
+
+	pub audience: Option<String>, // added with migration m20240606_000001
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -50,6 +52,8 @@ pub enum Relation {
 	Announces,
 	#[sea_orm(has_many = "super::attachment::Entity")]
 	Attachments,
+	#[sea_orm(has_many = "super::dislike::Entity")]
+	Dislikes,
 	#[sea_orm(has_many = "super::hashtag::Entity")]
 	Hashtags,
 	#[sea_orm(has_many = "super::like::Entity")]
@@ -60,10 +64,18 @@ pub enum Relation {
 		belongs_to = "Entity",
 		from = "Column::InReplyTo",
 		to = "Column::Id",
-		on_update = "Cascade",
+		on_update = "NoAction",
 		on_delete = "NoAction"
 	)]
-	Objects,
+	ObjectsReply,
+	#[sea_orm(
+		belongs_to = "Entity",
+		from = "Column::Quote",
+		to = "Column::Id",
+		on_update = "NoAction",
+		on_delete = "NoAction"
+	)]
+	ObjectsQuote,
 }
 
 impl Related<super::activity::Entity> for Entity {
@@ -96,6 +108,12 @@ impl Related<super::attachment::Entity> for Entity {
 	}
 }
 
+impl Related<super::dislike::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Dislikes.def()
+	}
+}
+
 impl Related<super::hashtag::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Hashtags.def()
@@ -116,7 +134,7 @@ impl Related<super::mention::Entity> for Entity {
 
 impl Related<Entity> for Entity {
 	fn to() -> RelationDef {
-		Relation::Objects.def()
+		Relation::ObjectsReply.def()
 	}
 }
 
@@ -131,72 +149,38 @@ impl Entity {
 		Entity::delete_many().filter(Column::Id.eq(id))
 	}
 
-	pub async fn ap_to_internal(id: &str, db: &DatabaseConnection) -> crate::Result<i64> {
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
 		Entity::find()
 			.filter(Column::Id.eq(id))
 			.select_only()
 			.select_column(Column::Internal)
 			.into_tuple::<i64>()
 			.one(db)
-			.await?
+			.await
-			.ok_or_else(UpubError::not_found)
-	}
-}
-
-impl ActiveModel {
-	pub fn new(object: &impl apb::Object) -> Result<Self, super::FieldError> {
-		let t = object.object_type().ok_or(super::FieldError("type"))?;
-		if matches!(t,
-			apb::ObjectType::Activity(_)
-			| apb::ObjectType::Actor(_)
-			| apb::ObjectType::Collection(_)
-			| apb::ObjectType::Document(_)
-		) {
-			return Err(super::FieldError("type"));
-		}
-		Ok(ActiveModel {
-			internal: sea_orm::ActiveValue::NotSet,
-			id: sea_orm::ActiveValue::Set(object.id().ok_or(super::FieldError("id"))?.to_string()),
-			object_type: sea_orm::ActiveValue::Set(t),
-			attributed_to: sea_orm::ActiveValue::Set(object.attributed_to().id()),
-			name: sea_orm::ActiveValue::Set(object.name().map(|x| x.to_string())),
-			summary: sea_orm::ActiveValue::Set(object.summary().map(|x| x.to_string())),
-			content: sea_orm::ActiveValue::Set(object.content().map(|x| x.to_string())),
-			context: sea_orm::ActiveValue::Set(object.context().id()),
-			in_reply_to: sea_orm::ActiveValue::Set(object.in_reply_to().id()),
-			published: sea_orm::ActiveValue::Set(object.published().unwrap_or_else(chrono::Utc::now)),
-			updated: sea_orm::ActiveValue::Set(object.updated().unwrap_or_else(chrono::Utc::now)),
-			url: sea_orm::ActiveValue::Set(object.url().id()),
-			replies: sea_orm::ActiveValue::Set(object.replies().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i32),
-			likes: sea_orm::ActiveValue::Set(object.likes().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i32),
-			announces: sea_orm::ActiveValue::Set(object.shares().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i32),
-			to: sea_orm::ActiveValue::Set(object.to().into()),
-			bto: sea_orm::ActiveValue::Set(object.bto().into()),
-			cc: sea_orm::ActiveValue::Set(object.cc().into()),
-			bcc: sea_orm::ActiveValue::Set(object.bcc().into()),
-
-			sensitive: sea_orm::ActiveValue::Set(object.sensitive().unwrap_or(false)),
-		})
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_object_type(Some(self.object_type))
 			.set_attributed_to(apb::Node::maybe_link(self.attributed_to))
 			.set_name(self.name.as_deref())
 			.set_summary(self.summary.as_deref())
 			.set_content(self.content.as_deref())
+			.set_image(apb::Node::maybe_object(self.image.map(|x| 
+				apb::new()
+					.set_document_type(Some(apb::DocumentType::Image))
+					.set_url(apb::Node::link(x))
+			)))
 			.set_context(apb::Node::maybe_link(self.context.clone()))
 			.set_conversation(apb::Node::maybe_link(self.context.clone())) // duplicate context for mastodon
 			.set_in_reply_to(apb::Node::maybe_link(self.in_reply_to.clone()))
+			.set_quote_url(apb::Node::maybe_link(self.quote.clone()))
 			.set_published(Some(self.published))
-			.set_updated(Some(self.updated))
+			.set_updated(if self.updated != self.published { Some(self.updated) } else { None })
+			.set_audience(apb::Node::maybe_link(self.audience))
 			.set_to(apb::Node::links(self.to.0.clone()))
 			.set_bto(apb::Node::Empty)
 			.set_cc(apb::Node::links(self.cc.0.clone()))
@@ -204,17 +188,17 @@ impl Model {
 			.set_url(apb::Node::maybe_link(self.url))
 			.set_sensitive(Some(self.sensitive))
 			.set_shares(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
 					.set_total_items(Some(self.announces as u64))
 			))
 			.set_likes(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
 					.set_total_items(Some(self.likes as u64))
 			))
 			.set_replies(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
 					.set_total_items(Some(self.replies as u64))
 			))
@@ -229,4 +213,10 @@ impl apb::target::Addressed for Model {
 		to.append(&mut self.bcc.0.clone());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to = self.to.0.clone();
+		to.append(&mut self.bto.0.clone());
+		to
+	}
 }

upub/core/src/model/relation.rs

@@ -0,0 +1,86 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "relations")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub follower: i64,
+	pub following: i64,
+	pub accept: Option<i64>,
+	pub activity: i64,
+	pub follower_instance: i64,
+	pub following_instance: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Accept",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	ActivitiesAccept,
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Activity",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	ActivitiesFollow,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Follower",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	ActorsFollower,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Following",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	ActorsFollowing,
+	#[sea_orm(
+		belongs_to = "super::instance::Entity",
+		from = "Column::FollowerInstance",
+		to = "super::instance::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	InstancesFollower,
+	#[sea_orm(
+		belongs_to = "super::instance::Entity",
+		from = "Column::FollowingInstance",
+		to = "super::instance::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	InstancesFollowing,
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::ActorsFollowing.def()
+	}
+}
+
+impl Related<super::activity::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::ActivitiesFollow.def()
+	}
+}
+
+impl Related<super::instance::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::InstancesFollowing.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}