fix: object updates should only touch some fields

also reject updates for other types

.tci

@@ -1,19 +1,19 @@
 #!/bin/bash
 
 echo "building release binary"
-cargo build --release --all-features -j 1 # limit memory usage
+cargo build --release --all-features -j 4
 echo "stopping service"
 systemctl --user stop upub
 echo "installing new binary"
 cp ./target/release/upub /opt/bin/upub
 echo "migrating database"
-/opt/bin/upub --db "sqlite:///srv/tci/upub.db" --domain https://feditest.alemi.dev migrate
+/opt/bin/upub -c /etc/upub/config.toml migrate
 echo "restarting service"
 systemctl --user start upub
 echo "rebuilding frontend"
 cd web
-CARGO_BUILD_JOBS=1 /opt/bin/trunk build --release --public-url 'https://feditest.alemi.dev/web'
+CARGO_BUILD_JOBS=4 /opt/bin/trunk build --release --public-url 'https://dev.upub.social/web'
 echo "deploying frontend"
-rm /srv/http/feditest/web/*
-mv ./dist/* /srv/http/feditest/web/
+rm /srv/http/upub/dev/web/*
+mv ./dist/* /srv/http/upub/dev/web/
 echo "done"

Cargo.toml

@@ -1,9 +1,20 @@
 [workspace]
-members = ["apb", "web", "mdhtml", "uriproxy"]
+members = [
+	"apb",
+	"upub/core",
+	"upub/cli",
+	"upub/migrations",
+	"upub/routes",
+	"upub/worker",
+	"web",
+	"utils/httpsign",
+	"utils/mdhtml",
+	"utils/uriproxy",
+]
 
 [package]
-name = "upub"
-version = "0.2.0"
+name = "upub-bin"
+version = "0.3.0"
 edition = "2021"
 authors = [ "alemi <me@alemi.dev>" ]
 description = "Traits and types to handle ActivityPub objects"
@@ -12,46 +23,30 @@ keywords = ["activitypub", "activitystreams", "json"]
 repository = "https://git.alemi.dev/upub.git"
 readme = "README.md"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "upub"
+path = "main.rs"
 
 [dependencies]
-thiserror = "1"
-rand = "0.8"
-sha256 = "1.5"
-openssl = "0.10" # TODO handle pubkeys with a smaller crate
-base64 = "0.22"
-chrono = { version = "0.4", features = ["serde"] }
-uuid = { version = "1.8", features = ["v4"] }
-regex = "1.10"
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-serde_default = "0.1"
-serde-inline-default = "0.2"
 toml = "0.8"
-mdhtml = { path = "mdhtml", features = ["markdown"] }
-uriproxy = { path = "uriproxy" }
-jrd = "0.1"
 tracing = "0.1"
 tracing-subscriber = "0.3"
 clap = { version = "4.5", features = ["derive"] }
+signal-hook = "0.3"
+signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
+tokio = { version = "1.40", features = ["full"] } # TODO slim this down
+sea-orm = { version = "1.0", features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls"] }
 futures = "0.3"
-tokio = { version = "1.35", features = ["full"] } # TODO slim this down
-sea-orm = { version = "0.12", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"] }
-reqwest = { version = "0.12", features = ["json"] }
-axum = "0.7"
-tower-http = { version = "0.5", features = ["cors", "trace"] }
-apb = { path = "apb", features = ["unstructured", "orm", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
-# nodeinfo = "0.0.2" # the version on crates.io doesn't re-export necessary types to build the struct!!!
-nodeinfo = { git = "https://codeberg.org/thefederationinfo/nodeinfo-rs", rev = "e865094804" }
-# migrations
-sea-orm-migration = { version = "0.12", optional = true }
-# mastodon
-mastodon-async-entities = { version = "1.1.0", optional = true }
-time = { version = "0.3", features = ["serde"], optional = true }
-async-recursion = "1.1"
+
+upub = { path = "upub/core" }
+upub-cli = { path = "upub/cli", optional = true }
+upub-migrations = { path = "upub/migrations", optional = true }
+upub-routes = { path = "upub/routes", optional = true }
+upub-worker = { path = "upub/worker", optional = true }
 
 [features]
-default = ["mastodon", "migrations", "cli"]
-cli = []
-migrations = ["dep:sea-orm-migration"]
-mastodon = ["dep:mastodon-async-entities", "dep:time"]
+default = ["serve", "migrate", "cli", "worker"]
+serve = ["dep:upub-routes"]
+migrate = ["dep:upub-migrations"]
+cli = ["dep:upub-cli"]
+worker = ["dep:upub-worker"]

README.md

@@ -1,7 +1,7 @@
 # μpub
-> micro social network, federated
+> [micro social network, federated](https://join.upub.social)
 
-![screenshot of upub simple frontend](https://cdn.alemi.dev/proj/upub/fe/20240514.png)
+![screenshot of upub simple frontend](https://cdn.alemi.dev/proj/upub/fe/20240704.png)
 
 μpub aims to be a private, lightweight, modular and **secure** [ActivityPub](https://www.w3.org/TR/activitypub/) server
 
@@ -13,7 +13,7 @@ all interactions happen with ActivityPub's client-server methods (basically POST
 
 development is still active, so expect more stuff to come! since most fediverse software uses Mastodon's API, μpub plans to implement it as an optional feature, becoming eventually compatible with most existing frontends and mobile applications, but focus right now is on producing something specific to μpub needs
 
-a test instance is _usually_ available at [feditest.alemi.dev](https://feditest.alemi.dev)
+a test instance is available at [dev.upub.social](https://dev.upub.social)
 
 ## about the database schema
 im going to be very real i tried to do migrations but its getting super messy so until further notice assume db to be volatile. next change may be a migration (easy!) or a whole db rebuild (aaaaaaaaaa...), so if you're not comfortable with either manually exporting/importing or dropping and starting from scratch, **you really shouldn't put upub in prod yet**!
@@ -34,6 +34,33 @@ most instances will have "authorized fetch" which kind of makes the issue less b
 
 note that followers get expanded: addressing to example.net/actor/followers will address to anyone following actor that the server knows of, at that time
 
+## media caching
+μpub doesn't download remote media to both minimize local resources requirement and avoid storing media that remotes want gone. to prevent leaking local user ip addresses, all media links are cloaked and proxied.
+
+while this just works for small instances, larger servers should set up aggressive caching on `/proxy/...` path
+
+for example, on `nginx`:
+```nginx
+proxy_cache_path /tmp/upub/cache levels=1:2 keys_zone=upub_cache:100m max_size=50g inactive=168h use_temp_path=off;
+
+server {
+	location /proxy/ {
+		# use our configured cache
+		slice              1m;
+		proxy_set_header   Range $slice_range;
+		chunked_transfer_encoding on;
+		proxy_ignore_client_abort on;
+		proxy_buffering    on;
+		proxy_cache        upub_cache;
+		proxy_cache_key    $host$uri$is_args$args$slice_range;
+		proxy_cache_valid  200 206 301 304 168h;
+		proxy_cache_lock   on;
+		proxy_pass         http://127.0.0.1/;
+	}
+}
+
+```
+
 ## contributing
 
 all help is extremely welcome! development mostly happens on [moonlit.technology](https://moonlit.technology/alemi/upub.git), but there's a [github mirror](https://github.com/alemidev/upub) available too
@@ -62,18 +89,16 @@ don't hesitate to get in touch, i'd be thrilled to showcase the project to you!
  - [x] like, share, reply via frontend
  - [x] backend config
  - [x] frontend config
- - [ ] mentions, notifications
+ - [x] optimize `addressing` database schema
+ - [x] mentions, notifications
+ - [x] hashtags
+ - [x] remote media proxy
+ - [x] user fields
+ - [ ] better editing via web frontend
+ - [ ] upload media
+ - [ ] public vs unlisted for discovery
  - [ ] mastodon-like search bar
  - [ ] polls
- - [ ] better editing via web frontend
- - [ ] remote media proxy
- - [ ] upload media
- - [ ] hashtags
- - [ ] public vs unlisted for discovery
- - [ ] user fields
  - [ ] lists
  - [ ] full mastodon api
- - [ ] optimize `addressing` database schema
-
-## what about the name?
-μpub (or simply `upub`) means "[micro](https://en.wikipedia.org/wiki/International_System_of_Units#Prefixes)-pub", but could also be read "upub", "you-pub" or "mu-pub"
+ - [ ] get rid of internal ids from code

apb/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "apb"
-version = "0.1.1"
+version = "0.2.2"
 edition = "2021"
 authors = [ "alemi <me@alemi.dev>" ]
 description = "Traits and types to handle ActivityPub objects"
 license = "MIT"
 keywords = ["activitypub", "activitystreams", "json"]
-repository = "https://git.alemi.dev/upub.git"
+repository = "https://moonlit.technology/alemi/upub"
 readme = "README.md"
 
 [lib]
@@ -18,9 +18,8 @@ chrono = { version = "0.4", features = ["serde"] }
 thiserror = "1"
 paste = "1.0"
 tracing = "0.1"
-async-trait = "0.1"
 serde_json = { version = "1", optional = true }
-sea-orm = { version = "0.12", optional = true }
+sea-orm = { version = "1.0", optional = true, default-features = false }
 reqwest = { version = "0.12", features = ["json"], optional = true }
 
 [features]
@@ -32,6 +31,9 @@ activitypub-fe = [] # https://ns.alemi.dev/as/fe/#
 ostatus = [] # https://ostatus.org#  , but it redirects and 403??? just need this for conversation
 toot = [] # http://joinmastodon.org/ns#  , mastodon is weird tho??
 litepub = [] # incomplete, https://litepub.social/
+did-core = [] # incomplete, may be cool to support all of this: https://www.w3.org/TR/did-core/
+# full jsonld utilities
+jsonld = []
 # builtin utils
 send = []
 orm = ["dep:sea-orm"]

apb/src/field.rs

@@ -0,0 +1,18 @@
+#[derive(Debug, thiserror::Error)]
+#[error("missing field '{0}'")]
+pub struct FieldErr(pub &'static str);
+
+pub type Field<T> = Result<T, FieldErr>;
+
+
+// TODO this trait is really ad-hoc and has awful naming...
+
+pub trait OptionalString {
+	fn str(self) -> Option<String>;
+}
+
+impl OptionalString for Field<&str> {
+	fn str(self) -> Option<String> {
+		self.ok().map(|x| x.to_string())
+	}
+}

apb/src/jsonld.rs

@@ -1,16 +1,7 @@
-// TODO
-// move this file somewhere else
-// it's not a route
-// maybe under src/server/jsonld.rs ??
-
-use apb::Object;
-use axum::response::{IntoResponse, Response};
+use crate::Object;
 
 pub trait LD {
 	fn ld_context(self) -> Self;
-	fn new_object() -> serde_json::Value {
-		serde_json::Value::Object(serde_json::Map::default())
-	}
 }
 
 impl LD for serde_json::Value {
@@ -21,7 +12,7 @@ impl LD for serde_json::Value {
 			ctx.insert("sensitive".to_string(), serde_json::Value::String("as:sensitive".into()));
 			ctx.insert("quoteUrl".to_string(), serde_json::Value::String("as:quoteUrl".into()));
 			match o_type {
-				Some(apb::ObjectType::Actor(_)) => {
+				Ok(crate::ObjectType::Actor(_)) => {
 					ctx.insert("counters".to_string(), serde_json::Value::String("https://ns.alemi.dev/as/counters/#".into()));
 					ctx.insert("followingCount".to_string(), serde_json::Value::String("counters:followingCount".into()));
 					ctx.insert("followersCount".to_string(), serde_json::Value::String("counters:followersCount".into()));
@@ -30,18 +21,24 @@ impl LD for serde_json::Value {
 					ctx.insert("followingMe".to_string(), serde_json::Value::String("fe:followingMe".into()));
 					ctx.insert("followedByMe".to_string(), serde_json::Value::String("fe:followedByMe".into()));
 				},
-				Some(_) => {
+				Ok(
+					crate::ObjectType::Note
+					| crate::ObjectType::Article
+					| crate::ObjectType::Event
+					| crate::ObjectType::Document(crate::DocumentType::Page) // TODO why Document lemmyyyyyy
+				) => {
 					ctx.insert("fe".to_string(), serde_json::Value::String("https://ns.alemi.dev/as/fe/#".into()));
 					ctx.insert("likedByMe".to_string(), serde_json::Value::String("fe:likedByMe".into()));
 					ctx.insert("ostatus".to_string(), serde_json::Value::String("http://ostatus.org#".into()));
 					ctx.insert("conversation".to_string(), serde_json::Value::String("ostatus:conversation".into()));
 				},
-				None => {},
+				_ => {},
 			}
 			obj.insert(
 				"@context".to_string(),
 				serde_json::Value::Array(vec![
 					serde_json::Value::String("https://www.w3.org/ns/activitystreams".into()),
+					serde_json::Value::String("https://w3id.org/security/v1".into()),
 					serde_json::Value::Object(ctx),
 				]),
 			);
@@ -51,15 +48,3 @@ impl LD for serde_json::Value {
 		self
 	}
 }
-
-// got this from https://github.com/kitsune-soc/kitsune/blob/b023a12b687dd9a274233a5a9950f2de5e192344/kitsune/src/http/responder.rs
-// i was trying to do it with middlewares but this is way cleaner
-pub struct JsonLD<T>(pub T);
-impl<T: serde::Serialize> IntoResponse for JsonLD<T> {
-	fn into_response(self) -> Response {
-		(
-			[("Content-Type", "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")],
-			axum::Json(self.0)
-		).into_response()
-	}
-}

apb/src/key.rs

@@ -1,7 +1,7 @@
 // TODO technically this is not part of ActivityStreams
 
 pub trait PublicKey : super::Base {
-	fn owner(&self) -> Option<&str> { None }
+	fn owner(&self) -> crate::Field<&str> { Err(crate::FieldErr("owner")) }
 	fn public_key_pem(&self) -> &str;
 }
 

apb/src/lib.rs

@@ -88,17 +88,28 @@
 
 
 mod macros;
-pub(crate) use macros::{strenum, getter, setter};
+pub(crate) use macros::strenum;
+
+#[cfg(feature = "unstructured")]
+pub(crate) use macros::{getter, setter};
 
 mod node;
 pub use node::Node;
 
-pub mod server;
 pub mod target;
 
 mod key;
 pub use key::{PublicKey, PublicKeyMut};
 
+pub mod field;
+pub use field::{Field, FieldErr};
+
+#[cfg(feature = "jsonld")]
+mod jsonld;
+
+#[cfg(feature = "jsonld")]
+pub use jsonld::LD;
+
 mod types;
 pub use types::{
 	base::{Base, BaseMut, BaseType},
@@ -120,8 +131,13 @@ pub use types::{
 		},
 		document::{Document, DocumentMut, DocumentType},
 		place::{Place, PlaceMut},
-		// profile::Profile,
+		profile::Profile,
 		relationship::{Relationship, RelationshipMut},
 		tombstone::{Tombstone, TombstoneMut},
 	},
 };
+
+#[cfg(feature = "unstructured")]
+pub fn new() -> serde_json::Value {
+	serde_json::Value::Object(serde_json::Map::default())
+}

apb/src/macros.rs

@@ -38,6 +38,12 @@ macro_rules! strenum {
 				$($deep($inner),)*
 			}
 
+			impl std::fmt::Display for $enum_name {
+				fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+					write!(f, "{}", self.as_ref())
+				}
+			}
+
 			impl AsRef<str> for $enum_name {
 				fn as_ref(&self) -> &str {
 					match self {
@@ -91,7 +97,7 @@ macro_rules! strenum {
 				}
 			
 				fn column_type() -> sea_orm::sea_query::ColumnType {
-					sea_orm::sea_query::ColumnType::String(Some(24))
+					sea_orm::sea_query::ColumnType::String(sea_orm::sea_query::table::StringLen::N(24))
 				}
 			}
 			
@@ -108,114 +114,109 @@ macro_rules! strenum {
 
 pub(crate) use strenum;
 
+#[cfg(feature = "unstructured")]
 macro_rules! getter {
 	($name:ident -> type $t:ty) => {
-		fn $name(&self) -> Option<$t> {
-			self.get("type")?.as_str()?.try_into().ok()
-		}
-	};
-
-	($name:ident -> bool) => {
-		fn $name(&self) -> Option<bool> {
-			self.get(stringify!($name))?.as_bool()
-		}
-	};
-
-	($name:ident -> &str) => {
-		fn $name(&self) -> Option<&str> {
-			self.get(stringify!($name))?.as_str()
-		}
-	};
-
-	($name:ident::$rename:ident -> bool) => {
-		fn $name(&self) -> Option<bool> {
-			self.get(stringify!($rename))?.as_bool()
-		}
-	};
-
-	($name:ident::$rename:ident -> &str) => {
-		fn $name(&self) -> Option<&str> {
-			self.get(stringify!($rename))?.as_str()
-		}
-	};
-
-	($name:ident -> f64) => {
-		fn $name(&self) -> Option<f64> {
-			self.get(stringify!($name))?.as_f64()
-		}
-	};
-
-	($name:ident::$rename:ident -> f64) => {
-		fn $name(&self) -> Option<f64> {
-			self.get(stringify!($rename))?.as_f64()
-		}
-	};
-
-	($name:ident -> u64) => {
-		fn $name(&self) -> Option<u64> {
-			self.get(stringify!($name))?.as_u64()
-		}
-	};
-
-	($name:ident::$rename:ident -> u64) => {
-		fn $name(&self) -> Option<u64> {
-			self.get(stringify!($rename))?.as_u64()
-		}
-	};
-
-	($name:ident -> chrono::DateTime<chrono::Utc>) => {
-		fn $name(&self) -> Option<chrono::DateTime<chrono::Utc>> {
-			Some(
-				chrono::DateTime::parse_from_rfc3339(
-						self
-							.get(stringify!($name))?
-							.as_str()?
-					)
-					.ok()?
-					.with_timezone(&chrono::Utc)
-			)
-		}
-	};
-
-	($name:ident::$rename:ident -> chrono::DateTime<chrono::Utc>) => {
-		fn $name(&self) -> Option<chrono::DateTime<chrono::Utc>> {
-			Some(
-				chrono::DateTime::parse_from_rfc3339(
-						self
-							.get(stringify!($rename))?
-							.as_str()?
-					)
-					.ok()?
-					.with_timezone(&chrono::Utc)
-			)
-		}
-	};
-
-	($name:ident -> node $t:ty) => {
-		fn $name(&self) -> $crate::Node<$t> {
-			match self.get(stringify!($name)) {
-				Some(x) => $crate::Node::from(x.clone()),
-				None => $crate::Node::Empty,
+		paste::paste! {
+			fn [< $name:snake >] (&self) -> $crate::Field<$t> {
+				self.get("type")
+					.and_then(|x| x.as_str())
+					.and_then(|x| x.try_into().ok())
+					.ok_or($crate::FieldErr("type"))
 			}
 		}
 	};
 
-	($name:ident::$rename:ident -> node $t:ty) => {
-		fn $name(&self) -> $crate::Node<$t> {
-			match self.get(stringify!($rename)) {
-				Some(x) => $crate::Node::from(x.clone()),
-				None => $crate::Node::Empty,
+	($name:ident -> bool) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<bool> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_bool())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> &str) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<&str> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_str())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> f64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<f64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_f64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> u64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<u64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_u64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> i64) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<i64> {
+				self.get(stringify!($name))
+					.and_then(|x| x.as_i64())
+					.ok_or($crate::FieldErr(stringify!($name)))
+			}
+		}
+	};
+
+	($name:ident -> chrono::DateTime<chrono::Utc>) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Field<chrono::DateTime<chrono::Utc>> {
+				Ok(
+					chrono::DateTime::parse_from_rfc3339(
+							self
+								.get(stringify!($name))
+								.and_then(|x| x.as_str())
+								.ok_or($crate::FieldErr(stringify!($name)))?
+						)
+						.map_err(|e| {
+							tracing::warn!("invalid time string ({e}), ignoring");
+							$crate::FieldErr(stringify!($name))
+						})?
+						.with_timezone(&chrono::Utc)
+				)
+			}
+		}
+	};
+
+	($name:ident -> node $t:ty) => {
+		paste::paste! {
+			fn [< $name:snake >](&self) -> $crate::Node<$t> {
+				match self.get(stringify!($name)) {
+					Some(x) => $crate::Node::from(x.clone()),
+					None => $crate::Node::Empty,
+				}
 			}
 		}
 	};
 }
 
+#[cfg(feature = "unstructured")]
 pub(crate) use getter;
 
+#[cfg(feature = "unstructured")]
 macro_rules! setter {
 	($name:ident -> bool) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<bool>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<bool>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::Bool(x))
 				);
@@ -224,20 +225,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> bool) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<bool>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::Bool(x))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> &str) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<&str>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<&str>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::String(x.to_string()))
 				);
@@ -246,20 +236,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> &str) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<&str>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::String(x.to_string()))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> u64) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<u64>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<u64>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
 				);
@@ -268,11 +247,11 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> u64) => {
+	($name:ident -> i64) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<u64>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<i64>) -> Self {
 				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
+					&mut self, stringify!($name), val.map(|x| serde_json::Value::Number(serde_json::Number::from(x)))
 				);
 				self
 			}
@@ -281,7 +260,7 @@ macro_rules! setter {
 
 	($name:ident -> chrono::DateTime<chrono::Utc>) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, stringify!($name), val.map(|x| serde_json::Value::String(x.to_rfc3339()))
 				);
@@ -290,20 +269,9 @@ macro_rules! setter {
 		}
 	};
 
-	($name:ident::$rename:ident -> chrono::DateTime<chrono::Utc>) => {
-		paste::item! {
-			fn [< set_$name >](mut self, val: Option<chrono::DateTime<chrono::Utc>>) -> Self {
-				$crate::macros::set_maybe_value(
-					&mut self, stringify!($rename), val.map(|x| serde_json::Value::String(x.to_rfc3339()))
-				);
-				self
-			}
-		}
-	};
-
 	($name:ident -> node $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: $crate::Node<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: $crate::Node<$t>) -> Self {
 				$crate::macros::set_maybe_node(
 					&mut self, stringify!($name), val
 				);
@@ -314,7 +282,7 @@ macro_rules! setter {
 
 	($name:ident::$rename:ident -> node $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: $crate::Node<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: $crate::Node<$t>) -> Self {
 				$crate::macros::set_maybe_node(
 					&mut self, stringify!($rename), val
 				);
@@ -325,7 +293,7 @@ macro_rules! setter {
 
 	($name:ident -> type $t:ty ) => {
 		paste::item! {
-			fn [< set_$name >](mut self, val: Option<$t>) -> Self {
+			fn [< set_$name:snake >](mut self, val: Option<$t>) -> Self {
 				$crate::macros::set_maybe_value(
 					&mut self, "type", val.map(|x| serde_json::Value::String(x.as_ref().to_string()))
 				);
@@ -335,6 +303,7 @@ macro_rules! setter {
 	};
 }
 
+#[cfg(feature = "unstructured")]
 pub(crate) use setter;
 
 #[cfg(feature = "unstructured")]
@@ -357,49 +326,3 @@ pub fn set_maybe_value(obj: &mut serde_json::Value, key: &str, value: Option<ser
 		tracing::error!("error setting '{key}' on json Value: not an object");
 	}
 }
-
-#[cfg(feature = "unstructured")]
-pub(crate) trait InsertValue {
-	fn insert_node(&mut self, k: &str, v: crate::Node<serde_json::Value>);
-	fn insert_str(&mut self, k: &str, v: Option<&str>);
-	fn insert_float(&mut self, k: &str, f: Option<f64>);
-	fn insert_timestr(&mut self, k: &str, t: Option<chrono::DateTime<chrono::Utc>>);
-}
-
-#[cfg(feature = "unstructured")]
-impl InsertValue for serde_json::Map<String, serde_json::Value> {
-	fn insert_node(&mut self, k: &str, node: crate::Node<serde_json::Value>) {
-		if !node.is_nothing() {
-			self.insert(k.to_string(), node.into());
-		}
-	}
-
-	fn insert_str(&mut self, k: &str, v: Option<&str>) {
-		if let Some(v) = v {
-			self.insert(
-				k.to_string(),
-				serde_json::Value::String(v.to_string()),
-			);
-		}
-	}
-
-	fn insert_float(&mut self, k: &str, v: Option<f64>) {
-		if let Some(v) = v {
-			if let Some(n) = serde_json::Number::from_f64(v) {
-				self.insert(
-					k.to_string(),
-					serde_json::Value::Number(n),
-				);
-			}
-		}
-	}
-
-	fn insert_timestr(&mut self, k: &str, t: Option<chrono::DateTime<chrono::Utc>>) {
-		if let Some(published) = t {
-			self.insert(
-				k.to_string(),
-				serde_json::Value::String(published.to_rfc3339()),
-			);
-		}
-	}
-}

apb/src/node.rs

@@ -1,9 +1,13 @@
 /// ActivityPub object node, representing either nothing, something, a link to something or
 /// multiple things
 pub enum Node<T : super::Base> {
+	/// this document node holds multiple objects
 	Array(std::collections::VecDeque<Node<T>>), // TODO would be cool to make it Box<[T]> so that Node is just a ptr
+	/// this document node holds one object
 	Object(Box<T>),
+	/// this document node holds a reference to an object
 	Link(Box<dyn crate::Link + Sync + Send>), // TODO feature flag to toggle these maybe?
+	/// this document node is not present
 	Empty,
 }
 
@@ -99,21 +103,21 @@ impl<T : super::Base> Node<T> {
 	}
 
 	/// returns id of object: url for link, id for object, None if empty or array
-	pub fn id(&self) -> Option<String> {
+	pub fn id(&self) -> crate::Field<&str> {
 		match self {
-			Node::Empty => None,
-			Node::Link(uri) => Some(uri.href().to_string()),
-			Node::Object(obj) => Some(obj.id()?.to_string()),
-			Node::Array(arr) => Some(arr.front()?.id()?.to_string()),
+			Node::Empty => Err(crate::FieldErr("id")),
+			Node::Link(uri) => uri.href(),
+			Node::Object(obj) => obj.id(),
+			Node::Array(arr) => arr.front().map(|x| x.id()).ok_or(crate::FieldErr("id"))?,
 		}
 	}
 
-	pub fn ids(&self) -> Vec<String> {
+	pub fn all_ids(&self) -> Vec<String> {
 		match self {
 			Node::Empty => vec![],
-			Node::Link(uri) => vec![uri.href().to_string()],
+			Node::Link(uri) => uri.href().map(|x| vec![x.to_string()]).unwrap_or_default(),
 			Node::Object(x) => x.id().map_or(vec![], |x| vec![x.to_string()]),
-			Node::Array(x) => x.iter().filter_map(Self::id).collect()
+			Node::Array(x) => x.iter().filter_map(|x| Some(x.id().ok()?.to_string())).collect()
 		}
 	}
 
@@ -169,6 +173,14 @@ impl Node<serde_json::Value> {
 		)
 	}
 
+	pub fn maybe_array(values: Vec<serde_json::Value>) -> Self {
+		if values.is_empty() {
+			Node::Empty
+		} else {
+			Node::array(values)
+		}
+	}
+
 	#[cfg(feature = "fetch")]
 	pub async fn fetch(&mut self) -> reqwest::Result<&mut Self> {
 		if let Node::Link(link) = self {
@@ -205,6 +217,7 @@ impl From<&str> for Node<serde_json::Value> {
 #[cfg(feature = "unstructured")]
 impl From<serde_json::Value> for Node<serde_json::Value> {
 	fn from(value: serde_json::Value) -> Self {
+		use crate::Link;
 		match value {
 			serde_json::Value::String(uri) => Node::Link(Box::new(uri)),
 			serde_json::Value::Array(arr) => Node::Array(
@@ -213,9 +226,9 @@ impl From<serde_json::Value> for Node<serde_json::Value> {
 						.map(Node::from)
 				)
 			),
-			serde_json::Value::Object(_) => match value.get("href") {
-				None => Node::Object(Box::new(value)),
-				Some(_) => Node::Link(Box::new(value)),
+			serde_json::Value::Object(_) => match value.link_type() {
+				Ok(_) => Node::Link(Box::new(value)),
+				Err(_) => Node::Object(Box::new(value)),
 			},
 			_ => Node::Empty,
 		}
@@ -227,7 +240,7 @@ impl From<Node<serde_json::Value>> for serde_json::Value {
 	fn from(value: Node<serde_json::Value>) -> Self {
 		match value {
 			Node::Empty => serde_json::Value::Null,
-			Node::Link(l) => serde_json::Value::String(l.href().to_string()), // TODO there could be more
+			Node::Link(l) => serde_json::Value::String(l.href().unwrap_or_default().to_string()), // TODO there could be more
 			Node::Object(o) => *o,
 			Node::Array(arr) =>
 				serde_json::Value::Array(arr.into_iter().map(|x| x.into()).collect()),

apb/src/server.rs

@@ -1,33 +0,0 @@
-#[async_trait::async_trait]
-pub trait Outbox {
-	type Object: crate::Object;
-	type Activity: crate::Activity;
-	type Error: std::error::Error;
-
-	async fn create_note(&self, uid: String, object: Self::Object) -> Result<String, Self::Error>;
-	async fn create(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn like(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn follow(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn announce(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn accept(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn reject(&self, _uid: String, _activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn undo(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn delete(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-	async fn update(&self, uid: String, activity: Self::Activity) -> Result<String, Self::Error>;
-}
-
-#[async_trait::async_trait]
-pub trait Inbox {
-	type Activity: crate::Activity;
-	type Error: std::error::Error;
-
-	async fn create(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn like(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn follow(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn announce(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn accept(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn reject(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn undo(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn delete(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-	async fn update(&self, server: String, activity: Self::Activity) -> Result<(), Self::Error>;
-}

apb/src/target.rs

@@ -3,15 +3,99 @@ use crate::Object;
 pub const PUBLIC : &str = "https://www.w3.org/ns/activitystreams#Public";
 
 pub trait Addressed {
-	fn addressed(&self) -> Vec<String>;
+	fn addressed(&self) -> Vec<String>; // TODO rename this? remate others? idk
+	fn mentioning(&self) -> Vec<String>;
+	// fn secondary_targets(&self) -> Vec<String>;
+	// fn public_targets(&self) -> Vec<String>;
+	// fn private_targets(&self) -> Vec<String>;
 }
 
 impl<T: Object> Addressed for T {
 	fn addressed(&self) -> Vec<String> {
-		let mut to : Vec<String> = self.to().ids();
-		to.append(&mut self.bto().ids());
-		to.append(&mut self.cc().ids());
-		to.append(&mut self.bcc().ids());
+		let mut to : Vec<String> = self.to().all_ids();
+		to.append(&mut self.bto().all_ids());
+		to.append(&mut self.cc().all_ids());
+		to.append(&mut self.bcc().all_ids());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to : Vec<String> = self.to().all_ids();
+		to.append(&mut self.bto().all_ids());
+		to
+	}
+
+	// fn secondary_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.cc().ids();
+	// 	to.append(&mut self.bcc().ids());
+	// 	to
+	// }
+
+	// fn public_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.to().ids();
+	// 	to.append(&mut self.cc().ids());
+	// 	to
+	// }
+
+	// fn private_targets(&self) -> Vec<String> {
+	// 	let mut to : Vec<String> = self.bto().ids();
+	// 	to.append(&mut self.bcc().ids());
+	// 	to
+	// }
+}
+
+#[cfg(test)]
+mod test {
+	use super::Addressed;
+
+	#[test]
+	#[cfg(feature = "unstructured")]
+	fn addressed_trait_finds_all_targets_on_json_objects() {
+		let obj = serde_json::json!({
+			"id": "http://localhost:8080/obj/1",
+			"type": "Note",
+			"content": "hello world!",
+			"published": "2024-06-04T17:09:20+00:00",
+			"to": ["http://localhost:8080/usr/root/followers"],
+			"bto": ["https://localhost:8080/usr/secret"],
+			"cc": [crate::target::PUBLIC],
+			"bcc": [],
+		});
+
+		let addressed = obj.addressed();
+
+		assert_eq!(
+			addressed,
+			vec![
+				"http://localhost:8080/usr/root/followers".to_string(), 
+				"https://localhost:8080/usr/secret".to_string(),
+				crate::target::PUBLIC.to_string(),
+			]
+		);
+	}
+
+	#[test]
+	#[cfg(feature = "unstructured")]
+	fn primary_targets_only_finds_to_and_bto() {
+		let obj = serde_json::json!({
+			"id": "http://localhost:8080/obj/1",
+			"type": "Note",
+			"content": "hello world!",
+			"published": "2024-06-04T17:09:20+00:00",
+			"to": ["http://localhost:8080/usr/root/followers"],
+			"bto": ["https://localhost:8080/usr/secret"],
+			"cc": [crate::target::PUBLIC],
+			"bcc": [],
+		});
+
+		let addressed = obj.mentioning();
+
+		assert_eq!(
+			addressed,
+			vec![
+				"http://localhost:8080/usr/root/followers".to_string(), 
+				"https://localhost:8080/usr/secret".to_string(),
+			]
+		);
+	}
 }

apb/src/types/base.rs

@@ -9,8 +9,8 @@ crate::strenum! {
 }
 
 pub trait Base : crate::macros::MaybeSend {
-	fn id(&self) -> Option<&str> { None }
-	fn base_type(&self) -> Option<BaseType> { None }
+	fn id(&self) -> crate::Field<&str> { Err(crate::FieldErr("id")) }
+	fn base_type(&self) -> crate::Field<BaseType> { Err(crate::FieldErr("type")) }
 }
 
 
@@ -21,30 +21,35 @@ pub trait BaseMut : crate::macros::MaybeSend {
 
 
 impl Base for String {
-	fn id(&self) -> Option<&str> {
-		Some(self)
+	fn id(&self) -> crate::Field<&str> {
+		Ok(self)
 	}
 
-	fn base_type(&self) -> Option<BaseType> {
-		Some(BaseType::Link(LinkType::Link))
+	fn base_type(&self) -> crate::Field<BaseType> {
+		Ok(BaseType::Link(LinkType::Link))
 	}
 }
 
 #[cfg(feature = "unstructured")]
 impl Base for serde_json::Value {
-	fn base_type(&self) -> Option<BaseType> {
+	fn base_type(&self) -> crate::Field<BaseType> {
 		if self.is_string() {
-			Some(BaseType::Link(LinkType::Link))
+			Ok(BaseType::Link(LinkType::Link))
 		} else {
-			self.get("type")?.as_str()?.try_into().ok()
+			self.get("type")
+				.and_then(|x| x.as_str())
+				.and_then(|x| x.try_into().ok())
+				.ok_or(crate::FieldErr("type"))
 		}
 	}
 
-	fn id(&self) -> Option<&str> {
+	fn id(&self) -> crate::Field<&str> {
 		if self.is_string() {
-			self.as_str()
+			Ok(self.as_str().ok_or(crate::FieldErr("id"))?)
 		} else {
-			self.get("id").map(|x| x.as_str())?
+			self.get("id")
+				.and_then(|x| x.as_str())
+				.ok_or(crate::FieldErr("id"))
 		}
 	}
 }

apb/src/types/link.rs

@@ -1,3 +1,5 @@
+use crate::{Field, FieldErr};
+
 #[cfg(feature = "activitypub-miscellaneous-terms")]
 crate::strenum! {
 	pub enum LinkType {
@@ -16,73 +18,80 @@ crate::strenum! {
 }
 
 pub trait Link : crate::Base {
-	fn href(&self) -> &str;
-	fn rel(&self) -> Option<&str> { None }
-	fn link_media_type(&self) -> Option<&str> { None } // also in obj
-	fn link_name(&self) -> Option<&str> { None }       // also in obj
-	fn hreflang(&self) -> Option<&str> { None }
-	fn height(&self) -> Option<u64> { None }
-	fn width(&self) -> Option<u64> { None }
-	fn link_preview(&self) -> Option<&str> { None }    // also in obj
+	fn link_type(&self) -> Field<LinkType> { Err(FieldErr("type")) }
+	fn href(&self) -> Field<&str>;
+	fn rel(&self) -> Field<&str> { Err(FieldErr("rel")) }
+	fn media_type(&self) -> Field<&str> { Err(FieldErr("mediaType")) } // also in obj
+	fn name(&self) -> Field<&str> { Err(FieldErr("name")) }       // also in obj
+	fn hreflang(&self) -> Field<&str> { Err(FieldErr("hreflang")) }
+	fn height(&self) -> Field<u64> { Err(FieldErr("height")) }
+	fn width(&self) -> Field<u64> { Err(FieldErr("width")) }
+	fn preview(&self) -> Field<&str> { Err(FieldErr("linkPreview")) }    // also in obj
 }
 
 pub trait LinkMut : crate::BaseMut {
-	fn set_href(self, href: &str) -> Self;
+	fn set_link_type(self, val: Option<LinkType>) -> Self;
+	fn set_href(self, href: Option<&str>) -> Self;
 	fn set_rel(self, val: Option<&str>) -> Self;
-	fn set_link_media_type(self, val: Option<&str>) -> Self; // also in obj
-	fn set_link_name(self, val: Option<&str>) -> Self;       // also in obj
+	fn set_media_type(self, val: Option<&str>) -> Self; // also in obj
+	fn set_name(self, val: Option<&str>) -> Self;       // also in obj
 	fn set_hreflang(self, val: Option<&str>) -> Self;
 	fn set_height(self, val: Option<u64>) -> Self;
 	fn set_width(self, val: Option<u64>) -> Self;
-	fn set_link_preview(self, val: Option<&str>) -> Self;    // also in obj
+	fn set_preview(self, val: Option<&str>) -> Self;    // also in obj
 }
 
 impl Link for String {
-	fn href(&self) -> &str {
-		self
+	fn href(&self) -> Field<&str> {
+		Ok(self)
 	}
 }
 
 #[cfg(feature = "unstructured")]
 impl Link for serde_json::Value {
 	// TODO this can fail, but it should never do!
-	fn href(&self) -> &str {
+	fn href(&self) -> Field<&str> {
 		if self.is_string() {
-			self.as_str().unwrap_or("")
+			self.as_str().ok_or(FieldErr("href"))
 		} else {
-			self.get("href").map(|x| x.as_str().unwrap_or("")).unwrap_or("")
+			self.get("href").and_then(|x| x.as_str()).ok_or(FieldErr("href"))
 		}
 	}
 
+	crate::getter! { link_type -> type LinkType }
 	crate::getter! { rel -> &str }
-	crate::getter! { link_media_type::mediaType -> &str }
-	crate::getter! { link_name::name -> &str }
+	crate::getter! { mediaType -> &str }
+	crate::getter! { name -> &str }
 	crate::getter! { hreflang -> &str }
 	crate::getter! { height -> u64 }
 	crate::getter! { width -> u64 }
-	crate::getter! { link_preview::preview -> &str }
+	crate::getter! { preview -> &str }
 }
 
 #[cfg(feature = "unstructured")]
 impl LinkMut for serde_json::Value {
-	fn set_href(mut self, href: &str) -> Self {
+	fn set_href(mut self, href: Option<&str>) -> Self {
 		match &mut self {
 			serde_json::Value::Object(map) => {
-				map.insert(
-					"href".to_string(),
-					serde_json::Value::String(href.to_string())
-				);
+				match href {
+					Some(href) => map.insert(
+						"href".to_string(),
+						serde_json::Value::String(href.to_string())
+					),
+					None => map.remove("href"),
+				};
 			},
-			x => *x = serde_json::Value::String(href.to_string()),
+			x => *x = serde_json::Value::String(href.unwrap_or_default().to_string()),
 		}
 		self
 	}
 
+	crate::setter! { link_type -> type LinkType }
 	crate::setter! { rel -> &str }
-	crate::setter! { link_media_type::mediaType -> &str }
-	crate::setter! { link_name::name -> &str }
+	crate::setter! { mediaType -> &str }
+	crate::setter! { name -> &str }
 	crate::setter! { hreflang -> &str }
 	crate::setter! { height -> u64 }
 	crate::setter! { width -> u64 }
-	crate::setter! { link_preview::preview -> &str }
+	crate::setter! { preview -> &str }
 }

apb/src/types/object/activity/accept.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Accept : super::Activity {
-	fn accept_type(&self) -> Option<AcceptType> { None }
+	fn accept_type(&self) -> crate::Field<AcceptType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait AcceptMut : super::ActivityMut {

apb/src/types/object/activity/ignore.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Ignore : super::Activity {
-	fn ignore_type(&self) -> Option<IgnoreType> { None }
+	fn ignore_type(&self) -> crate::Field<IgnoreType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait IgnoreMut : super::ActivityMut {

apb/src/types/object/activity/intransitive.rs

@@ -10,7 +10,7 @@ strenum! {
 }
 
 pub trait IntransitiveActivity : super::Activity {
-	fn intransitive_activity_type(&self) -> Option<IntransitiveActivityType> { None }
+	fn intransitive_activity_type(&self) -> crate::Field<IntransitiveActivityType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait IntransitiveActivityMut : super::ActivityMut {

apb/src/types/object/activity/mod.rs

@@ -4,7 +4,7 @@ pub mod intransitive;
 pub mod offer;
 pub mod reject;
 
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 use accept::AcceptType;
 use reject::RejectType;
 use offer::OfferType;
@@ -73,13 +73,29 @@ crate::strenum! {
 }
 
 pub trait Activity : Object {
-	fn activity_type(&self) -> Option<ActivityType> { None }
+	fn activity_type(&self) -> Field<ActivityType> { Err(FieldErr("type")) }
+	/// Describes one or more entities that either performed or are expected to perform the activity.
+	/// Any single activity can have multiple actors. The actor MAY be specified using an indirect Link. 
 	fn actor(&self) -> Node<Self::Actor> { Node::Empty }
+	/// Describes an object of any kind.
+	/// The Object type serves as the base type for most of the other kinds of objects defined in the Activity Vocabulary, including other Core types such as Activity, IntransitiveActivity, Collection and OrderedCollection. 
 	fn object(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes the indirect object, or target, of the activity.
+	/// The precise meaning of the target is largely dependent on the type of action being described but will often be the object of the English preposition "to".
+	/// For instance, in the activity "John added a movie to his wishlist", the target of the activity is John's wishlist. An activity can have more than one target. 
 	fn target(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes the result of the activity.
+	/// For instance, if a particular action results in the creation of a new resource, the result property can be used to describe that new resource. 
 	fn result(&self) -> Node<Self::Object> { Node::Empty }
+	/// Describes an indirect object of the activity from which the activity is directed.
+	/// The precise meaning of the origin is the object of the English preposition "from".
+	/// For instance, in the activity "John moved an item to List B from List A", the origin of the activity is "List A".
 	fn origin(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies one or more objects used (or to be used) in the completion of an Activity.
 	fn instrument(&self) -> Node<Self::Object> { Node::Empty }
+
+	#[cfg(feature = "activitypub-fe")]
+	fn seen(&self) -> Field<bool> { Err(FieldErr("seen")) }
 }
 
 pub trait ActivityMut : ObjectMut {
@@ -90,6 +106,9 @@ pub trait ActivityMut : ObjectMut {
 	fn set_result(self, val: Node<Self::Object>) -> Self;
 	fn set_origin(self, val: Node<Self::Object>) -> Self;
 	fn set_instrument(self, val: Node<Self::Object>) -> Self;
+
+	#[cfg(feature = "activitypub-fe")]
+	fn set_seen(self, val: Option<bool>) -> Self;
 }
 
 #[cfg(feature = "unstructured")]
@@ -101,6 +120,9 @@ impl Activity for serde_json::Value {
 	crate::getter! { result -> node <Self as Object>::Object }
 	crate::getter! { origin -> node <Self as Object>::Object }
 	crate::getter! { instrument -> node <Self as Object>::Object }
+
+	#[cfg(feature = "activitypub-fe")]
+	crate::getter! { seen -> bool }
 }
 
 #[cfg(feature = "unstructured")]
@@ -112,4 +134,7 @@ impl ActivityMut for serde_json::Value {
 	crate::setter! { result -> node <Self as Object>::Object }
 	crate::setter! { origin -> node <Self as Object>::Object }
 	crate::setter! { instrument -> node <Self as Object>::Object }
+
+	#[cfg(feature = "activitypub-fe")]
+	crate::setter! { seen -> bool }
 }

apb/src/types/object/activity/offer.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Offer : super::Activity {
-	fn offer_type(&self) -> Option<OfferType> { None }
+	fn offer_type(&self) -> crate::Field<OfferType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait OfferMut : super::ActivityMut {

apb/src/types/object/activity/reject.rs

@@ -8,7 +8,7 @@ strenum! {
 }
 
 pub trait Reject : super::Activity {
-	fn reject_type(&self) -> Option<RejectType> { None }
+	fn reject_type(&self) -> crate::Field<RejectType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait RejectMut : super::ActivityMut {

apb/src/types/object/actor.rs

@@ -1,4 +1,4 @@
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 
 crate::strenum! {
 	pub enum ActorType {
@@ -14,51 +14,67 @@ pub trait Actor : Object {
 	type PublicKey : crate::PublicKey;
 	type Endpoints : Endpoints;
 
-	fn actor_type(&self) -> Option<ActorType> { None }
-	fn preferred_username(&self) -> Option<&str> { None }
+	fn actor_type(&self) -> Field<ActorType> { Err(FieldErr("type")) }
+	/// A short username which may be used to refer to the actor, with no uniqueness guarantees.
+	fn preferred_username(&self) -> Field<&str> { Err(FieldErr("preferredUsername")) }
+	/// A reference to an [ActivityStreams] OrderedCollection comprised of all the messages received by the actor; see 5.2 Inbox. 
 	fn inbox(&self) -> Node<Self::Collection>;
+	/// An [ActivityStreams] OrderedCollection comprised of all the messages produced by the actor; see 5.1 Outbox. 
 	fn outbox(&self) -> Node<Self::Collection>;
+	/// A link to an [ActivityStreams] collection of the actors that this actor is following; see 5.4 Following Collection
 	fn following(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A link to an [ActivityStreams] collection of the actors that follow this actor; see 5.3 Followers Collection. 
 	fn followers(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A link to an [ActivityStreams] collection of objects this actor has liked; see 5.5 Liked Collection.
 	fn liked(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A list of supplementary Collections which may be of interest.
 	fn streams(&self) -> Node<Self::Collection> { Node::Empty }
+	/// A json object which maps additional (typically server/domain-wide) endpoints which may be useful either for this actor or someone referencing this actor.
+	/// This mapping may be nested inside the actor document as the value or may be a link to a JSON-LD document with these properties. 
 	fn endpoints(&self) -> Node<Self::Endpoints> { Node::Empty }
-	fn public_key(&self) -> Node<Self::PublicKey> { Node::Empty }
+	fn public_key(&self) -> Node<Self::PublicKey> { Node::Empty } // TODO hmmm where is this from??
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn moved_to(&self) -> Node<Self::Actor> { Node::Empty }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	fn manually_approves_followers(&self) -> Option<bool> { None }
+	fn manually_approves_followers(&self) -> Field<bool> { Err(FieldErr("manuallyApprovesFollowers")) }
+
+	#[cfg(feature = "did-core")]
+	fn also_known_as(&self) -> Node<Self::Actor> { Node::Empty }
 
 	#[cfg(feature = "activitypub-fe")]
-	fn following_me(&self) -> Option<bool> { None }
+	fn following_me(&self) -> Field<bool> { Err(FieldErr("followingMe")) }
 	#[cfg(feature = "activitypub-fe")]
-	fn followed_by_me(&self) -> Option<bool> { None }
+	fn followed_by_me(&self) -> Field<bool> { Err(FieldErr("followedByMe")) }
+	#[cfg(feature = "activitypub-fe")]
+	fn notifications(&self) -> Node<Self::Collection> { Node::Empty }
 
 	#[cfg(feature = "activitypub-counters")]
-	fn followers_count(&self) -> Option<u64> { None }
+	fn followers_count(&self) -> Field<u64> { Err(FieldErr("followersCount")) }
 	#[cfg(feature = "activitypub-counters")]
-	fn following_count(&self) -> Option<u64> { None }
+	fn following_count(&self) -> Field<u64> { Err(FieldErr("followingCount")) }
 	#[cfg(feature = "activitypub-counters")]
-	fn statuses_count(&self) -> Option<u64> { None }
+	fn statuses_count(&self) -> Field<u64> { Err(FieldErr("statusesCount")) }
 
 	#[cfg(feature = "toot")]
-	fn discoverable(&self) -> Option<bool> { None }
+	fn discoverable(&self) -> Field<bool> { Err(FieldErr("discoverable")) }
+	#[cfg(feature = "toot")]
+	fn featured(&self) -> Node<Self::Collection> { Node::Empty }
 }
 
 pub trait Endpoints : Object {
 	/// Endpoint URI so this actor's clients may access remote ActivityStreams objects which require authentication to access. To use this endpoint, the client posts an x-www-form-urlencoded id parameter with the value being the id of the requested ActivityStreams object. 
-	fn proxy_url(&self) -> Option<&str> { None }
+	fn proxy_url(&self) -> Field<&str> { Err(FieldErr("proxyUrl")) }
 	/// If OAuth 2.0 bearer tokens [RFC6749] [RFC6750] are being used for authenticating client to server interactions, this endpoint specifies a URI at which a browser-authenticated user may obtain a new authorization grant. 
-	fn oauth_authorization_endpoint(&self) -> Option<&str> { None }
+	fn oauth_authorization_endpoint(&self) -> Field<&str> { Err(FieldErr("oauthAuthorizationEndpoint")) }
 	/// If OAuth 2.0 bearer tokens [RFC6749] [RFC6750] are being used for authenticating client to server interactions, this endpoint specifies a URI at which a client may acquire an access token. 
-	fn oauth_token_endpoint(&self) -> Option<&str> { None }
+	fn oauth_token_endpoint(&self) -> Field<&str> { Err(FieldErr("oauthTokenEndpoint")) }
 	/// If Linked Data Signatures and HTTP Signatures are being used for authentication and authorization, this endpoint specifies a URI at which browser-authenticated users may authorize a client's public key for client to server interactions. 
-	fn provide_client_key(&self) -> Option<&str> { None }
+	fn provide_client_key(&self) -> Field<&str> { Err(FieldErr("provideClientKey")) }
 	/// If Linked Data Signatures and HTTP Signatures are being used for authentication and authorization, this endpoint specifies a URI at which a client key may be signed by the actor's key for a time window to act on behalf of the actor in interacting with foreign servers. 
-	fn sign_client_key(&self) -> Option<&str> { None }
+	fn sign_client_key(&self) -> Field<&str> { Err(FieldErr("signClientKey")) }
 	/// An optional endpoint used for wide delivery of publicly addressed activities and activities sent to followers. sharedInbox endpoints SHOULD also be publicly readable OrderedCollection objects containing objects addressed to the Public special collection. Reading from the sharedInbox endpoint MUST NOT present objects which are not addressed to the Public endpoint.
-	fn shared_inbox(&self) -> Option<&str> { None }
+	fn shared_inbox(&self) -> Field<&str> { Err(FieldErr("sharedInbox")) }
 }
 
 pub trait ActorMut : ObjectMut {
@@ -81,10 +97,15 @@ pub trait ActorMut : ObjectMut {
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn set_manually_approves_followers(self, val: Option<bool>) -> Self;
 
+	#[cfg(feature = "did-core")]
+	fn set_also_known_as(self, val: Node<Self::Actor>) -> Self;
+
 	#[cfg(feature = "activitypub-fe")]
 	fn set_following_me(self, val: Option<bool>) -> Self;
 	#[cfg(feature = "activitypub-fe")]
 	fn set_followed_by_me(self, val: Option<bool>) -> Self;
+	#[cfg(feature = "activitypub-fe")]
+	fn set_notifications(self, val: Node<Self::Collection>) -> Self;
 
 	#[cfg(feature = "activitypub-counters")]
 	fn set_followers_count(self, val: Option<u64>) -> Self;
@@ -95,6 +116,8 @@ pub trait ActorMut : ObjectMut {
 
 	#[cfg(feature = "toot")]
 	fn set_discoverable(self, val: Option<bool>) -> Self;
+	#[cfg(feature = "toot")]
+	fn set_featured(self, val: Node<Self::Collection>) -> Self;
 }
 
 pub trait EndpointsMut : ObjectMut {
@@ -117,46 +140,53 @@ impl Actor for serde_json::Value {
 	type PublicKey = serde_json::Value;
 	type Endpoints = serde_json::Value;
 
-	crate::getter! { actor_type -> type ActorType }
-	crate::getter! { preferred_username::preferredUsername -> &str }
+	crate::getter! { actorType -> type ActorType }
+	crate::getter! { preferredUsername -> &str }
 	crate::getter! { inbox -> node Self::Collection }
 	crate::getter! { outbox -> node Self::Collection }
 	crate::getter! { following -> node Self::Collection }
 	crate::getter! { followers -> node Self::Collection }
 	crate::getter! { liked -> node Self::Collection }
 	crate::getter! { streams -> node Self::Collection }
-	crate::getter! { public_key::publicKey -> node Self::PublicKey }
+	crate::getter! { publicKey -> node Self::PublicKey }
 	crate::getter! { endpoints -> node Self::Endpoints }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { moved_to::movedTo -> node Self::Actor }
+	crate::getter! { movedTo -> node Self::Actor }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { manually_approves_followers::manuallyApprovedFollowers -> bool }
+	crate::getter! { manuallyApprovesFollowers -> bool }
+
+	#[cfg(feature = "did-core")]
+	crate::getter! { alsoKnownAs -> node Self::Actor }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { following_me::followingMe -> bool }
+	crate::getter! { followingMe -> bool }
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { followed_by_me::followedByMe -> bool }
+	crate::getter! { followedByMe -> bool }
+	#[cfg(feature = "activitypub-fe")]
+	crate::getter! { notifications -> node Self::Collection }
 
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { following_count::followingCount -> u64 }
+	crate::getter! { followingCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { followers_count::followersCount -> u64 }
+	crate::getter! { followersCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::getter! { statuses_count::statusesCount -> u64 }
+	crate::getter! { statusesCount -> u64 }
 
 	#[cfg(feature = "toot")]
 	crate::getter! { discoverable -> bool }
+	#[cfg(feature = "toot")]
+	crate::getter! { featured -> node Self::Collection }
 }
 
 #[cfg(feature = "unstructured")]
 impl Endpoints for serde_json::Value {
-	crate::getter! { proxy_url::proxyUrl -> &str }
-	crate::getter! { oauth_authorization_endpoint::oauthAuthorizationEndpoint -> &str }
-	crate::getter! { oauth_token_endpoint::oauthTokenEndpoint -> &str }
-	crate::getter! { provide_client_key::provideClientKey -> &str }
-	crate::getter! { sign_client_key::signClientKey -> &str }
-	crate::getter! { shared_inbox::sharedInbox -> &str }
+	crate::getter! { proxyUrl -> &str }
+	crate::getter! { oauthAuthorizationEndpoint -> &str }
+	crate::getter! { oauthTokenEndpoint -> &str }
+	crate::getter! { provideClientKey -> &str }
+	crate::getter! { signClientKey -> &str }
+	crate::getter! { sharedInbox -> &str }
 }
 
 #[cfg(feature = "unstructured")]
@@ -165,43 +195,51 @@ impl ActorMut for serde_json::Value {
 	type Endpoints = serde_json::Value;
 
 	crate::setter! { actor_type -> type ActorType }
-	crate::setter! { preferred_username::preferredUsername -> &str }
+	crate::setter! { preferredUsername -> &str }
 	crate::setter! { inbox -> node Self::Collection }
 	crate::setter! { outbox -> node Self::Collection }
 	crate::setter! { following -> node Self::Collection }
 	crate::setter! { followers -> node Self::Collection }
 	crate::setter! { liked -> node Self::Collection }
 	crate::setter! { streams -> node Self::Collection }
-	crate::setter! { public_key::publicKey -> node Self::PublicKey }
+	crate::setter! { publicKey -> node Self::PublicKey }
 	crate::setter! { endpoints -> node Self::Endpoints }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { moved_to::movedTo -> node Self::Actor }
+	crate::setter! { movedTo -> node Self::Actor }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { manually_approves_followers::manuallyApprovedFollowers -> bool }
+	crate::setter! { manuallyApprovesFollowers -> bool }
+
+
+	#[cfg(feature = "did-core")]
+	crate::setter! { alsoKnownAs -> node Self::Actor }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { following_me::followingMe -> bool }
+	crate::setter! { followingMe -> bool }
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { followed_by_me::followedByMe -> bool }
+	crate::setter! { followedByMe -> bool }
+	#[cfg(feature = "activitypub-fe")]
+	crate::setter! { notifications -> node Self::Collection }
 
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { following_count::followingCount -> u64 }
+	crate::setter! { followingCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { followers_count::followersCount -> u64 }
+	crate::setter! { followersCount -> u64 }
 	#[cfg(feature = "activitypub-counters")]
-	crate::setter! { statuses_count::statusesCount -> u64 }
+	crate::setter! { statusesCount -> u64 }
 
 	#[cfg(feature = "toot")]
 	crate::setter! { discoverable -> bool }
+	#[cfg(feature = "toot")]
+	crate::setter! { featured -> node Self::Collection }
 }
 
 #[cfg(feature = "unstructured")]
 impl EndpointsMut for serde_json::Value {
-	crate::setter! { proxy_url::proxyUrl -> &str }
-	crate::setter! { oauth_authorization_endpoint::oauthAuthorizationEndpoint -> &str }
-	crate::setter! { oauth_token_endpoint::oauthTokenEndpoint -> &str }
-	crate::setter! { provide_client_key::provideClientKey -> &str }
-	crate::setter! { sign_client_key::signClientKey -> &str }
-	crate::setter! { shared_inbox::sharedInbox -> &str }
+	crate::setter! { proxyUrl -> &str }
+	crate::setter! { oauthAuthorizationEndpoint -> &str }
+	crate::setter! { oauthTokenEndpoint -> &str }
+	crate::setter! { provideClientKey -> &str }
+	crate::setter! { signClientKey -> &str }
+	crate::setter! { sharedInbox -> &str }
 }

apb/src/types/object/collection/mod.rs

@@ -1,7 +1,7 @@
 pub mod page;
 pub use page::CollectionPage;
 
-use crate::{Node, Object, ObjectMut};
+use crate::{Field, FieldErr, Node, Object, ObjectMut};
 
 crate::strenum! {
 	pub enum CollectionType {
@@ -15,13 +15,20 @@ crate::strenum! {
 pub trait Collection : Object {
 	type CollectionPage : CollectionPage;
 
-	fn collection_type(&self) -> Option<CollectionType> { None }
+	fn collection_type(&self) -> Field<CollectionType> { Err(FieldErr("type")) }
 
-	fn total_items(&self) -> Option<u64> { None }
+	/// A non-negative integer specifying the total number of objects contained by the logical view of the collection.
+	/// This number might not reflect the actual number of items serialized within the Collection object instance. 
+	fn total_items(&self) -> Field<u64> { Err(FieldErr("totalItems")) }
+	/// In a paged Collection, indicates the page that contains the most recently updated member items. 
 	fn current(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// In a paged Collection, indicates the furthest preceeding page of items in the collection. 
 	fn first(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// In a paged Collection, indicates the furthest proceeding page of the collection.
 	fn last(&self) -> Node<Self::CollectionPage> { Node::Empty }
+	/// Identifies the items contained in a collection. The items might be ordered or unordered.
 	fn items(&self) -> Node<Self::Object> { Node::Empty }
+	/// ??????????????? same as items but ordered?? spec just uses it without saying
 	fn ordered_items(&self) -> Node<Self::Object> { Node::Empty }
 }
 
@@ -42,12 +49,12 @@ impl Collection for serde_json::Value {
 	type CollectionPage = serde_json::Value;
 
 	crate::getter! { collection_type -> type CollectionType }
-	crate::getter! { total_items::totalItems -> u64 }
+	crate::getter! { totalItems -> u64 }
 	crate::getter! { current -> node Self::CollectionPage }
 	crate::getter! { first -> node Self::CollectionPage }
 	crate::getter! { last -> node Self::CollectionPage }
 	crate::getter! { items -> node <Self as Object>::Object }
-	crate::getter! { ordered_items::orderedItems -> node <Self as Object>::Object }
+	crate::getter! { orderedItems -> node <Self as Object>::Object }
 }
 
 #[cfg(feature = "unstructured")]
@@ -55,10 +62,10 @@ impl CollectionMut for serde_json::Value {
 	type CollectionPage = serde_json::Value;
 
 	crate::setter! { collection_type -> type CollectionType }
-	crate::setter! { total_items::totalItems -> u64 }
+	crate::setter! { totalItems -> u64 }
 	crate::setter! { current -> node Self::CollectionPage }
 	crate::setter! { first -> node Self::CollectionPage }
 	crate::setter! { last -> node Self::CollectionPage }
 	crate::setter! { items -> node <Self as Object>::Object }
-	crate::setter! { ordered_items::orderedItems -> node <Self as Object>::Object }
+	crate::setter! { orderedItems -> node <Self as Object>::Object }
 }

apb/src/types/object/collection/page.rs

@@ -14,14 +14,14 @@ pub trait CollectionPageMut : super::CollectionMut {
 
 #[cfg(feature = "unstructured")]
 impl CollectionPage for serde_json::Value {
-	crate::getter! { part_of::partOf -> node Self::Collection }
+	crate::getter! { partOf -> node Self::Collection }
 	crate::getter! { next -> node Self::CollectionPage }
 	crate::getter! { prev -> node Self::CollectionPage }
 }
 
 #[cfg(feature = "unstructured")]
 impl CollectionPageMut for serde_json::Value {
-	crate::setter! { part_of::partOf -> node Self::Collection }
+	crate::setter! { partOf -> node Self::Collection }
 	crate::setter! { next -> node Self::CollectionPage }
 	crate::setter! { prev -> node Self::CollectionPage }
 }

apb/src/types/object/document.rs

@@ -9,7 +9,7 @@ crate::strenum! {
 }
 
 pub trait Document : super::Object {
-	fn document_type(&self) -> Option<DocumentType> { None }
+	fn document_type(&self) -> crate::Field<DocumentType> { Err(crate::FieldErr("type")) }
 }
 
 pub trait DocumentMut : super::ObjectMut {
@@ -19,10 +19,10 @@ pub trait DocumentMut : super::ObjectMut {
 
 #[cfg(feature = "unstructured")]
 impl Document for serde_json::Value {
-	crate::getter! { document_type -> type DocumentType }
+	crate::getter! { documentType -> type DocumentType }
 }
 
 #[cfg(feature = "unstructured")]
 impl DocumentMut for serde_json::Value {
-	crate::setter! { document_type -> type DocumentType }
+	crate::setter! { documentType -> type DocumentType }
 }

apb/src/types/object/mod.rs

@@ -7,7 +7,7 @@ pub mod place;
 pub mod profile;
 pub mod relationship;
 
-use crate::{Base, BaseMut, Node};
+use crate::{Base, BaseMut, Field, FieldErr, Node};
 
 use actor::ActorType;
 use document::DocumentType;
@@ -40,51 +40,96 @@ pub trait Object : Base {
 	type Document : crate::Document;
 	type Activity : crate::Activity;
 
-	fn object_type(&self) -> Option<ObjectType> { None }
+	fn object_type(&self) -> Field<ObjectType> { Err(FieldErr("type")) }
+	/// Identifies a resource attached or related to an object that potentially requires special handling
+	/// The intent is to provide a model that is at least semantically similar to attachments in email. 
 	fn attachment(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies one or more entities to which this object is attributed.
+	/// The attributed entities might not be Actors. For instance, an object might be attributed to the completion of another activity. 
 	fn attributed_to(&self) -> Node<Self::Actor> { Node::Empty }
-	fn audience(&self) -> Node<Self::Actor> { Node::Empty }
-	fn content(&self) -> Option<&str> { None } // TODO handle language maps
+	/// Identifies one or more entities that represent the total population of entities for which the object can considered to be relevant
+	fn audience(&self) -> Node<Self::Object> { Node::Empty }
+	/// The content or textual representation of the Object encoded as a JSON string. By default, the value of content is HTML
+	/// The mediaType property can be used in the object to indicate a different content type
+	/// The content MAY be expressed using multiple language-tagged values
+	fn content(&self) -> Field<&str> { Err(FieldErr("content")) } // TODO handle language maps
+	/// Identifies the context within which the object exists or an activity was performed
+	/// The notion of "context" used is intentionally vague
+	/// The intended function is to serve as a means of grouping objects and activities that share a common originating context or purpose
+	/// An example could be all activities relating to a common project or event
 	fn context(&self) -> Node<Self::Object> { Node::Empty } 
-	fn name(&self) -> Option<&str> { None }       // also in link // TODO handle language maps
-	fn end_time(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	/// A simple, human-readable, plain-text name for the object. HTML markup MUST NOT be included. The name MAY be expressed using multiple language-tagged values
+	fn name(&self) -> Field<&str> { Err(FieldErr("name")) }       // also in link // TODO handle language maps
+	/// The date and time describing the actual or expected ending time of the object
+	/// When used with an Activity object, for instance, the endTime property specifies the moment the activity concluded or is expected to conclude. 
+	fn end_time(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("endTime")) }
+	/// Identifies the entity (e.g. an application) that generated the object
 	fn generator(&self) -> Node<Self::Actor> { Node::Empty }
+	/// Indicates an entity that describes an icon for this object
+	/// The image should have an aspect ratio of one (horizontal) to one (vertical) and should be suitable for presentation at a small size
 	fn icon(&self) -> Node<Self::Document> { Node::Empty }
+	/// Indicates an entity that describes an image for this object
+	/// Unlike the icon property, there are no aspect ratio or display size limitations assumed
 	fn image(&self) -> Node<Self::Document> { Node::Empty }
+	/// Indicates one or more entities for which this object is considered a response
 	fn in_reply_to(&self) -> Node<Self::Object> { Node::Empty }
+	/// Indicates one or more physical or logical locations associated with the object
 	fn location(&self) -> Node<Self::Object> { Node::Empty }
+	/// Identifies an entity that provides a preview of this object
 	fn preview(&self) -> Node<Self::Object> { Node::Empty }    // also in link
-	fn published(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
-	fn updated(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	/// The date and time at which the object was published
+	fn published(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("published")) }
+	/// The date and time at which the object was updated
+	fn updated(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("updated")) }
+	/// Identifies a Collection containing objects considered to be responses to this object
 	fn replies(&self) -> Node<Self::Collection> { Node::Empty }
 	fn likes(&self) -> Node<Self::Collection> { Node::Empty }
 	fn shares(&self) -> Node<Self::Collection> { Node::Empty }
-	fn start_time(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
-	fn summary(&self) -> Option<&str> { None }
-	fn tag(&self) -> Node<Self::Object> { Node::Empty }
+	/// The date and time describing the actual or expected starting time of the object.
+	/// When used with an Activity object, for instance, the startTime property specifies the moment the activity began or is scheduled to begin. 
+	fn start_time(&self) -> Field<chrono::DateTime<chrono::Utc>> { Err(FieldErr("startTime")) }
+	/// A natural language summarization of the object encoded as HTML. Multiple language tagged summaries MAY be provided
+	fn summary(&self) -> Field<&str> { Err(FieldErr("summary")) }
+	/// One or more "tags" that have been associated with an objects. A tag can be any kind of Object
+	/// The key difference between attachment and tag is that the former implies association by inclusion, while the latter implies associated by reference
+	// TODO technically this is an object? but spec says that it works my reference, idk
+	fn tag(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies one or more links to representations of the object
 	fn url(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an entity considered to be part of the public primary audience of an Object
 	fn to(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an Object that is part of the private primary audience of this Object
 	fn bto(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies an Object that is part of the public secondary audience of this Object
 	fn cc(&self) -> Node<Self::Link> { Node::Empty }
+	/// Identifies one or more Objects that are part of the private secondary audience of this Object
 	fn bcc(&self) -> Node<Self::Link> { Node::Empty }
-	fn media_type(&self) -> Option<&str> { None } // also in link
-	fn duration(&self) -> Option<&str> { None } // TODO how to parse xsd:duration ?
+	/// When used on a Link, identifies the MIME media type of the referenced resource.
+	/// When used on an Object, identifies the MIME media type of the value of the content property.
+	/// If not specified, the content property is assumed to contain text/html content. 
+	fn media_type(&self) -> Field<&str> { Err(FieldErr("mediaType")) } // also in link
+	/// When the object describes a time-bound resource, such as an audio or video, a meeting, etc, the duration property indicates the object's approximate duration.
+	/// The value MUST be expressed as an xsd:duration as defined by [ xmlschema11-2], section 3.3.6 (e.g. a period of 5 seconds is represented as "PT5S"). 
+	fn duration(&self) -> Field<&str> { Err(FieldErr("duration")) } // TODO how to parse xsd:duration ?
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	fn sensitive(&self) -> Option<bool> { None }
+	fn sensitive(&self) -> Field<bool> { Err(FieldErr("sensitive")) }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	fn quote_url(&self) -> Node<Self::Object> { Node::Empty }
 
 	#[cfg(feature = "activitypub-fe")]
-	fn liked_by_me(&self) -> Option<bool> { None }
+	fn liked_by_me(&self) -> Field<bool> { Err(FieldErr("likedByMe")) }
 
 	#[cfg(feature = "ostatus")]
 	fn conversation(&self) -> Node<Self::Object> { Node::Empty }
 
-	fn as_activity(&self) -> Option<&Self::Activity> { None }
-	fn as_actor(&self) -> Option<&Self::Actor> { None }
-	fn as_collection(&self) -> Option<&Self::Collection> { None }
-	fn as_document(&self) -> Option<&Self::Document> { None }
+	fn as_activity(&self) -> Result<&Self::Activity, FieldErr> { Err(FieldErr("type")) }
+	fn as_actor(&self) -> Result<&Self::Actor, FieldErr> { Err(FieldErr("type")) }
+	fn as_collection(&self) -> Result<&Self::Collection, FieldErr> { Err(FieldErr("type")) }
+	fn as_document(&self) -> Result<&Self::Document, FieldErr> { Err(FieldErr("type")) }
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	fn value(&self) -> Field<&str> { Err(FieldErr("value")) }
 }
 
 pub trait ObjectMut : BaseMut {
@@ -134,6 +179,9 @@ pub trait ObjectMut : BaseMut {
 
 	#[cfg(feature = "ostatus")]
 	fn set_conversation(self, val: Node<Self::Object>) -> Self;
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	fn set_value(self, val: Option<&str>) -> Self;
 }
 
 #[cfg(feature = "unstructured")]
@@ -145,18 +193,18 @@ impl Object for serde_json::Value {
 	type Collection = serde_json::Value;
 	type Activity = serde_json::Value;
 
-	crate::getter! { object_type -> type ObjectType }
+	crate::getter! { objectType -> type ObjectType }
 	crate::getter! { attachment -> node <Self as Object>::Object }
-	crate::getter! { attributed_to::attributedTo -> node Self::Actor }
+	crate::getter! { attributedTo -> node Self::Actor }
 	crate::getter! { audience -> node Self::Actor }
 	crate::getter! { content -> &str }
 	crate::getter! { context -> node <Self as Object>::Object }
 	crate::getter! { name -> &str }
-	crate::getter! { end_time::endTime -> chrono::DateTime<chrono::Utc> }
+	crate::getter! { endTime -> chrono::DateTime<chrono::Utc> }
 	crate::getter! { generator -> node Self::Actor }
 	crate::getter! { icon -> node Self::Document }
 	crate::getter! { image -> node Self::Document }
-	crate::getter! { in_reply_to::inReplyTo -> node <Self as Object>::Object }
+	crate::getter! { inReplyTo -> node <Self as Object>::Object }
 	crate::getter! { location -> node <Self as Object>::Object }
 	crate::getter! { preview -> node <Self as Object>::Object }
 	crate::getter! { published -> chrono::DateTime<chrono::Utc> }
@@ -164,53 +212,56 @@ impl Object for serde_json::Value {
 	crate::getter! { replies -> node Self::Collection }
 	crate::getter! { likes -> node Self::Collection }
 	crate::getter! { shares -> node Self::Collection }
-	crate::getter! { start_time::startTime -> chrono::DateTime<chrono::Utc> }
+	crate::getter! { startTime -> chrono::DateTime<chrono::Utc> }
 	crate::getter! { summary -> &str }
 	crate::getter! { tag -> node <Self as Object>::Object }
 	crate::getter! { to -> node Self::Link }
 	crate::getter! { bto -> node Self::Link }
 	crate::getter! { cc -> node Self::Link }
 	crate::getter! { bcc -> node Self::Link }
-	crate::getter! { media_type::mediaType -> &str }
+	crate::getter! { mediaType -> &str }
 	crate::getter! { duration -> &str }
 	crate::getter! { url -> node Self::Link }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	crate::getter! { sensitive -> bool }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::getter! { quote_url::quoteUrl -> node <Self as Object>::Object }
+	crate::getter! { quoteUrl -> node <Self as Object>::Object }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::getter! { liked_by_me::likedByMe -> bool }
+	crate::getter! { likedByMe -> bool }
 
 	#[cfg(feature = "ostatus")]
 	crate::getter! { conversation -> node <Self as Object>::Object }
 
-	fn as_activity(&self) -> Option<&Self::Activity> {
-		match self.object_type() {
-			Some(ObjectType::Activity(_)) => Some(self),
-			_ => None,
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	crate::getter! { value -> &str }
+
+	fn as_activity(&self) -> Result<&Self::Activity, FieldErr> {
+		match self.object_type()? {
+			ObjectType::Activity(_) => Ok(self),
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_actor(&self) -> Option<&Self::Actor> {
-		match self.object_type() {
-			Some(ObjectType::Actor(_)) => Some(self),
-			_ => None,
+	fn as_actor(&self) -> Result<&Self::Actor, FieldErr> {
+		match self.object_type()? {
+			ObjectType::Actor(_) => Ok(self),
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_collection(&self) -> Option<&Self::Collection> {
-		match self.object_type() {
-			Some(ObjectType::Collection(_)) => Some(self),
-			_ => None,
+	fn as_collection(&self) -> Result<&Self::Collection, FieldErr> {
+		match self.object_type()? {
+			ObjectType::Collection(_) => Ok(self),
+			_ => Err(FieldErr("type")),
 		}
 	}
 
-	fn as_document(&self) -> Option<&Self::Document> {
-		match self.object_type() {
-			Some(ObjectType::Document(_)) => Some(self),
-			_ => None,
+	fn as_document(&self) -> Result<&Self::Document, FieldErr> {
+		match self.object_type()? {
+			ObjectType::Document(_) => Ok(self),
+			_ => Err(FieldErr("type")),
 		}
 	}
 }
@@ -225,16 +276,16 @@ impl ObjectMut for serde_json::Value {
 
 	crate::setter! { object_type -> type ObjectType }
 	crate::setter! { attachment -> node <Self as Object>::Object }
-	crate::setter! { attributed_to::attributedTo -> node Self::Actor }
+	crate::setter! { attributedTo -> node Self::Actor }
 	crate::setter! { audience -> node Self::Actor }
 	crate::setter! { content -> &str }
 	crate::setter! { context -> node <Self as Object>::Object }
 	crate::setter! { name -> &str }
-	crate::setter! { end_time::endTime -> chrono::DateTime<chrono::Utc> }
+	crate::setter! { endTime -> chrono::DateTime<chrono::Utc> }
 	crate::setter! { generator -> node Self::Actor }
 	crate::setter! { icon -> node Self::Document }
 	crate::setter! { image -> node Self::Document }
-	crate::setter! { in_reply_to::inReplyTo -> node <Self as Object>::Object }
+	crate::setter! { inReplyTo -> node <Self as Object>::Object }
 	crate::setter! { location -> node <Self as Object>::Object }
 	crate::setter! { preview -> node <Self as Object>::Object }
 	crate::setter! { published -> chrono::DateTime<chrono::Utc> }
@@ -242,25 +293,28 @@ impl ObjectMut for serde_json::Value {
 	crate::setter! { replies -> node Self::Collection }
 	crate::setter! { likes -> node Self::Collection }
 	crate::setter! { shares -> node Self::Collection }
-	crate::setter! { start_time::startTime -> chrono::DateTime<chrono::Utc> }
+	crate::setter! { startTime -> chrono::DateTime<chrono::Utc> }
 	crate::setter! { summary -> &str }
 	crate::setter! { tag -> node <Self as Object>::Object }
 	crate::setter! { to -> node Self::Link }
 	crate::setter! { bto -> node Self::Link}
 	crate::setter! { cc -> node Self::Link }
 	crate::setter! { bcc -> node Self::Link }
-	crate::setter! { media_type::mediaType -> &str }
+	crate::setter! { mediaType -> &str }
 	crate::setter! { duration -> &str }
 	crate::setter! { url -> node Self::Link }
 
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
 	crate::setter! { sensitive -> bool }
 	#[cfg(feature = "activitypub-miscellaneous-terms")]
-	crate::setter! { quote_url::quoteUrl -> node <Self as Object>::Object }
+	crate::setter! { quoteUrl -> node <Self as Object>::Object }
 
 	#[cfg(feature = "activitypub-fe")]
-	crate::setter! { liked_by_me::likedByMe -> bool }
+	crate::setter! { likedByMe -> bool }
 
 	#[cfg(feature = "ostatus")]
 	crate::setter! { conversation -> node <Self as Object>::Object }
+
+	#[cfg(feature = "did-core")] // TODO this isn't from did-core actually!?!?!?!?!
+	crate::setter! { value -> &str }
 }

apb/src/types/object/place.rs

@@ -1,10 +1,12 @@
+use crate::{Field, FieldErr};
+
 pub trait Place : super::Object {
-	fn accuracy(&self) -> Option<f64> { None }
-	fn altitude(&self) -> Option<f64> { None }
-	fn latitude(&self) -> Option<f64> { None }
-	fn longitude(&self) -> Option<f64> { None }
-	fn radius(&self) -> Option<f64> { None }
-	fn units(&self) -> Option<&str> { None }
+	fn accuracy(&self) -> Field<f64> { Err(FieldErr("accuracy")) }
+	fn altitude(&self) -> Field<f64> { Err(FieldErr("altitude")) }
+	fn latitude(&self) -> Field<f64> { Err(FieldErr("latitude")) }
+	fn longitude(&self) -> Field<f64> { Err(FieldErr("longitude")) }
+	fn radius(&self) -> Field<f64> { Err(FieldErr("radius")) }
+	fn units(&self) -> Field<&str> { Err(FieldErr("units")) }
 }
 
 pub trait PlaceMut : super::ObjectMut {

apb/src/types/object/tombstone.rs

@@ -1,6 +1,6 @@
 pub trait Tombstone : super::Object {
-	fn former_type(&self) -> Option<crate::BaseType> { None }
-	fn deleted(&self) -> Option<chrono::DateTime<chrono::Utc>> { None }
+	fn former_type(&self) -> crate::Field<crate::BaseType> { Err(crate::FieldErr("formerType")) }
+	fn deleted(&self) -> crate::Field<chrono::DateTime<chrono::Utc>> { Err(crate::FieldErr("deleted")) }
 }
 
 pub trait TombstoneMut : super::ObjectMut {

main.rs

@@ -0,0 +1,284 @@
+use std::path::PathBuf;
+use clap::{Parser, Subcommand};
+use sea_orm::{ConnectOptions, Database};
+use signal_hook::consts::signal::*;
+use signal_hook_tokio::Signals;
+use futures::stream::StreamExt;
+use upub::{context, ext::LoggableError};
+
+#[cfg(feature = "cli")]
+use upub_cli as cli;
+
+#[cfg(feature = "migrate")]
+use upub_migrations as migrations;
+
+#[cfg(feature = "serve")]
+use upub_routes as routes;
+
+#[cfg(feature = "worker")]
+use upub_worker as worker;
+
+
+#[derive(Parser)]
+/// all names were taken
+struct Args {
+	#[clap(subcommand)]
+	/// command to run
+	command: Mode,
+
+	/// path to config file, leave empty to not use any
+	#[arg(short, long)]
+	config: Option<PathBuf>,
+
+	#[arg(long = "db")]
+	/// database connection uri, overrides config value
+	database: Option<String>,
+
+	#[arg(long)]
+	/// instance base domain, for AP ids, overrides config value
+	domain: Option<String>,
+
+	#[arg(long, default_value_t=false)]
+	/// run with debug level tracing
+	debug: bool,
+
+	#[arg(long)]
+	/// force set number of worker threads for async runtime, defaults to number of cores
+	threads: Option<usize>,
+}
+
+#[derive(Clone, Subcommand)]
+enum Mode {
+	/// print current or default configuration
+	Config,
+
+	#[cfg(feature = "migrate")]
+	/// apply database migrations
+	Migrate,
+
+	#[cfg(feature = "cli")]
+	/// run maintenance CLI tasks
+	Cli {
+		#[clap(subcommand)]
+		/// task to run
+		command: cli::CliCommand,
+	},
+
+	#[cfg(all(feature = "serve", feature = "worker"))]
+	/// start both api routes and background workers
+	Monolith {
+		#[arg(short, long, default_value="127.0.0.1:3000")]
+		/// addr to bind and serve onto
+		bind: String,
+
+		#[arg(short, long, default_value_t = 4)]
+		/// how many concurrent jobs to process with this worker
+		tasks: usize,
+
+		#[arg(short, long, default_value_t = 20)]
+		/// interval for polling new tasks
+		poll: u64,
+	},
+
+	#[cfg(feature = "serve")]
+	/// start api routes server
+	Serve {
+		#[arg(short, long, default_value="127.0.0.1:3000")]
+		/// addr to bind and serve onto
+		bind: String,
+	},
+
+	#[cfg(feature = "worker")]
+	/// start background job worker
+	Work {
+		/// only run tasks of this type, run all if not given
+		filter: Filter,
+
+		/// how many concurrent jobs to process with this worker
+		#[arg(short, long, default_value_t = 4)]
+		tasks: usize,
+
+		#[arg(short, long, default_value_t = 20)]
+		/// interval for polling new tasks
+		poll: u64,
+	},
+}
+
+fn main() {
+	let args = Args::parse();
+
+	tracing_subscriber::fmt()
+		.compact()
+		.with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO })
+		.init();
+
+	let config = upub::Config::load(args.config.as_ref());
+
+	if matches!(args.command, Mode::Config) {
+		println!("{}", toml::to_string_pretty(&config).expect("failed serializing config"));
+		return;
+	}
+
+	let mut runtime = tokio::runtime::Builder::new_multi_thread();
+
+	if let Some(threads) = args.threads {
+		runtime.worker_threads(threads);
+	}
+
+	runtime
+		.enable_io()
+		.enable_time()
+		.thread_name("upub-worker")
+		.build()
+		.expect("failed creating tokio async runtime")
+		.block_on(async { init(args, config).await })
+}
+
+async fn init(args: Args, config: upub::Config) {
+	let database = args.database.unwrap_or(config.datasource.connection_string.clone());
+	let domain = args.domain.unwrap_or(config.instance.domain.clone());
+
+	// TODO can i do connectoptions.into() or .connect() and skip these ugly bindings?
+	let mut opts = ConnectOptions::new(&database);
+
+	opts
+		.sqlx_logging(true)
+		.sqlx_logging_level(tracing::log::LevelFilter::Debug)
+		.max_connections(config.datasource.max_connections)
+		.min_connections(config.datasource.min_connections)
+		.acquire_timeout(std::time::Duration::from_secs(config.datasource.acquire_timeout_seconds))
+		.connect_timeout(std::time::Duration::from_secs(config.datasource.connect_timeout_seconds))
+		.sqlx_slow_statements_logging_settings(
+			if config.datasource.slow_query_warn_enable { tracing::log::LevelFilter::Warn } else { tracing::log::LevelFilter::Debug },
+			std::time::Duration::from_secs(config.datasource.slow_query_warn_seconds)
+		);
+
+	let db = Database::connect(opts)
+		.await.expect("error connecting to db");
+
+	#[cfg(feature = "migrate")]
+	if matches!(args.command, Mode::Migrate) {
+		use migrations::MigratorTrait;
+
+		migrations::Migrator::up(&db, None)
+			.await
+			.expect("error applying migrations");
+
+		return;
+	}
+
+	let (tx_wake, rx_wake) = tokio::sync::mpsc::unbounded_channel();
+	let wake = WakeToken(rx_wake);
+
+	let ctx = upub::Context::new(db, domain, config.clone(), Some(Box::new(WakerToken(tx_wake))))
+		.await.expect("failed creating server context");
+
+	#[cfg(feature = "cli")]
+	if let Mode::Cli { command } = args.command {
+		cli::run(ctx, command)
+			.await.expect("failed running cli task");
+
+		return;
+	}
+
+
+	// register signal handler only for long-lasting modes, such as server or worker
+	let (tx, rx) = tokio::sync::watch::channel(false);
+	let signals = Signals::new([SIGTERM, SIGINT]).expect("failed registering signal handler");
+	let handle = signals.handle();
+	let signals_task = tokio::spawn(handle_signals(signals, tx));
+	let stop = CancellationToken(rx);
+
+	match args.command {
+		#[cfg(feature = "serve")]
+		Mode::Serve { bind } =>
+			routes::serve(ctx, bind, stop)
+				.await.expect("failed serving api routes"),
+
+		#[cfg(feature = "worker")]
+		Mode::Work { filter, tasks, poll } =>
+			worker::spawn(ctx, tasks, poll, filter.into(), stop, wake)
+				.await.expect("failed running worker"),
+
+		#[cfg(all(feature = "serve", feature = "worker"))]
+		Mode::Monolith { bind, tasks, poll } => {
+			worker::spawn(ctx.clone(), tasks, poll, None, stop.clone(), wake);
+
+			routes::serve(ctx, bind, stop)
+				.await.expect("failed serving api routes");
+		},
+
+		Mode::Config => unreachable!(),
+		#[cfg(feature = "migrate")]
+		Mode::Migrate => unreachable!(),
+		#[cfg(feature = "cli")]
+		Mode::Cli { .. } => unreachable!(),
+	}
+
+	handle.close();
+	signals_task.await.expect("failed joining signal handler task");
+}
+
+struct WakerToken(tokio::sync::mpsc::UnboundedSender<()>);
+impl context::WakerToken for WakerToken {
+	fn wake(&self) {
+		self.0.send(()).warn_failed("failed waking up workers");
+	}
+}
+
+struct WakeToken(tokio::sync::mpsc::UnboundedReceiver<()>);
+
+impl worker::WakeToken for WakeToken {
+	async fn wait(&mut self) {
+		let _ = self.0.recv().await;
+	}
+}
+
+#[derive(Clone)]
+struct CancellationToken(tokio::sync::watch::Receiver<bool>);
+
+impl worker::StopToken for CancellationToken {
+	fn stop(&self) -> bool {
+		*self.0.borrow()
+	}
+}
+
+impl routes::ShutdownToken for CancellationToken {
+	async fn event(mut self) {
+		self.0.changed().await.warn_failed("cancellation token channel closed, stopping...");
+	}
+}
+
+async fn handle_signals(
+	mut signals: signal_hook_tokio::Signals,
+	tx: tokio::sync::watch::Sender<bool>,
+) {
+	while let Some(signal) = signals.next().await {
+		match signal {
+			SIGTERM | SIGINT => {
+				tracing::info!("received stop signal, closing tasks");
+				tx.send(true).info_failed("error sending stop signal to tasks")
+			},
+			_ => unreachable!(),
+		}
+	}
+}
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+enum Filter {
+	All,
+	Delivery,
+	Inbound,
+	Outbound,
+}
+
+impl From<Filter> for Option<upub::model::job::JobType> {
+	fn from(value: Filter) -> Self {
+		match value {
+			Filter::All => None,
+			Filter::Delivery => Some(upub::model::job::JobType::Delivery),
+			Filter::Inbound => Some(upub::model::job::JobType::Inbound),
+			Filter::Outbound => Some(upub::model::job::JobType::Outbound),
+		}
+	}
+}

mdhtml/src/lib.rs

@@ -1,89 +0,0 @@
-use html5ever::tendril::*;
-use html5ever::tokenizer::{BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer};
-use comrak::{markdown_to_html, Options};
-
-/// In our case, our sink only contains a tokens vector
-#[derive(Debug, Clone, Default)]
-struct Sink(String);
-
-impl TokenSink for Sink {
-	type Handle = ();
-
-	/// Each processed token will be handled by this method
-	fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
-		match token {
-			Token::TagToken(tag) => {
-				if !matches!(
-					tag.name.as_ref(),
-					"h1" | "h2" | "h3" 
-					| "hr" | "br" | "p" | "b" | "i"
-					| "blockquote" | "pre" | "code"
-					| "ul" | "ol" | "li"
-					| "img" | "a"
-				) { return TokenSinkResult::Continue } // skip this tag
-
-				self.0.push('<');
-				if !tag.self_closing && matches!(tag.kind, TagKind::EndTag) {
-					self.0.push('/');
-				}
-
-				self.0.push_str(tag.name.as_ref());
-
-				match tag.name.as_ref() {
-					"img" => for attr in tag.attrs {
-						match attr.name.local.as_ref() {
-							"src" => self.0.push_str(&format!(" src=\"{}\"", attr.value.as_ref())),
-							"title" => self.0.push_str(&format!(" title=\"{}\"", attr.value.as_ref())),
-							"alt" => self.0.push_str(&format!(" alt=\"{}\"", attr.value.as_ref())),
-							_ => {},
-						}
-					},
-					"a" => {
-						for attr in tag.attrs {
-							match attr.name.local.as_ref() {
-								"href" => self.0.push_str(&format!(" href=\"{}\"", attr.value.as_ref())),
-								"title" => self.0.push_str(&format!(" title=\"{}\"", attr.value.as_ref())),
-								_ => {},
-							}
-						}
-						self.0.push_str(" rel=\"nofollow noreferrer\" target=\"_blank\"");
-					},
-					_ => {},
-				}
-
-				if tag.self_closing {
-					self.0.push('/');
-				}
-				self.0.push('>');
-			},
-			Token::CharacterTokens(txt) => self.0.push_str(txt.as_ref()),
-			Token::CommentToken(_) => {},
-			Token::DoctypeToken(_) => {},
-			Token::NullCharacterToken => {},
-			Token::EOFToken => {},
-			Token::ParseError(e) => tracing::error!("error parsing html: {e}"),
-		}
-		TokenSinkResult::Continue
-	}
-}
-
-pub fn safe_markdown(text: &str) -> String {
-	safe_html(&markdown_to_html(text, &Options::default()))
-}
-
-pub fn safe_html(text: &str) -> String {
-	let mut input = BufferQueue::default();
-	input.push_back(text.to_tendril().try_reinterpret().unwrap());
-
-	let sink = Sink::default();
-
-	let mut tok = Tokenizer::new(sink, Default::default());
-	let _ = tok.feed(&mut input);
-
-	if !input.is_empty() {
-		tracing::warn!("buffer input not empty after processing html");
-	}
-	tok.end();
-
-	tok.sink.0
-}

src/cli/fetch.rs

@@ -1,39 +0,0 @@
-use sea_orm::{EntityTrait, IntoActiveModel};
-
-use crate::server::fetcher::Fetchable;
-
-pub async fn fetch(ctx: crate::server::Context, uri: String, save: bool) -> crate::Result<()> {
-	use apb::Base;
-
-	let mut node = apb::Node::link(uri.to_string());
-	node.fetch(&ctx).await?;
-
-	let obj = node.get().expect("node still empty after fetch?");
-
-	if save {
-		match obj.base_type() {
-			Some(apb::BaseType::Object(apb::ObjectType::Actor(_))) => {
-				crate::model::user::Entity::insert(
-					crate::model::user::Model::new(obj).unwrap().into_active_model()
-				).exec(ctx.db()).await.unwrap();
-			},
-			Some(apb::BaseType::Object(apb::ObjectType::Activity(_))) => {
-				crate::model::activity::Entity::insert(
-					crate::model::activity::Model::new(obj).unwrap().into_active_model()
-				).exec(ctx.db()).await.unwrap();
-			},
-			Some(apb::BaseType::Object(apb::ObjectType::Note)) => {
-				crate::model::object::Entity::insert(
-					crate::model::object::Model::new(obj).unwrap().into_active_model()
-				).exec(ctx.db()).await.unwrap();
-			},
-			Some(apb::BaseType::Object(t)) => tracing::warn!("not implemented: {:?}", t),
-			Some(apb::BaseType::Link(_)) => tracing::error!("fetched another link?"),
-			None => tracing::error!("no type on object"),
-		}
-	}
-
-	println!("{}", serde_json::to_string_pretty(&obj).unwrap());
-
-	Ok(())
-}

src/cli/mod.rs

@@ -1,119 +0,0 @@
-mod fix;
-pub use fix::*;
-
-mod fetch;
-pub use fetch::*;
-
-mod faker;
-pub use faker::*;
-
-mod relay;
-pub use relay::*;
-
-mod register;
-pub use register::*;
-
-mod update;
-pub use update::*;
-
-#[derive(Debug, Clone, clap::Subcommand)]
-pub enum CliCommand {
-	/// generate fake user, note and activity
-	Faker{
-		/// how many fake statuses to insert for root user
-		count: u64,
-	},
-
-	/// fetch a single AP object
-	Fetch {
-		/// object id, or uri, to fetch
-		uri: String,
-
-		#[arg(long, default_value_t = false)]
-		/// store fetched object in local db
-		save: bool,
-	},
-
-	/// follow a remote relay
-	Relay {
-		/// actor url, same as with pleroma
-		actor: String,
-
-		#[arg(long, default_value_t = false)]
-		/// instead of sending a follow request, send an accept
-		accept: bool
-	},
-
-	/// run db maintenance tasks
-	Fix {
-		#[arg(long, default_value_t = false)]
-		/// fix likes counts for posts
-		likes: bool,
-
-		#[arg(long, default_value_t = false)]
-		/// fix shares counts for posts
-		shares: bool,
-
-		#[arg(long, default_value_t = false)]
-		/// fix replies counts for posts
-		replies: bool,
-	},
-
-	/// update remote users
-	Update {
-		#[arg(long, short, default_value_t = 7)]
-		/// number of days after which users should get updated
-		days: i64,
-	},
-
-	/// register a new local user
-	Register {
-		/// username for new user, must be unique locally and cannot be changed
-		username: String,
-
-		/// password for new user
-		// TODO get this with getpass rather than argv!!!!
-		password: String,
-
-		/// display name for new user
-		#[arg(long = "name")]
-		display_name: Option<String>,
-
-		/// summary text for new user
-		#[arg(long = "summary")]
-		summary: Option<String>,
-
-		/// url for avatar image of new user
-		#[arg(long = "avatar")]
-		avatar_url: Option<String>,
-
-		/// url for banner image of new user
-		#[arg(long = "banner")]
-		banner_url: Option<String>,
-	}
-}
-
-pub async fn run(
-	command: CliCommand,
-	db: sea_orm::DatabaseConnection,
-	domain: String,
-	config: crate::config::Config,
-) -> crate::Result<()> {
-	let ctx = crate::server::Context::new(
-		db, domain, config,
-	).await?;
-	match command {
-		CliCommand::Faker { count } =>
-			Ok(faker(ctx, count).await?),
-		CliCommand::Fetch { uri, save } =>
-			Ok(fetch(ctx, uri, save).await?),
-		CliCommand::Relay { actor, accept } =>
-			Ok(relay(ctx, actor, accept).await?),
-		CliCommand::Fix { likes, shares, replies } =>
-			Ok(fix(ctx, likes, shares, replies).await?),
-		CliCommand::Update { days } =>
-			Ok(update_users(ctx, days).await?),
-		CliCommand::Register { username, password, display_name, summary, avatar_url, banner_url } =>
-			Ok(register(ctx, username, password, display_name, summary, avatar_url, banner_url).await?),
-	}
-}

src/cli/relay.rs

@@ -1,38 +0,0 @@
-use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, QueryOrder};
-
-pub async fn relay(ctx: crate::server::Context, actor: String, accept: bool) -> crate::Result<()> {
-	let aid = ctx.aid(&uuid::Uuid::new_v4().to_string());
-
-	let mut activity_model = crate::model::activity::Model {
-		id: aid.clone(),
-		activity_type: apb::ActivityType::Follow,
-		actor: ctx.base().to_string(),
-		object: Some(actor.clone()),
-		target: None,
-		published: chrono::Utc::now(),
-		to: crate::model::Audience(vec![actor.clone()]),
-		bto: crate::model::Audience::default(),
-		cc: crate::model::Audience(vec![apb::target::PUBLIC.to_string()]),
-		bcc: crate::model::Audience::default(),
-	};
-
-	if accept {
-		let follow_req = crate::model::activity::Entity::find()
-			.filter(crate::model::activity::Column::ActivityType.eq("Follow"))
-			.filter(crate::model::activity::Column::Actor.eq(&actor))
-			.filter(crate::model::activity::Column::Object.eq(ctx.base()))
-			.order_by_desc(crate::model::activity::Column::Published)
-			.one(ctx.db())
-			.await?
-			.expect("no follow request to accept");
-		activity_model.activity_type = apb::ActivityType::Accept(apb::AcceptType::Accept);
-		activity_model.object = Some(follow_req.id);
-	};
-
-	crate::model::activity::Entity::insert(activity_model.into_active_model())
-		.exec(ctx.db()).await?;
-
-	ctx.dispatch(ctx.base(), vec![actor, apb::target::PUBLIC.to_string()], &aid, None).await?;
-
-	Ok(())
-}

src/cli/update.rs

@@ -1,38 +0,0 @@
-use futures::TryStreamExt;
-use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter};
-
-use crate::server::fetcher::Fetcher;
-
-pub async fn update_users(ctx: crate::server::Context, days: i64) -> crate::Result<()> {
-	let mut count = 0;
-	let mut insertions = Vec::new();
-
-	{
-		let mut stream = crate::model::user::Entity::find()
-			.filter(crate::model::user::Column::Updated.lt(chrono::Utc::now() - chrono::Duration::days(days)))
-			.stream(ctx.db())
-			.await?;
-
-
-		while let Some(user) = stream.try_next().await? {
-			if ctx.is_local(&user.id) { continue }
-			match ctx.pull_user(&user.id).await {
-				Err(e) => tracing::warn!("could not update user {}: {e}", user.id),
-				Ok(u) => {
-					insertions.push(u);
-					count += 1;
-				},
-			}
-		}
-	}
-
-	for u in insertions {
-		tracing::info!("updating user {}", u.id);
-		crate::model::user::Entity::delete_by_id(&u.id).exec(ctx.db()).await?;
-		crate::model::user::Entity::insert(u.into_active_model()).exec(ctx.db()).await?;
-	}
-
-	tracing::info!("updated {count} users");
-
-	Ok(())
-}

src/main.rs

@@ -1,140 +0,0 @@
-pub mod server; // TODO there are some methods that i dont use yet, make it public so that ra shuts up
-mod model;
-mod routes;
-
-mod errors;
-mod config;
-
-#[cfg(feature = "cli")]
-mod cli;
-
-#[cfg(feature = "migrations")]
-mod migrations;
-
-#[cfg(feature = "migrations")]
-use sea_orm_migration::MigratorTrait;
-use std::path::PathBuf;
-use config::Config;
-
-use clap::{Parser, Subcommand};
-use sea_orm::{ConnectOptions, Database};
-
-pub use errors::UpubResult as Result;
-use tower_http::{cors::CorsLayer, trace::TraceLayer};
-
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-
-#[derive(Parser)]
-/// all names were taken
-struct Args {
-	#[clap(subcommand)]
-	/// command to run
-	command: Mode,
-
-	/// path to config file, leave empty to not use any
-	#[arg(short, long)]
-	config: Option<PathBuf>,
-
-	#[arg(long = "db")]
-	/// database connection uri, overrides config value
-	database: Option<String>,
-
-	#[arg(long)]
-	/// instance base domain, for AP ids, overrides config value
-	domain: Option<String>,
-
-	#[arg(long, default_value_t=false)]
-	/// run with debug level tracing
-	debug: bool,
-}
-
-#[derive(Clone, Subcommand)]
-enum Mode {
-	/// run fediverse server
-	Serve,
-
-	/// print current or default configuration
-	Config,
-
-	#[cfg(feature = "migrations")]
-	/// apply database migrations
-	Migrate,
-
-	#[cfg(feature = "cli")]
-	/// run maintenance CLI tasks
-	Cli {
-		#[clap(subcommand)]
-		/// task to run
-		command: cli::CliCommand,
-	},
-}
-
-#[tokio::main]
-async fn main() {
-
-	let args = Args::parse();
-
-	tracing_subscriber::fmt()
-		.compact()
-		.with_max_level(if args.debug { tracing::Level::DEBUG } else { tracing::Level::INFO })
-		.init();
-
-	let config = Config::load(args.config);
-
-	let database = args.database.unwrap_or(config.datasource.connection_string.clone());
-	let domain = args.domain.unwrap_or(config.instance.domain.clone());
-
-	// TODO can i do connectoptions.into() or .connect() and skip these ugly bindings?
-	let mut opts = ConnectOptions::new(&database);
-
-	opts
-		.sqlx_logging_level(tracing::log::LevelFilter::Debug)
-		.max_connections(config.datasource.max_connections)
-		.min_connections(config.datasource.min_connections)
-		.acquire_timeout(std::time::Duration::from_secs(config.datasource.acquire_timeout_seconds))
-		.connect_timeout(std::time::Duration::from_secs(config.datasource.connect_timeout_seconds))
-		.sqlx_slow_statements_logging_settings(
-			if config.datasource.slow_query_warn_enable { tracing::log::LevelFilter::Warn } else { tracing::log::LevelFilter::Off },
-			std::time::Duration::from_secs(config.datasource.slow_query_warn_seconds)
-		);
-
-	let db = Database::connect(opts)
-		.await.expect("error connecting to db");
-
-	match args.command {
-		#[cfg(feature = "migrations")]
-		Mode::Migrate =>
-			migrations::Migrator::up(&db, None)
-				.await.expect("error applying migrations"),
-
-		#[cfg(feature = "cli")]
-		Mode::Cli { command } =>
-			cli::run(command, db, domain, config)
-				.await.expect("failed running cli task"),
-
-		Mode::Config => println!("{}", toml::to_string_pretty(&config).expect("failed serializing config")),
-
-		Mode::Serve => {
-			let ctx = server::Context::new(db, domain, config)
-				.await.expect("failed creating server context");
-
-			use routes::activitypub::ActivityPubRouter;
-			use routes::mastodon::MastodonRouter;
-
-			let router = axum::Router::new()
-				.ap_routes()
-				.mastodon_routes() // no-op if mastodon feature is disabled
-				.layer(CorsLayer::permissive())
-				.layer(TraceLayer::new_for_http())
-				.with_state(ctx);
-
-			// run our app with hyper, listening locally on port 3000
-			let listener = tokio::net::TcpListener::bind("127.0.0.1:3000")
-				.await.expect("could not bind tcp socket");
-
-			axum::serve(listener, router)
-				.await
-				.expect("failed serving application")
-		},
-	}
-}

src/migrations/README.md

@@ -1,3 +0,0 @@
-# migrations
-
-there are sea_orm migrations to apply to your database

src/migrations/mod.rs

@@ -1,22 +0,0 @@
-use sea_orm_migration::prelude::*;
-
-mod m20240524_000001_create_actor_activity_object_tables;
-mod m20240524_000002_create_relations_likes_shares;
-mod m20240524_000003_create_users_auth_and_config;
-mod m20240524_000004_create_addressing_deliveries;
-mod m20240524_000005_create_attachments_tags_mentions;
-
-pub struct Migrator;
-
-#[async_trait::async_trait]
-impl MigratorTrait for Migrator {
-	fn migrations() -> Vec<Box<dyn MigrationTrait>> {
-		vec![
-			Box::new(m20240524_000001_create_actor_activity_object_tables::Migration),
-			Box::new(m20240524_000002_create_relations_likes_shares::Migration),
-			Box::new(m20240524_000003_create_users_auth_and_config::Migration),
-			Box::new(m20240524_000004_create_addressing_deliveries::Migration),
-			Box::new(m20240524_000005_create_attachments_tags_mentions::Migration),
-		]
-	}
-}

src/model/addressing.rs

@@ -1,192 +0,0 @@
-use apb::{ActivityMut, ObjectMut};
-use sea_orm::{entity::prelude::*, sea_query::IntoCondition, Condition, FromQueryResult, Iterable, Order, QueryOrder, QuerySelect, SelectColumns};
-
-use crate::routes::activitypub::jsonld::LD;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "addressing")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
-	pub instance: i32,
-	pub activity: Option<i32>,
-	pub object: Option<i32>,
-	pub published: ChronoDateTimeUtc,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Activities,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Actor",
-		to = "super::actor::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Actors,
-	#[sea_orm(
-		belongs_to = "super::instance::Entity",
-		from = "Column::Instance",
-		to = "super::instance::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Instances,
-	#[sea_orm(
-		belongs_to = "super::object::Entity",
-		from = "Column::Object",
-		to = "super::object::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	Objects,
-}
-
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Activities.def()
-	}
-}
-
-impl Related<super::actor::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Actors.def()
-	}
-}
-
-impl Related<super::instance::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Instances.def()
-	}
-}
-
-impl Related<super::object::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Objects.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-
-
-#[allow(clippy::large_enum_variant)] // tombstone is an outlier, not the norm! this is a beefy enum
-#[derive(Debug, Clone)]
-pub enum Event {
-	Tombstone,
-	Activity(crate::model::activity::Model),
-	StrayObject {
-		object: crate::model::object::Model,
-		liked: Option<String>,
-	},
-	DeepActivity {
-		activity: crate::model::activity::Model,
-		object: crate::model::object::Model,
-		liked: Option<String>,
-	}
-}
-
-
-impl Event {
-	pub fn id(&self) -> &str {
-		match self {
-			Event::Tombstone => "",
-			Event::Activity(x) => x.id.as_str(),
-			Event::StrayObject { object, liked: _ } => object.id.as_str(),
-			Event::DeepActivity { activity: _, liked: _, object } => object.id.as_str(),
-		}
-	}
-
-	pub fn ap(self, attachment: Option<Vec<crate::model::attachment::Model>>) -> serde_json::Value {
-		let attachment = match attachment {
-			None => apb::Node::Empty,
-			Some(vec) => apb::Node::array(
-				vec.into_iter().map(|x| x.ap()).collect()
-			),
-		};
-		match self {
-			Event::Activity(x) => x.ap(),
-			Event::DeepActivity { activity, object, liked } =>
-				activity.ap().set_object(apb::Node::object(
-					object.ap()
-						.set_attachment(attachment)
-						.set_liked_by_me(if liked.is_some() { Some(true) } else { None })
-				)),
-			Event::StrayObject { object, liked } => serde_json::Value::new_object()
-				.set_activity_type(Some(apb::ActivityType::Activity))
-				.set_object(apb::Node::object(
-					object.ap()
-						.set_attachment(attachment)
-						.set_liked_by_me(if liked.is_some() { Some(true) } else { None })
-				)),
-			Event::Tombstone => serde_json::Value::new_object()
-				.set_activity_type(Some(apb::ActivityType::Activity))
-				.set_object(apb::Node::object(
-					serde_json::Value::new_object()
-						.set_object_type(Some(apb::ObjectType::Tombstone))
-				)),
-		}
-	}
-}
-
-impl FromQueryResult for Event {
-	fn from_query_result(res: &sea_orm::QueryResult, _pre: &str) -> Result<Self, sea_orm::DbErr> {
-		let activity = crate::model::activity::Model::from_query_result(res, crate::model::activity::Entity.table_name()).ok();
-		let object = crate::model::object::Model::from_query_result(res, crate::model::object::Entity.table_name()).ok();
-		let liked = res.try_get(crate::model::like::Entity.table_name(), &crate::model::like::Column::Actor.to_string()).ok();
-		match (activity, object) {
-			(Some(activity), Some(object)) => Ok(Self::DeepActivity { activity, object, liked }),
-			(Some(activity), None) => Ok(Self::Activity(activity)),
-			(None, Some(object)) => Ok(Self::StrayObject { object, liked }),
-			(None, None) => Ok(Self::Tombstone),
-		}
-	}
-}
-
-
-impl Entity {
-	pub fn find_addressed(uid: Option<&str>) -> Select<Entity> {
-		let mut select = Entity::find()
-			.distinct()
-			.select_only()
-			.join(sea_orm::JoinType::LeftJoin, Relation::Object.def())
-			.join(sea_orm::JoinType::LeftJoin, Relation::Activity.def())
-			.filter(
-				// TODO ghetto double inner join because i want to filter out tombstones
-				Condition::any()
-					.add(crate::model::activity::Column::Id.is_not_null())
-					.add(crate::model::object::Column::Id.is_not_null())
-			)
-			.order_by(Column::Published, Order::Desc);
-
-		if let Some(uid) = uid {
-			let uid = uid.to_string();
-			select = select
-				.join(
-					sea_orm::JoinType::LeftJoin,
-					crate::model::object::Relation::Like.def()
-						.on_condition(move |_l, _r| crate::model::like::Column::Actor.eq(uid.clone()).into_condition()),
-				)
-				.select_column_as(crate::model::like::Column::Actor, format!("{}{}", crate::model::like::Entity.table_name(), crate::model::like::Column::Actor.to_string()));
-		}
-
-		for col in crate::model::object::Column::iter() {
-			select = select.select_column_as(col, format!("{}{}", crate::model::object::Entity.table_name(), col.to_string()));
-		}
-
-		for col in crate::model::activity::Column::iter() {
-			select = select.select_column_as(col, format!("{}{}", crate::model::activity::Entity.table_name(), col.to_string()));
-		}
-
-		select
-	}
-}

src/model/attachment.rs

@@ -1,99 +0,0 @@
-use apb::{DocumentMut, ObjectMut};
-use sea_orm::entity::prelude::*;
-
-use crate::routes::activitypub::jsonld::LD;
-
-use super::addressing::Event;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "attachments")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub id: i32,
-	#[sea_orm(unique)]
-	pub url: String,
-	pub object: i32,
-	pub document_type: String,
-	pub name: Option<String>,
-	pub media_type: String,
-	pub created: ChronoDateTimeUtc,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::object::Entity",
-		from = "Column::Object",
-		to = "super::object::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Objects,
-}
-
-impl Related<super::object::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Objects.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Model {
-	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
-			.set_url(apb::Node::link(self.url))
-			.set_document_type(Some(self.document_type))
-			.set_media_type(Some(&self.media_type))
-			.set_name(self.name.as_deref())
-			.set_published(Some(self.created))
-	}
-}
-
-#[axum::async_trait]
-pub trait BatchFillable {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<String, Vec<Model>>, DbErr>;
-}
-
-#[axum::async_trait]
-impl BatchFillable for &[Event] {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<String, Vec<Model>>, DbErr> {
-		let objects : Vec<crate::model::object::Model> = self
-			.iter()
-			.filter_map(|x| match x {
-				Event::Tombstone => None,
-				Event::Activity(_) => None,
-				Event::StrayObject { object, liked: _ } => Some(object.clone()),
-				Event::DeepActivity { activity: _, liked: _, object } => Some(object.clone()),
-			})
-			.collect();
-
-		let attachments = objects.load_many(Entity, db).await?;
-
-		let mut out : std::collections::BTreeMap<String, Vec<Model>> = std::collections::BTreeMap::new();
-		for attach in attachments.into_iter().flatten() {
-			if out.contains_key(&attach.object) {
-				out.get_mut(&attach.object).expect("contains but get failed?").push(attach);
-			} else {
-				out.insert(attach.object.clone(), vec![attach]);
-			}
-		}
-
-		Ok(out)
-	}
-}
-
-#[axum::async_trait]
-impl BatchFillable for Vec<Event> {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<String, Vec<Model>>, DbErr> {
-		self.as_slice().load_attachments_batch(db).await
-	}
-}
-
-#[axum::async_trait]
-impl BatchFillable for Event {
-	async fn load_attachments_batch(&self, db: &DatabaseConnection) -> Result<std::collections::BTreeMap<String, Vec<Model>>, DbErr> {
-		let x = vec![self.clone()]; // TODO wasteful clone and vec![] but ehhh convenient
-		x.load_attachments_batch(db).await
-	}
-}

src/model/delivery.rs

@@ -1,66 +0,0 @@
-use sea_orm::entity::prelude::*;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "deliveries")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
-	pub target: String,
-	pub activity: i32,
-	pub created: ChronoDateTimeUtc,
-	pub not_before: ChronoDateTimeUtc,
-	pub attempt: i32,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Activities,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Actor",
-		to = "super::actor::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	Actors,
-}
-
-impl Related<super::activity::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Activities.def()
-	}
-}
-
-impl Related<super::actor::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Actors.def()
-	}
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Model {
-	pub fn next_delivery(&self) -> ChronoDateTimeUtc {
-		match self.attempt {
-			0 => chrono::Utc::now() + std::time::Duration::from_secs(10),
-			1 => chrono::Utc::now() + std::time::Duration::from_secs(60),
-			2 => chrono::Utc::now() + std::time::Duration::from_secs(5 * 60),
-			3 => chrono::Utc::now() + std::time::Duration::from_secs(20 * 60),
-			4 => chrono::Utc::now() + std::time::Duration::from_secs(60 * 60),
-			5 => chrono::Utc::now() + std::time::Duration::from_secs(12 * 60 * 60),
-			_ => chrono::Utc::now() + std::time::Duration::from_secs(24 * 60 * 60),
-		}
-	}
-
-	pub fn expired(&self) -> bool {
-		chrono::Utc::now() - self.created > chrono::Duration::days(7)
-	}
-}

src/model/mod.rs

@@ -1,48 +0,0 @@
-pub mod actor;
-pub mod object;
-pub mod activity;
-
-pub mod config;
-pub mod credential;
-pub mod session;
-
-pub mod instance;
-pub mod delivery;
-
-pub mod relation;
-pub mod announce;
-pub mod like;
-
-pub mod hashtag;
-pub mod mention;
-pub mod attachment;
-
-pub mod addressing;
-
-#[derive(Debug, Clone, thiserror::Error)]
-#[error("missing required field: '{0}'")]
-pub struct FieldError(pub &'static str);
-
-impl From<FieldError> for axum::http::StatusCode {
-	fn from(value: FieldError) -> Self {
-		tracing::error!("bad request: {value}");
-		axum::http::StatusCode::BAD_REQUEST
-	}
-}
-
-#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize, sea_orm::FromJsonQueryResult)]
-pub struct Audience(pub Vec<String>);
-
-impl<T: apb::Base> From<apb::Node<T>> for Audience {
-	fn from(value: apb::Node<T>) -> Self {
-		Audience(
-			match value {
-				apb::Node::Empty => vec![],
-				apb::Node::Link(l) => vec![l.href().to_string()],
-				apb::Node::Object(o) => if let Some(id) = o.id() { vec![id.to_string()] } else { vec![] },
-				apb::Node::Array(arr) => arr.into_iter().filter_map(|l| Some(l.id()?.to_string())).collect(),
-			}
-		)
-	}
-}
-

src/model/relation.rs

@@ -1,62 +0,0 @@
-use sea_orm::{entity::prelude::*, sea_query::Alias, QuerySelect, SelectColumns};
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
-#[sea_orm(table_name = "relations")]
-pub struct Model {
-	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub follower: i32,
-	pub following: i32,
-	pub accept: Option<i32>,
-	pub activity: i32,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Accept",
-		to = "super::activity::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	ActivitiesAccept,
-	#[sea_orm(
-		belongs_to = "super::activity::Entity",
-		from = "Column::Activity",
-		to = "super::activity::Column::Id",
-		on_update = "Cascade",
-		on_delete = "NoAction"
-	)]
-	ActivitiesFollow,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Follower",
-		to = "super::actor::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	ActorsFollower,
-	#[sea_orm(
-		belongs_to = "super::actor::Entity",
-		from = "Column::Following",
-		to = "super::actor::Column::Id",
-		on_update = "Cascade",
-		on_delete = "Cascade"
-	)]
-	ActorsFollowing,
-}
-
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Entity {
-	pub fn find_followers(id: &str) -> Select<Entity> {
-		Entity::find()
-			.inner_join(Relation::ActorsFollowing.def())
-			.filter(super::actor::Column::ApId.eq(id))
-			.left_join(Relation::ActorsFollower.def())
-			.select_only()
-			.select_column(super::actor::Column::ApId)
-			.into_tuple::<String>()
-	}
-}

src/routes/activitypub/activity.rs

@@ -1,34 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, QueryFilter};
-use crate::{errors::UpubError, model::{self, addressing::Event, attachment::BatchFillable}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}};
-
-use super::{jsonld::LD, JsonLD, TryFetch};
-
-pub async fn view(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(query): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let aid = ctx.aid(&id);
-	if auth.is_local() && query.fetch && !ctx.is_local(&aid) {
-		let obj = ctx.fetch_activity(&aid).await?;
-		if obj.ap_id != aid {
-			return Err(UpubError::Redirect(obj.ap_id));
-		}
-	}
-
-	let row = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(model::activity::Column::Id.eq(&aid))
-		.filter(auth.filter_condition())
-		.into_model::<Event>()
-		.one(ctx.db())
-		.await?
-		.ok_or_else(UpubError::not_found)?;
-
-	let mut attachments = row.load_attachments_batch(ctx.db()).await?;
-	let attach = attachments.remove(row.id());
-
-	Ok(JsonLD(row.ap(attach).ld_context()))
-}
-

src/routes/activitypub/application.rs

@@ -1,93 +0,0 @@
-use apb::{ActorMut, BaseMut, ObjectMut, PublicKeyMut};
-use axum::{extract::{Query, State}, http::HeaderMap, response::{IntoResponse, Redirect, Response}, Form, Json};
-use reqwest::Method;
-
-use crate::{errors::UpubError, server::{auth::{AuthIdentity, Identity}, fetcher::Fetcher, Context}, url};
-
-use super::{jsonld::LD, JsonLD};
-
-
-pub async fn view(
-	headers: HeaderMap,
-	State(ctx): State<Context>,
-) -> crate::Result<Response> {
-	if let Some(accept) = headers.get("Accept") {
-		if let Ok(accept) = accept.to_str() {
-			if accept.contains("text/html") && !accept.contains("application/ld+json") {
-				return Ok(Redirect::to("/web").into_response());
-			}
-		}
-	}
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(&url!(ctx, "")))
-			.set_actor_type(Some(apb::ActorType::Application))
-			.set_name(Some(&ctx.cfg().instance.name))
-			.set_summary(Some(&ctx.cfg().instance.description))
-			.set_inbox(apb::Node::link(url!(ctx, "/inbox")))
-			.set_outbox(apb::Node::link(url!(ctx, "/outbox")))
-			.set_published(Some(ctx.app().created))
-			.set_endpoints(apb::Node::Empty)
-			.set_preferred_username(Some(ctx.domain()))
-			.set_public_key(apb::Node::object(
-				serde_json::Value::new_object()
-					.set_id(Some(&url!(ctx, "#main-key")))
-					.set_owner(Some(&url!(ctx, "")))
-					.set_public_key_pem(&ctx.app().public_key)
-			))
-			.ld_context()
-	).into_response())
-}
-
-#[derive(Debug, serde::Deserialize)]
-pub struct FetchPath {
-	id: String,
-}
-
-pub async fn proxy_get(
-	State(ctx): State<Context>,
-	Query(query): Query<FetchPath>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<Json<serde_json::Value>> {
-	// only local users can request fetches
-	if !ctx.cfg().security.allow_public_debugger && !matches!(auth, Identity::Local(_)) {
-		return Err(UpubError::unauthorized());
-	}
-	Ok(Json(
-		Context::request(
-			Method::GET,
-			&query.id,
-			None,
-			ctx.base(),
-			&ctx.app().private_key,
-			&format!("{}+proxy", ctx.domain()),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?
-	))
-}
-
-pub async fn proxy_form(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Form(query): Form<FetchPath>,
-) -> crate::Result<Json<serde_json::Value>> {
-	// only local users can request fetches
-	if !ctx.cfg().security.allow_public_debugger && !matches!(auth, Identity::Local(_)) {
-		return Err(UpubError::unauthorized());
-	}
-	Ok(Json(
-		Context::request(
-			Method::GET,
-			&query.id,
-			None,
-			ctx.base(),
-			&ctx.app().private_key,
-			&format!("{}+proxy", ctx.domain()),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?
-	))
-}

src/routes/activitypub/auth.rs

@@ -1,93 +0,0 @@
-use axum::{http::StatusCode, extract::State, Json};
-use rand::Rng;
-use sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter};
-
-use crate::{errors::UpubError, model, server::{admin::Administrable, Context}};
-
-
-#[derive(Debug, Clone, serde::Deserialize)]
-pub struct LoginForm {
-	email: String,
-	password: String,
-}
-
-#[derive(Debug, Clone, serde::Serialize)]
-pub struct AuthSuccess {
-	token: String,
-	user: String,
-	expires: chrono::DateTime<chrono::Utc>,
-}
-
-pub async fn login(
-	State(ctx): State<Context>,
-	Json(login): Json<LoginForm>
-) -> crate::Result<Json<AuthSuccess>> {
-	// TODO salt the pwd
-	match model::credential::Entity::find()
-		.filter(Condition::all()
-			.add(model::credential::Column::Login.eq(login.email))
-			.add(model::credential::Column::Password.eq(sha256::digest(login.password)))
-		)
-		.one(ctx.db())
-		.await?
-	{
-		Some(x) => {
-			let user = model::actor::Entity::find_by_id(x.actor)
-				.one(ctx.db())
-				.await?
-				.ok_or_else(UpubError::not_found)?;
-			// TODO should probably use crypto-safe rng
-			let token : String = rand::thread_rng()
-				.sample_iter(&rand::distributions::Alphanumeric)
-				.take(128)
-				.map(char::from)
-				.collect();
-			let expires = chrono::Utc::now() + std::time::Duration::from_secs(3600 * 6);
-			model::session::Entity::insert(
-				model::session::ActiveModel {
-					id: sea_orm::ActiveValue::NotSet,
-					secret: sea_orm::ActiveValue::Set(token.clone()),
-					actor: sea_orm::ActiveValue::Set(x.id.clone()),
-					expires: sea_orm::ActiveValue::Set(expires),
-				}
-			)
-				.exec(ctx.db())
-				.await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
-			Ok(Json(AuthSuccess {
-				token, expires,
-				user: user.ap_id,
-			}))
-		},
-		None => Err(UpubError::unauthorized()),
-	}
-}
-
-#[derive(Debug, Clone, serde::Deserialize)]
-pub struct RegisterForm {
-	username: String,
-	password: String,
-	display_name: Option<String>,
-	summary: Option<String>,
-	avatar_url: Option<String>,
-	banner_url: Option<String>,
-}
-
-pub async fn register(
-	State(ctx): State<Context>,
-	Json(registration): Json<RegisterForm>
-) -> crate::Result<Json<String>> {
-	if !ctx.cfg().security.allow_registration {
-		return Err(UpubError::forbidden());
-	}
-
-	ctx.register_user(
-		registration.username.clone(),
-		registration.password,
-		registration.display_name,
-		registration.summary,
-		registration.avatar_url,
-		registration.banner_url
-	).await?;
-
-	Ok(Json(ctx.uid(&registration.username)))
-}

src/routes/activitypub/context.rs

@@ -1,41 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, Condition, PaginatorTrait, QueryFilter};
-
-use crate::{model, routes::activitypub::{JsonLD, Pagination}, server::{auth::AuthIdentity, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let local_context_id = url!(ctx, "/context/{id}");
-	let context = ctx.context_id(&id);
-
-	let count = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(auth.filter_condition())
-		.filter(model::object::Column::Context.eq(context))
-		.count(ctx.db())
-		.await?;
-
-	crate::server::builders::collection(&local_context_id, Some(count))
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let context = ctx.context_id(&id);
-
-	crate::server::builders::paginate(
-		url!(ctx, "/context/{id}/page"),
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(model::object::Column::Context.eq(context)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}

src/routes/activitypub/inbox.rs

@@ -1,96 +0,0 @@
-use apb::{server::Inbox, Activity, ActivityType};
-use axum::{extract::{Query, State}, http::StatusCode, Json};
-use sea_orm::{sea_query::IntoCondition, ColumnTrait};
-
-use crate::{errors::UpubError, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-use super::{JsonLD, Pagination};
-
-
-pub async fn get(
-	State(ctx): State<Context>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/inbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::paginate(
-		url!(ctx, "/inbox/page"),
-		crate::model::addressing::Column::Actor.eq(apb::target::PUBLIC)
-			.into_condition(),
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}
-
-macro_rules! pretty_json {
-	($json:ident) => {
-		serde_json::to_string_pretty(&$json).expect("failed serializing to string serde_json::Value")
-	}
-}
-
-
-pub async fn post(
-	State(ctx): State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>
-) -> crate::Result<()> {
-	let Identity::Remote(server) = auth else {
-		if activity.activity_type() == Some(ActivityType::Delete) {
-			// this is spammy af, ignore them!
-			// we basically received a delete for a user we can't fetch and verify, meaning remote
-			// deleted someone we never saw. technically we deleted nothing so we should return error,
-			// but mastodon keeps hammering us trying to delete this user, so just make mastodon happy
-			// and return 200 without even bothering checking this stuff
-			// would be cool if mastodon played nicer with the network...
-			return Ok(());
-		}
-		tracing::warn!("refusing unauthorized activity: {}", pretty_json!(activity));
-		if matches!(auth, Identity::Anonymous) {
-			return Err(UpubError::unauthorized());
-		} else {
-			return Err(UpubError::forbidden());
-		}
-	};
-
-	let Some(actor) = activity.actor().id() else {
-		return Err(UpubError::bad_request());
-	};
-
-	// TODO add whitelist of relays
-	if !server.ends_with(&Context::server(&actor)) {
-		return Err(UpubError::unauthorized());
-	}
-
-	tracing::debug!("processing federated activity: '{}'", serde_json::to_string(&activity).unwrap_or_default());
-
-	// TODO we could process Links and bare Objects maybe, but probably out of AP spec?
-	match activity.activity_type().ok_or_else(UpubError::bad_request)? {
-		ActivityType::Activity => {
-			tracing::warn!("skipping unprocessable base activity: {}", pretty_json!(activity));
-			Err(StatusCode::UNPROCESSABLE_ENTITY.into()) // won't ingest useless stuff
-		},
-
-		// TODO emojireacts are NOT likes, but let's process them like ones for now maybe?
-		ActivityType::Like | ActivityType::EmojiReact => Ok(ctx.like(server, activity).await?),
-		ActivityType::Create => Ok(ctx.create(server, activity).await?),
-		ActivityType::Follow => Ok(ctx.follow(server, activity).await?),
-		ActivityType::Announce => Ok(ctx.announce(server, activity).await?),
-		ActivityType::Accept(_) => Ok(ctx.accept(server, activity).await?),
-		ActivityType::Reject(_) => Ok(ctx.reject(server, activity).await?),
-		ActivityType::Undo => Ok(ctx.undo(server, activity).await?),
-		ActivityType::Delete => Ok(ctx.delete(server, activity).await?),
-		ActivityType::Update => Ok(ctx.update(server, activity).await?),
-
-		_x => {
-			tracing::info!("received unimplemented activity on inbox: {}", pretty_json!(activity));
-			Err(StatusCode::NOT_IMPLEMENTED.into())
-		},
-	}
-}

src/routes/activitypub/object/mod.rs

@@ -1,76 +0,0 @@
-pub mod replies;
-
-use apb::{CollectionMut, ObjectMut};
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, EntityTrait, ModelTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{errors::UpubError, model::{self, addressing::Event}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}};
-
-use super::{jsonld::LD, JsonLD, TryFetch};
-
-pub async fn view(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(query): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let oid = ctx.oid(&id);
-	if auth.is_local() && query.fetch && !ctx.is_local(&oid) {
-		let obj = ctx.fetch_object(&oid).await?;
-		// some implementations serve statuses on different urls than their AP id
-		if obj.ap_id != oid {
-			return Err(UpubError::Redirect(crate::url!(ctx, "/objects/{}", ctx.id(&obj.ap_id))));
-		}
-	}
-
-	let item = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(model::object::Column::Id.eq(&oid))
-		.filter(auth.filter_condition())
-		.into_model::<Event>()
-		.one(ctx.db())
-		.await?
-		.ok_or_else(UpubError::not_found)?;
-
-	let object = match item {
-		Event::Tombstone => return Err(UpubError::not_found()),
-		Event::Activity(_) => return Err(UpubError::not_found()),
-		Event::StrayObject { liked: _, object } => object,
-		Event::DeepActivity { activity: _, liked: _, object } => object,
-	};
-
-	let attachments = object.find_related(model::attachment::Entity)
-		.all(ctx.db())
-		.await?
-		.into_iter()
-		.map(|x| x.ap())
-		.collect::<Vec<serde_json::Value>>();
-
-	let mut replies = apb::Node::Empty;
-	
-	if ctx.cfg().security.show_reply_ids {
-		let replies_ids = model::addressing::Entity::find_addressed(None)
-			.filter(model::object::Column::InReplyTo.eq(oid))
-			.filter(auth.filter_condition())
-			.select_only()
-			.select_column(model::object::Column::Id)
-			.into_tuple::<String>()
-			.all(ctx.db())
-			.await?;
-
-		replies = apb::Node::object(
-			serde_json::Value::new_object()
-				// .set_id(Some(&crate::url!(ctx, "/objects/{id}/replies")))
-				// .set_first(apb::Node::link(crate::url!(ctx, "/objects/{id}/replies/page")))
-				.set_collection_type(Some(apb::CollectionType::Collection))
-				.set_total_items(Some(object.replies as u64))
-				.set_items(apb::Node::links(replies_ids))
-		);
-	}
-	
-	Ok(JsonLD(
-		object.ap()
-			.set_attachment(apb::Node::array(attachments))
-			.set_replies(replies)
-			.ld_context()
-	))
-}

src/routes/activitypub/object/replies.rs

@@ -1,47 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, Condition, PaginatorTrait, QueryFilter};
-
-use crate::{model, routes::activitypub::{JsonLD, Pagination, TryFetch}, server::{auth::AuthIdentity, fetcher::Fetcher, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(q): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let replies_id = url!(ctx, "/objects/{id}/replies");
-	let oid = ctx.oid(&id);
-
-	if auth.is_local() && q.fetch {
-		ctx.fetch_thread(&oid).await?;
-	}
-
-	let count = model::addressing::Entity::find_addressed(auth.my_id())
-		.filter(auth.filter_condition())
-		.filter(model::object::Column::InReplyTo.eq(oid))
-		.count(ctx.db())
-		.await?;
-
-	crate::server::builders::collection(&replies_id, Some(count))
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let page_id = url!(ctx, "/objects/{id}/replies/page");
-	let oid = ctx.oid(&id);
-
-	crate::server::builders::paginate(
-		page_id,
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(model::object::Column::InReplyTo.eq(oid)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}

src/routes/activitypub/outbox.rs

@@ -1,31 +0,0 @@
-use axum::{extract::{Query, State}, http::StatusCode, Json};
-
-use crate::{errors::UpubError, routes::activitypub::{CreationResult, JsonLD, Pagination}, server::{auth::AuthIdentity, Context}, url};
-
-pub async fn get(State(ctx): State<Context>) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/outbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::paginate(
-		url!(ctx, "/outbox/page"),
-		auth.filter_condition(), // TODO filter local only stuff
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}
-
-pub async fn post(
-	State(_ctx): State<Context>,
-	AuthIdentity(_auth): AuthIdentity,
-	Json(_activity): Json<serde_json::Value>,
-) -> Result<CreationResult, UpubError> {
-	// TODO administrative actions may be carried out against this outbox?
-	Err(StatusCode::NOT_IMPLEMENTED.into())
-}

src/routes/activitypub/user/following.rs

@@ -1,47 +0,0 @@
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use crate::{routes::activitypub::{JsonLD, Pagination}, model, server::Context, url};
-
-use model::relation::Column::{Following, Follower};
-
-pub async fn get<const OUTGOING: bool>(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let follow___ = if OUTGOING { "following" } else { "followers" };
-	let count = model::relation::Entity::find()
-		.filter(if OUTGOING { Follower } else { Following }.eq(ctx.uid(&id)))
-		.count(ctx.db()).await.unwrap_or_else(|e| {
-			tracing::error!("failed counting {follow___} for {id}: {e}");
-			0
-		});
-
-	crate::server::builders::collection(&url!(ctx, "/users/{id}/{follow___}"), Some(count))
-}
-
-pub async fn page<const OUTGOING: bool>(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let follow___ = if OUTGOING { "following" } else { "followers" };
-	let limit = page.batch.unwrap_or(20).min(50);
-	let offset = page.offset.unwrap_or(0);
-
-	let following = model::relation::Entity::find()
-		.filter(if OUTGOING { Follower } else { Following }.eq(ctx.uid(&id)))
-		.select_only()
-		.select_column(if OUTGOING { Following } else { Follower })
-		.limit(limit)
-		.offset(page.offset.unwrap_or(0))
-		.into_tuple::<String>()
-		.all(ctx.db())
-		.await?;
-
-	crate::server::builders::collection_page(
-		&url!(ctx, "/users/{id}/{follow___}/page"),
-		offset, limit,
-		following.into_iter().map(serde_json::Value::String).collect()
-	)
-}

src/routes/activitypub/user/inbox.rs

@@ -1,57 +0,0 @@
-use axum::{extract::{Path, Query, State}, http::StatusCode, Json};
-
-use sea_orm::{ColumnTrait, Condition};
-use crate::{errors::UpubError, model, routes::activitypub::{JsonLD, Pagination}, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	match auth {
-		Identity::Anonymous => Err(StatusCode::FORBIDDEN.into()),
-		Identity::Remote(_) => Err(StatusCode::FORBIDDEN.into()),
-		Identity::Local(user) => if ctx.uid(&id) == user {
-			crate::server::builders::collection(&url!(ctx, "/users/{id}/inbox"), None)
-		} else {
-			Err(StatusCode::FORBIDDEN.into())
-		},
-	}
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Query(page): Query<Pagination>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let Identity::Local(uid) = &auth else {
-		// local inbox is only for local users
-		return Err(UpubError::forbidden());
-	};
-	if uid != &ctx.uid(&id) {
-		return Err(UpubError::forbidden());
-	}
-
-	crate::server::builders::paginate(
-		url!(ctx, "/users/{id}/inbox/page"),
-		Condition::any()
-			.add(model::addressing::Column::Actor.eq(uid))
-			.add(model::object::Column::AttributedTo.eq(uid))
-			.add(model::activity::Column::Actor.eq(uid)),
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}
-
-pub async fn post(
-	State(ctx): State<Context>,
-	Path(_id): Path<String>,
-	AuthIdentity(_auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>,
-) -> Result<(), UpubError> {
-	// POSTing to user inboxes is effectively the same as POSTing to the main inbox
-	super::super::inbox::post(State(ctx), AuthIdentity(_auth), Json(activity)).await
-}

src/routes/activitypub/user/mod.rs

@@ -1,103 +0,0 @@
-pub mod inbox;
-
-pub mod outbox;
-
-pub mod following;
-
-use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QuerySelect, SelectColumns};
-
-use apb::{ActorMut, EndpointsMut, Node};
-use crate::{errors::UpubError, model, server::{auth::AuthIdentity, fetcher::Fetcher, Context}, url};
-
-use super::{jsonld::LD, JsonLD, TryFetch};
-
-
-pub async fn view(
-	State(ctx) : State<Context>,
-	AuthIdentity(auth): AuthIdentity,
-	Path(id): Path<String>,
-	Query(query): Query<TryFetch>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let mut uid = ctx.uid(&id);
-	if auth.is_local() {
-		if id.starts_with('@') {
-			if let Some((user, host)) = id.replacen('@', "", 1).split_once('@') {
-				uid = ctx.webfinger(user, host).await?;
-			}
-		}
-		if query.fetch && !ctx.is_local(&uid) {
-			ctx.fetch_user(&uid).await?;
-		}
-	}
-
-	let (followed_by_me, following_me) = match auth.my_id() {
-		None => (None, None),
-		Some(my_id) => {
-			// TODO these two queries are fast because of indexes but still are 2 subqueries for each
-			// user GET, not even parallelized... should really add these as joins on the main query, so
-			// that it's one roundtrip only
-			let followed_by_me = model::relation::Entity::find()
-				.filter(model::relation::Column::Follower.eq(my_id))
-				.filter(model::relation::Column::Following.eq(&uid))
-				.select_only()
-				.select_column(model::relation::Column::Follower)
-				.into_tuple::<String>()
-				.one(ctx.db())
-				.await?
-				.map(|_| true);
-
-			let following_me = model::relation::Entity::find()
-				.filter(model::relation::Column::Following.eq(my_id))
-				.filter(model::relation::Column::Follower.eq(&uid))
-				.select_only()
-				.select_column(model::relation::Column::Follower)
-				.into_tuple::<String>()
-				.one(ctx.db())
-				.await?
-				.map(|_| true);
-
-			(followed_by_me, following_me)
-		},
-	};
-
-	match model::actor::Entity::find_by_ap_id(&uid)
-		.find_also_related(model::config::Entity)
-		.one(ctx.db()).await?
-	{
-		// local user
-		Some((user_model, Some(cfg))) => {
-			let mut user = user_model.ap()
-				.set_inbox(Node::link(url!(ctx, "/users/{id}/inbox")))
-				.set_outbox(Node::link(url!(ctx, "/users/{id}/outbox")))
-				.set_following(Node::link(url!(ctx, "/users/{id}/following")))
-				.set_followers(Node::link(url!(ctx, "/users/{id}/followers")))
-				.set_following_me(following_me)
-				.set_followed_by_me(followed_by_me)
-				.set_endpoints(Node::object(
-					serde_json::Value::new_object()
-						.set_shared_inbox(Some(&url!(ctx, "/inbox")))
-						.set_proxy_url(Some(&url!(ctx, "/proxy")))
-				));
-
-			if !auth.is(&uid) && !cfg.show_followers_count {
-				user = user.set_followers_count(None);
-			}
-
-			if !auth.is(&uid) && !cfg.show_following_count {
-				user = user.set_following_count(None);
-			}
-
-			Ok(JsonLD(user.ld_context()))
-		},
-		// remote user
-		Some((user_model, None)) => Ok(JsonLD(
-			user_model.ap()
-				.set_following_me(following_me)
-				.set_followed_by_me(followed_by_me)
-				.ld_context()
-		)),
-		None => Err(UpubError::not_found()),
-	}
-}
-

src/routes/activitypub/user/outbox.rs

@@ -1,89 +0,0 @@
-use axum::{extract::{Path, Query, State}, http::StatusCode, Json};
-use sea_orm::{ColumnTrait, Condition};
-
-use apb::{server::Outbox, AcceptType, ActivityType, Base, BaseType, ObjectType, RejectType};
-use crate::{errors::UpubError, model, routes::activitypub::{CreationResult, JsonLD, Pagination}, server::{auth::{AuthIdentity, Identity}, Context}, url};
-
-pub async fn get(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	crate::server::builders::collection(&url!(ctx, "/users/{id}/outbox"), None)
-}
-
-pub async fn page(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	Query(page): Query<Pagination>,
-	AuthIdentity(auth): AuthIdentity,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let uid = ctx.uid(&id);
-	crate::server::builders::paginate(
-		url!(ctx, "/users/{id}/outbox/page"),
-		Condition::all()
-			.add(auth.filter_condition())
-			.add(
-				Condition::any()
-					.add(model::activity::Column::Actor.eq(&uid))
-					.add(model::object::Column::AttributedTo.eq(&uid))
-				),
-		ctx.db(),
-		page,
-		auth.my_id(),
-	)
-		.await
-}
-
-pub async fn post(
-	State(ctx): State<Context>,
-	Path(id): Path<String>,
-	AuthIdentity(auth): AuthIdentity,
-	Json(activity): Json<serde_json::Value>,
-) -> Result<CreationResult, UpubError> {
-	match auth {
-		Identity::Anonymous => Err(StatusCode::UNAUTHORIZED.into()),
-		Identity::Remote(_) => Err(StatusCode::NOT_IMPLEMENTED.into()),
-		Identity::Local(uid) => if ctx.uid(&id) == uid {
-			tracing::debug!("processing new local activity: {}", serde_json::to_string(&activity).unwrap_or_default());
-			match activity.base_type() {
-				None => Err(StatusCode::BAD_REQUEST.into()),
-
-				Some(BaseType::Link(_)) => Err(StatusCode::UNPROCESSABLE_ENTITY.into()),
-
-				Some(BaseType::Object(ObjectType::Note)) =>
-					Ok(CreationResult(ctx.create_note(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Create))) =>
-					Ok(CreationResult(ctx.create(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Like))) =>
-					Ok(CreationResult(ctx.like(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Follow))) =>
-					Ok(CreationResult(ctx.follow(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Announce))) =>
-					Ok(CreationResult(ctx.announce(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Accept(AcceptType::Accept)))) =>
-					Ok(CreationResult(ctx.accept(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Reject(RejectType::Reject)))) =>
-					Ok(CreationResult(ctx.reject(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Undo))) =>
-					Ok(CreationResult(ctx.undo(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Delete))) =>
-					Ok(CreationResult(ctx.delete(uid, activity).await?)),
-
-				Some(BaseType::Object(ObjectType::Activity(ActivityType::Update))) =>
-					Ok(CreationResult(ctx.update(uid, activity).await?)),
-
-				Some(_) => Err(StatusCode::NOT_IMPLEMENTED.into()),
-			}
-		} else {
-			Err(StatusCode::FORBIDDEN.into())
-		}
-	}
-}

src/routes/mod.rs

@@ -1,16 +0,0 @@
-pub mod activitypub;
-
-#[cfg(feature = "web")]
-pub mod web;
-
-#[cfg(feature = "mastodon")]
-pub mod mastodon;
-
-#[cfg(not(feature = "mastodon"))]
-pub mod mastodon {
-	pub trait MastodonRouter {
-		fn mastodon_routes(self) -> Self where Self: Sized { self }
-	}
-	
-	impl MastodonRouter for axum::Router<crate::server::Context> {}
-}

src/server/auth.rs

@@ -1,146 +0,0 @@
-use axum::{extract::{FromRef, FromRequestParts}, http::{header, request::Parts}};
-use reqwest::StatusCode;
-use sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter};
-
-use crate::{errors::UpubError, model, server::Context};
-
-use super::{fetcher::Fetcher, httpsign::HttpSignature};
-
-#[derive(Debug, Clone)]
-pub enum Identity {
-	Anonymous,
-	Local(i64),
-	Remote(i64),
-}
-
-impl Identity {
-	pub fn filter_condition(&self) -> Condition {
-		let base_cond = Condition::any().add(model::addressing::Column::Actor.eq(apb::target::PUBLIC));
-		match self {
-			Identity::Anonymous => base_cond,
-			Identity::Remote(server_id) => base_cond.add(model::addressing::Column::Instance.eq(*server_id)),
-			// TODO should we allow all users on same server to see? or just specific user??
-			Identity::Local(user_id) => base_cond
-				.add(model::addressing::Column::Actor.eq(*user_id))
-				.add(model::activity::Column::Actor.eq(*user_id))
-				.add(model::object::Column::AttributedTo.eq(*user_id)),
-		}
-	}
-
-	pub fn user_id(&self) -> Option<i64> {
-		match self {
-			Identity::Local(x) => Some(*x),
-			_ => None,
-		}
-	}
-
-	pub fn server_id(&self) -> Option<i64> {
-		match self {
-			Identity::Remote(x) => Some(*x),
-			_ => None,
-		}
-	}
-
-	pub fn is(&self, id: i64) -> bool {
-		match self {
-			Identity::Anonymous => false,
-			Identity::Remote(_) => false, // TODO per-actor server auth should check this
-			Identity::Local(user_id) => *user_id == id
-		}
-	}
-
-	pub fn is_anon(&self) -> bool {
-		matches!(self, Self::Anonymous)
-	}
-
-	pub fn is_local(&self) -> bool {
-		matches!(self, Self::Local(_))
-	}
-
-	pub fn is_remote(&self) -> bool {
-		matches!(self, Self::Remote(_))
-	}
-
-	pub fn is_user(&self, usr: i64) -> bool {
-		self.user_id().map(|id| id == usr).unwrap_or(false)
-	}
-
-	pub fn is_server(&self, server: i64) -> bool {
-		self.server_id().map(|id| id == server).unwrap_or(false)
-	}
-}
-
-pub struct AuthIdentity(pub Identity);
-
-#[axum::async_trait]
-impl<S> FromRequestParts<S> for AuthIdentity
-where
-	Context: FromRef<S>,
-	S: Send + Sync,
-{
-	type Rejection = UpubError;
-
-	async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
-		let ctx = Context::from_ref(state);
-		let mut identity = Identity::Anonymous;
-
-		let auth_header = parts
-			.headers
-			.get(header::AUTHORIZATION)
-			.map(|v| v.to_str().unwrap_or(""))
-			.unwrap_or("");
-
-		if auth_header.starts_with("Bearer ") {
-			match model::session::Entity::find_by_secret(&auth_header.replace("Bearer ", ""))
-				.filter(model::session::Column::Expires.gt(chrono::Utc::now()))
-				.one(ctx.db())
-				.await
-			{
-				Ok(Some(x)) => identity = Identity::Local(x.actor),
-				Ok(None) => return Err(UpubError::unauthorized()),
-				Err(e) => {
-					tracing::error!("failed querying user session: {e}");
-					return Err(UpubError::internal_server_error())
-				},
-			}
-		}
-
-		if let Some(sig) = parts
-			.headers
-			.get("Signature")
-			.map(|v| v.to_str().unwrap_or(""))
-		{
-			let mut http_signature = HttpSignature::parse(sig);
-
-			// TODO assert payload's digest is equal to signature's
-			let user_id = http_signature.key_id
-				.split('#')
-				.next().ok_or(UpubError::bad_request())?
-				.to_string();
-
-			match ctx.fetch_user(&user_id).await {
-				Ok(user) => match http_signature
-						.build_from_parts(parts)
-						.verify(&user.public_key)
-					{
-						Ok(true) => identity = Identity::Remote(Context::server(&user_id)),
-						Ok(false) => tracing::warn!("invalid signature: {http_signature:?}"),
-						Err(e) => tracing::error!("error verifying signature: {e}"),
-					},
-				Err(e) => {
-					// since most activities are deletions for users we never saw, let's handle this case
-					// if while fetching we receive a GONE, it means we didn't have this user and it doesn't
-					// exist anymore, so it must be a deletion we can ignore
-					if let UpubError::Reqwest(ref x) = e {
-						if let Some(StatusCode::GONE) = x.status() {
-							return Err(UpubError::Status(StatusCode::OK)); // 200 so mastodon will shut uppp
-						}
-					}
-					tracing::warn!("could not fetch user (won't verify): {e}");
-				}
-			}
-		}
-
-		Ok(AuthIdentity(identity))
-	}
-}

src/server/builders.rs

@@ -1,65 +0,0 @@
-use apb::{BaseMut, CollectionMut, CollectionPageMut};
-use sea_orm::{Condition, DatabaseConnection, QueryFilter, QuerySelect};
-
-use crate::{model::{addressing::Event, attachment::BatchFillable}, routes::activitypub::{jsonld::LD, JsonLD, Pagination}};
-
-pub async fn paginate(
-	id: String,
-	filter: Condition,
-	db: &DatabaseConnection,
-	page: Pagination,
-	my_id: Option<&str>,
-) -> crate::Result<JsonLD<serde_json::Value>> {
-	let limit = page.batch.unwrap_or(20).min(50);
-	let offset = page.offset.unwrap_or(0);
-
-	let items = crate::model::addressing::Entity::find_addressed(my_id)
-		.filter(filter)
-		// TODO also limit to only local activities
-		.limit(limit)
-		.offset(offset)
-		.into_model::<Event>()
-		.all(db)
-		.await?;
-
-	let mut attachments = items.load_attachments_batch(db).await?;
-
-	let items : Vec<serde_json::Value> = items
-		.into_iter()
-		.map(|item| {
-			let attach = attachments.remove(item.id());
-			item.ap(attach)
-		})
-		.collect();
-
-	collection_page(&id, offset, limit, items)
-}
-
-pub fn collection_page(id: &str, offset: u64, limit: u64, items: Vec<serde_json::Value>) -> crate::Result<JsonLD<serde_json::Value>> {
-	let next = if items.len() < limit as usize {
-		apb::Node::Empty
-	} else {
-		apb::Node::link(format!("{id}?offset={}", offset+limit))
-	};
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(&format!("{id}?offset={offset}")))
-			.set_collection_type(Some(apb::CollectionType::OrderedCollectionPage))
-			.set_part_of(apb::Node::link(id.replace("/page", "")))
-			.set_ordered_items(apb::Node::array(items))
-			.set_next(next)
-			.ld_context()
-	))
-}
-
-
-pub fn collection(id: &str, total_items: Option<u64>) -> crate::Result<JsonLD<serde_json::Value>> {
-	Ok(JsonLD(
-		serde_json::Value::new_object()
-			.set_id(Some(id))
-			.set_collection_type(Some(apb::CollectionType::OrderedCollection))
-			.set_first(apb::Node::link(format!("{id}/page")))
-			.set_total_items(total_items)
-			.ld_context()
-	))
-}

src/server/context.rs

@@ -1,256 +0,0 @@
-use std::{collections::BTreeSet, sync::Arc};
-
-use openssl::rsa::Rsa;
-use sea_orm::{ActiveValue::{Set, NotSet}, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QuerySelect, SelectColumns, Set};
-
-use crate::{config::Config, model, server::fetcher::Fetcher};
-use uriproxy::UriClass;
-
-use super::dispatcher::Dispatcher;
-
-
-#[derive(Clone)]
-pub struct Context(Arc<ContextInner>);
-struct ContextInner {
-	db: DatabaseConnection,
-	config: Config,
-	domain: String,
-	protocol: String,
-	base_url: String,
-	dispatcher: Dispatcher,
-	// TODO keep these pre-parsed
-	app: model::actor::Model,
-	relays: BTreeSet<String>,
-}
-
-#[macro_export]
-macro_rules! url {
-	($ctx:expr, $($args: tt)*) => {
-		format!("{}{}{}", $ctx.protocol(), $ctx.domain(), format!($($args)*))
-	};
-}
-
-impl Context {
-
-	// TODO slim constructor down, maybe make a builder?
-	pub async fn new(db: DatabaseConnection, mut domain: String, config: Config) -> crate::Result<Self> {
-		let protocol = if domain.starts_with("http://")
-		{ "http://" } else { "https://" }.to_string();
-		if domain.ends_with('/') {
-			domain.replace_range(domain.len()-1.., "");
-		}
-		if domain.starts_with("http") {
-			domain = domain.replace("https://", "").replace("http://", "");
-		}
-		let base_url = format!("{}{}", protocol, domain);
-
-		let dispatcher = Dispatcher::default();
-		for _ in 0..1 { // TODO customize delivery workers amount
-			dispatcher.spawn(db.clone(), domain.clone(), 30); // TODO ew don't do it this deep and secretly!!
-		}
-
-		let app = match model::actor::Entity::find_by_ap_id(&base_url).one(&db).await? {
-			Some(model) => model,
-			None => {
-				tracing::info!("generating application keys");
-				let rsa = Rsa::generate(2048)?;
-				let privk = std::str::from_utf8(&rsa.private_key_to_pem()?)?.to_string();
-				let pubk = std::str::from_utf8(&rsa.public_key_to_pem()?)?.to_string();
-				let system = model::actor::ActiveModel {
-					id: NotSet,
-					ap_id: Set(base_url.clone()),
-					instance: NotSet, // TODO!!! this will fail
-					preferred_username: Set(domain.clone()),
-					name: Set(Some("μpub".to_string())),
-					icon: Set(Some("https://cdn.alemi.dev/social/circle-square.png".to_string())),
-					actor_type: Set(apb::ActorType::Application),
-					private_key: Set(Some(privk.clone())),
-					public_key: Set(pubk.clone()),
-					created: Set(chrono::Utc::now()),
-					updated: Set(chrono::Utc::now()),
-					..Default::default()
-				};
-				model::actor::Entity::insert(system).exec(&db).await?;
-				// sqlite doesn't resurn last inserted id so we're better off just querying again, it's just one time
-				model::actor::Entity::find_by_ap_id(&base_url).one(&db).await?.expect("could not find app config just inserted")
-			}
-		};
-
-		let relays = model::relation::Entity::find_followers(&base_url)
-			.into_tuple::<String>()
-			.all(&db)
-			.await?;
-
-		Ok(Context(Arc::new(ContextInner {
-			base_url, db, domain, protocol, app, dispatcher, config,
-			relays: BTreeSet::from_iter(relays.into_iter()),
-		})))
-	}
-
-	pub fn app(&self) -> &model::actor::Model {
-		&self.0.app
-	}
-
-	pub fn db(&self) -> &DatabaseConnection {
-		&self.0.db
-	}
-
-	pub fn cfg(&self) -> &Config {
-		&self.0.config
-	}
-
-	pub fn domain(&self) -> &str {
-		&self.0.domain
-	}
-
-	pub fn protocol(&self) -> &str {
-		&self.0.protocol
-	}
-
-	pub fn base(&self) -> &str {
-		&self.0.base_url
-	}
-
-	/// get full user id uri
-	pub fn uid(&self, id: &str) -> String {
-		uriproxy::uri(self.base(), UriClass::User, id)
-	}
-
-	/// get full object id uri
-	pub fn oid(&self, id: &str) -> String {
-		uriproxy::uri(self.base(), UriClass::Object, id)
-	}
-
-	/// get full activity id uri
-	pub fn aid(&self, id: &str) -> String {
-		uriproxy::uri(self.base(), UriClass::Activity, id)
-	}
-
-	// TODO remove this!!
-	pub fn context_id(&self, id: &str) -> String {
-		if id.starts_with("tag:") {
-			return id.to_string();
-		}
-		uriproxy::uri(self.base(), UriClass::Context, id)
-	}
-
-	/// get bare id, which is uuid for local stuff and +{uri|base64} for remote stuff
-	pub fn id(&self, full_id: &str) -> String {
-		if self.is_local(full_id) {
-			uriproxy::decompose_id(full_id)
-		} else {
-			uriproxy::compact_id(full_id)
-		}
-	}
-
-	pub fn server(id: &str) -> String {
-		id
-			.replace("https://", "")
-			.replace("http://", "")
-			.split('/')
-			.next()
-			.unwrap_or("")
-			.to_string()
-	}
-
-	pub fn is_local(&self, id: &str) -> bool {
-		id.starts_with(self.base())
-	}
-
-	pub async fn expand_addressing(&self, targets: Vec<String>) -> crate::Result<Vec<String>> {
-		let mut out = Vec::new();
-		for target in targets {
-			if target.ends_with("/followers") {
-				let target_id = target.replace("/followers", "");
-				model::relation::Entity::find()
-					.filter(model::relation::Column::Following.eq(target_id))
-					.select_only()
-					.select_column(model::relation::Column::Follower)
-					.into_tuple::<String>()
-					.all(self.db())
-					.await?
-					.into_iter()
-					.for_each(|x| out.push(x));
-			} else {
-				out.push(target);
-			}
-		}
-		Ok(out)
-	}
-
-	pub async fn address_to(&self, aid: Option<&str>, oid: Option<&str>, targets: &[String]) -> crate::Result<()> {
-		let local_activity = aid.map(|x| self.is_local(x)).unwrap_or(false);
-		let local_object = oid.map(|x| self.is_local(x)).unwrap_or(false);
-		let addressings : Vec<model::addressing::ActiveModel> = targets
-			.iter()
-			.filter(|to| !to.is_empty())
-			.filter(|to| !to.ends_with("/followers"))
-			.filter(|to| local_activity || local_object || to.as_str() == apb::target::PUBLIC || self.is_local(to))
-			.map(|to| model::addressing::ActiveModel {
-				id: sea_orm::ActiveValue::NotSet,
-				server: Set(Context::server(to)),
-				actor: Set(to.to_string()),
-				activity: Set(aid.map(|x| x.to_string())),
-				object: Set(oid.map(|x| x.to_string())),
-				published: Set(chrono::Utc::now()),
-			})
-			.collect();
-
-		if !addressings.is_empty() {
-			model::addressing::Entity::insert_many(addressings)
-				.exec(self.db())
-				.await?;
-		}
-
-		Ok(())
-	}
-
-	pub async fn deliver_to(&self, aid: &str, from: &str, targets: &[String]) -> crate::Result<()> {
-		let mut deliveries = Vec::new();
-		for target in targets.iter()
-			.filter(|to| !to.is_empty())
-			.filter(|to| Context::server(to) != self.domain())
-			.filter(|to| to != &apb::target::PUBLIC)
-		{
-			// TODO fetch concurrently
-			match self.fetch_user(target).await {
-				Ok(model::user::Model { inbox: Some(inbox), .. }) => deliveries.push(
-					model::delivery::ActiveModel {
-						id: sea_orm::ActiveValue::NotSet,
-						actor: Set(from.to_string()),
-						// TODO we should resolve each user by id and check its inbox because we can't assume
-						// it's /users/{id}/inbox for every software, but oh well it's waaaaay easier now
-						target: Set(inbox),
-						activity: Set(aid.to_string()),
-						created: Set(chrono::Utc::now()),
-						not_before: Set(chrono::Utc::now()),
-						attempt: Set(0),
-					}
-				),
-				Ok(_) => tracing::error!("resolved target but missing inbox: '{target}', skipping delivery"),
-				Err(e) => tracing::error!("failed resolving target inbox: {e}, skipping delivery to '{target}'"),
-			}
-		}
-
-		if !deliveries.is_empty() {
-			model::delivery::Entity::insert_many(deliveries)
-				.exec(self.db())
-				.await?;
-		}
-
-		self.0.dispatcher.wakeup();
-
-		Ok(())
-	}
-
-	pub async fn dispatch(&self, uid: &str, activity_targets: Vec<String>, aid: &str, oid: Option<&str>) -> crate::Result<()> {
-		let addressed = self.expand_addressing(activity_targets).await?;
-		self.address_to(Some(aid), oid, &addressed).await?;
-		self.deliver_to(aid, uid, &addressed).await?;
-		Ok(())
-	}
-
-	pub fn is_relay(&self, id: &str) -> bool {
-		self.0.relays.contains(id)
-	}
-}

src/server/dispatcher.rs

@@ -1,133 +0,0 @@
-use reqwest::Method;
-use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, Order, QueryFilter, QueryOrder};
-use tokio::{sync::broadcast, task::JoinHandle};
-
-use apb::{ActivityMut, Node};
-use crate::{model, routes::activitypub::jsonld::LD, server::{fetcher::Fetcher, Context}};
-
-pub struct Dispatcher {
-	waker: broadcast::Sender<()>,
-}
-
-impl Default for Dispatcher {
-	fn default() -> Self {
-		let (waker, _) = broadcast::channel(1);
-		Dispatcher { waker }
-	}
-}
-
-impl Dispatcher {
-	pub fn spawn(&self, db: DatabaseConnection, domain: String, poll_interval: u64) -> JoinHandle<()> {
-		let mut waker = self.waker.subscribe();
-		tokio::spawn(async move {
-			loop {
-				if let Err(e) = worker(&db, &domain, poll_interval, &mut waker).await {
-					tracing::error!("delivery worker exited with error: {e}");
-				}
-				tokio::time::sleep(std::time::Duration::from_secs(poll_interval * 10)).await;
-			}
-		})
-	}
-
-	pub fn wakeup(&self) {
-		match self.waker.send(()) {
-			Err(_) => tracing::error!("no worker to wakeup"), 
-			Ok(n) => tracing::debug!("woken {n} workers"),
-		}
-	}
-}
-
-async fn worker(db: &DatabaseConnection, domain: &str, poll_interval: u64, waker: &mut broadcast::Receiver<()>) -> crate::Result<()> {
-	loop {
-		let Some(delivery) = model::delivery::Entity::find()
-			.filter(model::delivery::Column::NotBefore.lte(chrono::Utc::now()))
-			.order_by(model::delivery::Column::NotBefore, Order::Asc)
-			.one(db)
-			.await?
-		else {
-			tokio::select! {
-				biased;
-				_ = waker.recv() => {},
-				_ = tokio::time::sleep(std::time::Duration::from_secs(poll_interval)) => {},
-			}
-			continue
-		};
-
-		let del_row = model::delivery::ActiveModel {
-			id: sea_orm::ActiveValue::Set(delivery.id),
-			..Default::default()
-		};
-		let del = model::delivery::Entity::delete(del_row)
-			.exec(db)
-			.await?;
-
-		if del.rows_affected == 0 {
-			// another worker claimed this delivery
-			continue; // go back to the top
-		}
-		if delivery.expired() {
-			// try polling for another one
-			continue; // go back to top
-		}
-
-		tracing::info!("delivering {} to {}", delivery.activity, delivery.target);
-
-		let payload = match model::activity::Entity::find_by_id(delivery.activity)
-			.find_also_related(model::object::Entity)
-			.one(db)
-			.await? // TODO probably should not fail here and at least re-insert the delivery
-		{
-			Some((activity, None)) => activity.ap().ld_context(),
-			Some((activity, Some(object))) => {
-				let always_embed = matches!(
-					activity.activity_type,
-					apb::ActivityType::Create
-					| apb::ActivityType::Undo
-					| apb::ActivityType::Update
-					| apb::ActivityType::Accept(_)
-					| apb::ActivityType::Reject(_)
-				);
-				if always_embed {
-					activity.ap().set_object(Node::object(object.ap())).ld_context()
-				} else {
-					activity.ap().ld_context()
-				}
-			},
-			None => {
-				tracing::warn!("skipping dispatch for deleted object {}", delivery.activity);
-				continue;
-			},
-		};
-
-		let Some(actor) = model::actor::Entity::find_by_id(delivery.actor)
-			.one(db)
-			.await?
-		else {
-			tracing::error!("failed delivery, missing actor {}", delivery.actor);
-			continue;
-		};
-
-		let Some(key) = actor.private_key else {
-			tracing::error!("can not dispatch activity for actor without private key: {}", delivery.actor);
-			continue;
-		};
-
-		if let Err(e) = Context::request(
-			Method::POST, &delivery.target,
-			Some(&serde_json::to_string(&payload).unwrap()),
-			&actor.ap_id, &key, domain
-		).await {
-			tracing::warn!("failed delivery of {} to {} : {e}", delivery.activity, delivery.target);
-			let new_delivery = model::delivery::ActiveModel {
-				id: sea_orm::ActiveValue::NotSet,
-				not_before: sea_orm::ActiveValue::Set(delivery.next_delivery()),
-				actor: sea_orm::ActiveValue::Set(delivery.actor),
-				target: sea_orm::ActiveValue::Set(delivery.target),
-				activity: sea_orm::ActiveValue::Set(delivery.activity),
-				created: sea_orm::ActiveValue::Set(delivery.created),
-				attempt: sea_orm::ActiveValue::Set(delivery.attempt + 1),
-			};
-			model::delivery::Entity::insert(new_delivery).exec(db).await?;
-		}
-	}
-}

src/server/fetcher.rs

@@ -1,371 +0,0 @@
-use std::collections::BTreeMap;
-
-use apb::{target::Addressed, Activity, Actor, ActorMut, Base, Collection, CollectionPage, Link, Object};
-use base64::Engine;
-use reqwest::{header::{ACCEPT, CONTENT_TYPE, USER_AGENT}, Method, Response};
-use sea_orm::EntityTrait;
-
-use crate::{errors::UpubError, model, VERSION};
-
-use super::{httpsign::HttpSignature, normalizer::Normalizer, Context};
-
-#[axum::async_trait]
-pub trait Fetcher {
-	async fn webfinger(&self, user: &str, host: &str) -> crate::Result<String>;
-
-	async fn fetch_user(&self, id: &str) -> crate::Result<model::actor::Model>;
-	async fn pull_user(&self, id: &str) -> crate::Result<serde_json::Value>;
-
-	async fn fetch_object(&self, id: &str) -> crate::Result<model::object::Model>;
-	async fn pull_object(&self, id: &str) -> crate::Result<serde_json::Value>;
-
-	async fn fetch_activity(&self, id: &str) -> crate::Result<model::activity::Model>;
-	async fn pull_activity(&self, id: &str) -> crate::Result<serde_json::Value>;
-
-	async fn fetch_thread(&self, id: &str) -> crate::Result<()>;
-
-	async fn request(
-		method: reqwest::Method,
-		url: &str,
-		payload: Option<&str>,
-		from: &str,
-		key: &str,
-		domain: &str,
-	) -> crate::Result<Response> {
-		let host = Context::server(url);
-		let date = chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string(); // lmao @ "GMT"
-		let path = url.replace("https://", "").replace("http://", "").replace(&host, "");
-		let digest = format!("sha-256={}",
-			base64::prelude::BASE64_STANDARD.encode(
-				openssl::sha::sha256(payload.unwrap_or("").as_bytes())
-			)
-		);
-
-		let headers = vec!["(request-target)", "host", "date", "digest"];
-		let headers_map : BTreeMap<String, String> = [
-			("host".to_string(), host.clone()),
-			("date".to_string(), date.clone()),
-			("digest".to_string(), digest.clone()),
-		].into();
-
-		let mut signer = HttpSignature::new(
-			format!("{from}#main-key"), // TODO don't hardcode #main-key
-			//"hs2019".to_string(), // pixelfeed/iceshrimp made me go back
-			"rsa-sha256".to_string(),
-			&headers,
-		);
-		
-		signer
-			.build_manually(&method.to_string().to_lowercase(), &path, headers_map)
-			.sign(key)?;
-
-		let response = reqwest::Client::new()
-			.request(method.clone(), url)
-			.header(ACCEPT, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")
-			.header(CONTENT_TYPE, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")
-			.header(USER_AGENT, format!("upub+{VERSION} ({domain})"))
-			.header("Host", host.clone())
-			.header("Date", date.clone())
-			.header("Digest", digest)
-			.header("Signature", signer.header())
-			.body(payload.unwrap_or("").to_string())
-			.send()
-			.await?;
-
-		// TODO this is ugly but i want to see the raw response text when it's a failure
-		match response.error_for_status_ref() {
-			Ok(_) => Ok(response),
-			Err(e) => Err(UpubError::FetchError(e, response.text().await?)),
-		}
-	}
-}
-
-
-#[axum::async_trait]
-impl Fetcher for Context {
-	async fn webfinger(&self, user: &str, host: &str) -> crate::Result<String> {
-		let subject = format!("acct:{user}@{host}");
-		let webfinger_uri = format!("https://{host}/.well-known/webfinger?resource={subject}");
-		let resource = reqwest::Client::new()
-			.get(webfinger_uri)
-			.header(ACCEPT, "application/jrd+json")
-			.header(USER_AGENT, format!("upub+{VERSION} ({})", self.domain()))
-			.send()
-			.await?
-			.json::<jrd::JsonResourceDescriptor>()
-			.await?;
-
-		if resource.subject != subject {
-			return Err(UpubError::unprocessable());
-		}
-
-		for link in resource.links {
-			if link.rel == "self" {
-				if let Some(href) = link.href {
-					return Ok(href);
-				}
-			}
-		}
-
-		if let Some(alias) = resource.aliases.into_iter().next() {
-			return Ok(alias);
-		}
-
-		Err(UpubError::not_found())
-	}
-
-
-	async fn fetch_user(&self, id: &str) -> crate::Result<model::actor::Model> {
-		if let Some(x) = model::actor::Entity::find_by_ap_id(id).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		// TODO PULL INSTANCE!!!!!!!
-
-		let user = self.pull_user(id).await?;
-		let user_model = model::actor::ActiveModel::new(&user, 0)?;
-
-		// TODO this may fail: while fetching, remote server may fetch our service actor.
-		//      if it does so with http signature, we will fetch that actor in background
-		//      meaning that, once we reach here, it's already inserted and returns an UNIQUE error
-		model::actor::Entity::insert(user_model)
-			.exec(self.db()).await?;
-
-		// TODO we could fetch only the internal id and avoid getting back the whole user, but this
-		// happens rarely anyway because after the first time we just get the cached one in our db
-		let user = model::actor::Entity::find_by_ap_id(id)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::internal_server_error)?;
-
-		Ok(user)
-	}
-
-	async fn pull_user(&self, id: &str) -> crate::Result<serde_json::Value> {
-		let pkey = self.app().private_key.ok_or_else(UpubError::internal_server_error)?;
-
-		let mut user = Self::request(
-			Method::GET, id, None, &format!("https://{}", self.domain()), &pkey, self.domain(),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?;
-
-		// TODO try fetching these numbers from audience/generator fields
-		if let Some(followers_url) = user.followers().id() {
-			let req = Self::request(
-				Method::GET, &followers_url, None,
-				&format!("https://{}", self.domain()), &pkey, self.domain(),
-			).await;
-			if let Ok(res) = req {
-				if let Ok(user_followers) = res.json::<serde_json::Value>().await {
-					if let Some(total) = user_followers.total_items() {
-						user = user.set_followers_count(Some(total));
-					}
-				}
-			}
-		}
-
-		if let Some(following_url) = user.following().id() {
-			let req =  Self::request(
-				Method::GET, &following_url, None,
-				&format!("https://{}", self.domain()), &pkey, self.domain(),
-			).await;
-			if let Ok(res) = req {
-				if let Ok(user_following) = res.json::<serde_json::Value>().await {
-					if let Some(total) = user_following.total_items() {
-						user = user.set_following_count(Some(total));
-					}
-				}
-			}
-		}
-
-		Ok(user)
-	}
-
-	async fn fetch_activity(&self, id: &str) -> crate::Result<model::activity::Model> {
-		if let Some(x) = model::activity::Entity::find_by_ap_id(id).one(self.db()).await? {
-			return Ok(x); // already in db, easy
-		}
-
-		let activity = self.pull_activity(id).await?;
-		let activity_model = model::activity::ActiveModel::new(&activity)?;
-
-		model::activity::Entity::insert(activity_model).exec(self.db()).await?;
-	
-		let activity = model::activity::Entity::find_by_ap_id(id)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::internal_server_error)?;
-
-		let addressed = activity.addressed();
-		let expanded_addresses = self.expand_addressing(addressed).await?;
-		self.address_to(Some(&activity.ap_id), None, &expanded_addresses).await?;
-
-		Ok(activity)
-	}
-
-	async fn pull_activity(&self, id: &str) -> crate::Result<serde_json::Value> {
-		let pkey = self.app().private_key.ok_or_else(UpubError::internal_server_error)?;
-		let activity = Self::request(
-			Method::GET, id, None, &format!("https://{}", self.domain()), &pkey, self.domain(),
-		).await?.json::<serde_json::Value>().await?;
-
-		if let Some(activity_actor) = activity.actor().id() {
-			if let Err(e) = self.fetch_user(&activity_actor).await {
-				tracing::warn!("could not get actor of fetched activity: {e}");
-			}
-		}
-
-		if let Some(activity_object) = activity.object().id() {
-			if let Err(e) = self.fetch_object(&activity_object).await {
-				tracing::warn!("could not get object of fetched activity: {e}");
-			}
-		}
-
-		Ok(activity)
-	}
-
-	async fn fetch_thread(&self, id: &str) -> crate::Result<()> {
-		// crawl_replies(self, id, 0).await
-		todo!()
-	}
-
-	async fn fetch_object(&self, id: &str) -> crate::Result<model::object::Model> {
-		fetch_object_inner(self, id, 0).await
-	}
-
-	async fn pull_object(&self, id: &str) -> crate::Result<serde_json::Value> {
-		let pkey = self.app().private_key.ok_or_else(UpubError::internal_server_error)?;
-
-		let object = Context::request(
-			Method::GET, id, None, &format!("https://{}", self.domain()), &pkey, self.domain(),
-		)
-			.await?
-			.json::<serde_json::Value>()
-			.await?;
-
-		Ok(object)
-	}
-}
-
-#[async_recursion::async_recursion]
-async fn fetch_object_inner(ctx: &Context, id: &str, depth: usize) -> crate::Result<model::object::Model> {
-	if let Some(x) = model::object::Entity::find_by_ap_id(id).one(ctx.db()).await? {
-		return Ok(x); // already in db, easy
-	}
-
-	let pkey = ctx.app().private_key.ok_or_else(UpubError::internal_server_error)?;
-
-	let object = Context::request(
-		Method::GET, id, None, &format!("https://{}", ctx.domain()), &pkey, ctx.domain(),
-	).await?.json::<serde_json::Value>().await?;
-
-	if let Some(oid) = object.id() {
-		if oid != id {
-			if let Some(x) = model::object::Entity::find_by_ap_id(oid).one(ctx.db()).await? {
-				return Ok(x); // already in db, but with id different that given url
-			}
-		}
-	}
-
-	if let Some(attributed_to) = object.attributed_to().id() {
-		if let Err(e) = ctx.fetch_user(&attributed_to).await {
-			tracing::warn!("could not get actor of fetched object: {e}");
-		}
-	}
-
-	let addressed = object.addressed();
-
-	if let Some(reply) = object.in_reply_to().id() {
-		if depth <= 16 {
-			fetch_object_inner(ctx, &reply, depth + 1).await?;
-		} else {
-			tracing::warn!("thread deeper than 16, giving up fetching more replies");
-		}
-	}
-
-	let object_model = ctx.insert_object(object, None).await?;
-
-	let expanded_addresses = ctx.expand_addressing(addressed).await?;
-	ctx.address_to(None, Some(&object_model.ap_id), &expanded_addresses).await?;
-
-	Ok(object_model)
-}
-
-#[axum::async_trait]
-pub trait Fetchable : Sync + Send {
-	async fn fetch(&mut self, ctx: &crate::server::Context) -> crate::Result<&mut Self>;
-}
-
-#[axum::async_trait]
-impl Fetchable for apb::Node<serde_json::Value> {
-	async fn fetch(&mut self, ctx: &crate::server::Context) -> crate::Result<&mut Self> {
-		if let apb::Node::Link(uri) = self {
-			let from = format!("{}{}", ctx.protocol(), ctx.domain()); // TODO helper to avoid this?
-			let pkey = &ctx.app().private_key.ok_or_else(UpubError::internal_server_error)?;
-			*self = Context::request(Method::GET, uri.href(), None, &from, pkey, ctx.domain())
-				.await?
-				.json::<serde_json::Value>()
-				.await?
-				.into();
-		}
-
-		Ok(self)
-	}
-}
-
-
-
-// #[async_recursion::async_recursion]
-// async fn crawl_replies(ctx: &Context, id: &str, depth: usize) -> crate::Result<()> {
-// 	tracing::info!("crawling replies of '{id}'");
-// 	let object = Context::request(
-// 		Method::GET, id, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 	).await?.json::<serde_json::Value>().await?;
-// 
-// 	let object_model = model::object::Model::new(&object)?;
-// 	match model::object::Entity::insert(object_model.into_active_model())
-// 		.exec(ctx.db()).await
-// 	{
-// 		Ok(_) => {},
-// 		Err(sea_orm::DbErr::RecordNotInserted) => {},
-// 		Err(sea_orm::DbErr::Exec(_)) => {}, // ughhh bad fix for sqlite
-// 		Err(e) => return Err(e.into()),
-// 	}
-// 
-// 	if depth > 16 {
-// 		tracing::warn!("stopping thread crawling: too deep!");
-// 		return Ok(());
-// 	}
-// 
-// 	let mut page_url = match object.replies().get() {
-// 		Some(serde_json::Value::String(x)) => {
-// 			let replies = Context::request(
-// 				Method::GET, x, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 			).await?.json::<serde_json::Value>().await?;
-// 			replies.first().id()
-// 		},
-// 		Some(serde_json::Value::Object(x)) => {
-// 			let obj = serde_json::Value::Object(x.clone()); // lol putting it back, TODO!
-// 			obj.first().id()
-// 		},
-// 		_ => return Ok(()),
-// 	};
-// 
-// 	while let Some(ref url) = page_url {
-// 		let replies = Context::request(
-// 			Method::GET, url, None, &format!("https://{}", ctx.domain()), &ctx.app().private_key, ctx.domain(),
-// 		).await?.json::<serde_json::Value>().await?;
-// 
-// 		for reply in replies.items() {
-// 			// TODO right now it crawls one by one, could be made in parallel but would be quite more
-// 			// abusive, so i'll keep it like this while i try it out
-// 			crawl_replies(ctx, reply.href(), depth + 1).await?;
-// 		}
-// 
-// 		page_url = replies.next().id();
-// 	}
-// 
-// 	Ok(())
-// }

src/server/inbox.rs

@@ -1,346 +0,0 @@
-use apb::{target::Addressed, Activity, Base, Object};
-use reqwest::StatusCode;
-use sea_orm::{sea_query::Expr, ActiveModelTrait, ColumnTrait, Condition, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns, Set};
-
-use crate::{errors::{LoggableError, UpubError}, model::{self, FieldError}, server::normalizer::Normalizer};
-
-use super::{fetcher::Fetcher, Context};
-
-
-#[axum::async_trait]
-impl apb::server::Inbox for Context {
-	type Error = UpubError;
-	type Activity = serde_json::Value;
-
-	async fn create(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let activity_model = model::activity::Model::new(&activity)?;
-		let aid = activity_model.id.clone();
-		let Some(object_node) = activity.object().extract() else {
-			// TODO we could process non-embedded activities or arrays but im lazy rn
-			tracing::error!("refusing to process activity without embedded object: {}", serde_json::to_string_pretty(&activity).unwrap());
-			return Err(UpubError::unprocessable());
-		};
-		let object_model = self.insert_object(object_node, Some(server)).await?;
-		let expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		self.address_to(Some(&aid), Some(&object_model.id), &expanded_addressing).await?;
-		tracing::info!("{} posted {}", aid, object_model.id);
-		Ok(())
-	}
-
-	async fn like(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		let aid = activity.id().ok_or(UpubError::bad_request())?;
-		let uid = activity.actor().id().ok_or(UpubError::bad_request())?;
-		let object_uri = activity.object().id().ok_or(UpubError::bad_request())?;
-		let obj = self.fetch_object(&object_uri).await?;
-		let oid = obj.id;
-		let like = model::like::ActiveModel {
-			id: sea_orm::ActiveValue::NotSet,
-			actor: sea_orm::Set(uid.clone()),
-			likes: sea_orm::Set(oid.clone()),
-			date: sea_orm::Set(activity.published().unwrap_or(chrono::Utc::now())),
-		};
-		match model::like::Entity::insert(like).exec(self.db()).await {
-			Err(sea_orm::DbErr::RecordNotInserted) => Err(UpubError::not_modified()),
-			Err(sea_orm::DbErr::Exec(_)) => Err(UpubError::not_modified()), // bad fix for sqlite
-			Err(e) => {
-				tracing::error!("unexpected error procesing like from {uid} to {oid}: {e}");
-				Err(UpubError::internal_server_error())
-			}
-			Ok(_) => {
-				let activity_model = model::activity::Model::new(&activity)?.into_active_model();
-				model::activity::Entity::insert(activity_model)
-					.exec(self.db())
-					.await?;
-				let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-				if expanded_addressing.is_empty() { // WHY MASTODON!!!!!!!
-					expanded_addressing.push(
-						model::object::Entity::find_by_id(&oid)
-							.select_only()
-							.select_column(model::object::Column::AttributedTo)
-							.into_tuple::<String>()
-							.one(self.db())
-							.await?
-							.ok_or_else(UpubError::not_found)?
-						);
-				}
-				self.address_to(Some(aid), None, &expanded_addressing).await?;
-				model::object::Entity::update_many()
-					.col_expr(model::object::Column::Likes, Expr::col(model::object::Column::Likes).add(1))
-					.filter(model::object::Column::Id.eq(oid.clone()))
-					.exec(self.db())
-					.await?;
-				tracing::info!("{} liked {}", uid, oid);
-				Ok(())
-			},
-		}
-	}
-
-	async fn follow(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		let activity_model = model::activity::Model::new(&activity)?;
-		let aid = activity_model.id.clone();
-		let target_user_uri = activity_model.object
-			.as_deref()
-			.ok_or_else(UpubError::bad_request)?
-			.to_string();
-		let usr = self.fetch_user(&target_user_uri).await?;
-		let target_user_id = usr.id;
-		tracing::info!("{} wants to follow {}", activity_model.actor, target_user_id);
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&target_user_id) {
-			expanded_addressing.push(target_user_id);
-		}
-		self.address_to(Some(&aid), None, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn accept(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		// TODO what about TentativeAccept
-		let activity_model = model::activity::Model::new(&activity)?;
-
-		if let Some(mut r) = model::relay::Entity::find_by_id(&activity_model.actor)
-			.one(self.db())
-			.await?
-		{
-			r.accepted = true;
-			model::relay::Entity::update(r.into_active_model()).exec(self.db()).await?;
-			model::activity::Entity::insert(activity_model.clone().into_active_model())
-				.exec(self.db())
-				.await?;
-			tracing::info!("relay {} is now broadcasting to us", activity_model.actor);
-			return Ok(());
-		}
-
-		let Some(follow_request_id) = &activity_model.object else {
-			return Err(UpubError::bad_request());
-		};
-		let Some(follow_activity) = model::activity::Entity::find_by_id(follow_request_id)
-			.one(self.db()).await?
-		else {
-			return Err(UpubError::not_found());
-		};
-		if follow_activity.object.unwrap_or("".into()) != activity_model.actor {
-			return Err(UpubError::forbidden());
-		}
-
-		tracing::info!("{} accepted follow request by {}", activity_model.actor, follow_activity.actor);
-
-		model::activity::Entity::insert(activity_model.clone().into_active_model())
-			.exec(self.db())
-			.await?;
-		model::user::Entity::update_many()
-			.col_expr(
-				model::user::Column::FollowingCount,
-				Expr::col(model::user::Column::FollowingCount).add(1)
-			)
-			.filter(model::user::Column::Id.eq(&follow_activity.actor))
-			.exec(self.db())
-			.await?;
-		model::relation::Entity::insert(
-			model::relation::ActiveModel {
-				follower: Set(follow_activity.actor.clone()),
-				following: Set(activity_model.actor),
-				..Default::default()
-			}
-		).exec(self.db()).await?;
-
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&follow_activity.actor) {
-			expanded_addressing.push(follow_activity.actor);
-		}
-		self.address_to(Some(&activity_model.id), None, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn reject(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		// TODO what about TentativeReject?
-		let activity_model = model::activity::Model::new(&activity)?;
-		let Some(follow_request_id) = &activity_model.object else {
-			return Err(UpubError::bad_request());
-		};
-		let Some(follow_activity) = model::activity::Entity::find_by_id(follow_request_id)
-			.one(self.db()).await?
-		else {
-			return Err(UpubError::not_found());
-		};
-		if follow_activity.object.unwrap_or("".into()) != activity_model.actor {
-			return Err(UpubError::forbidden());
-		}
-
-		tracing::info!("{} rejected follow request by {}", activity_model.actor, follow_activity.actor);
-
-		model::activity::Entity::insert(activity_model.clone().into_active_model())
-			.exec(self.db())
-			.await?;
-
-		let mut expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		if !expanded_addressing.contains(&follow_activity.actor) {
-			expanded_addressing.push(follow_activity.actor);
-		}
-		self.address_to(Some(&activity_model.id), None, &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn delete(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		// TODO verify the signature before just deleting lmao
-		let oid = activity.object().id().ok_or(UpubError::bad_request())?;
-		tracing::debug!("deleting '{oid}'"); // this is so spammy wtf!
-		// TODO maybe we should keep the tombstone?
-		model::user::Entity::delete_by_id(&oid).exec(self.db()).await.info_failed("failed deleting from users");
-		model::activity::Entity::delete_by_id(&oid).exec(self.db()).await.info_failed("failed deleting from activities");
-		model::object::Entity::delete_by_id(&oid).exec(self.db()).await.info_failed("failed deleting from objects");
-		Ok(())
-	}
-
-	async fn update(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let activity_model = model::activity::Model::new(&activity)?;
-		let aid = activity_model.id.clone();
-		let Some(object_node) = activity.object().extract() else {
-			// TODO we could process non-embedded activities or arrays but im lazy rn
-			tracing::error!("refusing to process activity without embedded object: {}", serde_json::to_string_pretty(&activity).unwrap());
-			return Err(UpubError::unprocessable());
-		};
-		let Some(oid) = object_node.id().map(|x| x.to_string()) else {
-			return Err(UpubError::bad_request());
-		};
-		// make sure we're allowed to edit this object
-		if let Some(object_author) = object_node.attributed_to().id() {
-			if server != Context::server(&object_author) {
-				return Err(UpubError::forbidden());
-			}
-		} else if server != Context::server(&oid) {
-			return Err(UpubError::forbidden());
-		};
-		match object_node.object_type() {
-			Some(apb::ObjectType::Actor(_)) => {
-				// TODO oof here is an example of the weakness of this model, we have to go all the way
-				// back up to serde_json::Value because impl Object != impl Actor
-				let actor_model = model::user::Model::new(&object_node)?;
-				let mut update_model = actor_model.into_active_model();
-				update_model.updated = sea_orm::Set(chrono::Utc::now());
-				update_model.reset(model::user::Column::Name);
-				update_model.reset(model::user::Column::Summary);
-				update_model.reset(model::user::Column::Image);
-				update_model.reset(model::user::Column::Icon);
-				model::user::Entity::update(update_model)
-					.exec(self.db()).await?;
-			},
-			Some(apb::ObjectType::Note) => {
-				let object_model = model::object::Model::new(&object_node)?;
-				let mut  update_model = object_model.into_active_model();
-				update_model.updated = sea_orm::Set(Some(chrono::Utc::now()));
-				update_model.reset(model::object::Column::Name);
-				update_model.reset(model::object::Column::Summary);
-				update_model.reset(model::object::Column::Content);
-				update_model.reset(model::object::Column::Sensitive);
-				model::object::Entity::update(update_model)
-					.exec(self.db()).await?;
-			},
-			Some(t) => tracing::warn!("no side effects implemented for update type {t:?}"),
-			None => tracing::warn!("empty type on embedded updated object"),
-		}
-
-		tracing::info!("{} updated {}", aid, oid);
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db())
-			.await?;
-		let expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		self.address_to(Some(&aid), Some(&oid), &expanded_addressing).await?;
-		Ok(())
-	}
-
-	async fn undo(&self, server: String, activity: serde_json::Value) -> crate::Result<()> {
-		let uid = activity.actor().id().ok_or_else(UpubError::bad_request)?;
-		// TODO in theory we could work with just object_id but right now only accept embedded
-		let undone_activity = activity.object().extract().ok_or_else(UpubError::bad_request)?;
-		let undone_aid = undone_activity.id().ok_or_else(UpubError::bad_request)?;
-		let undone_object_uri = undone_activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let activity_type = undone_activity.activity_type().ok_or_else(UpubError::bad_request)?;
-		let undone_activity_author = undone_activity.actor().id().ok_or_else(UpubError::bad_request)?;
-
-		// can't undo activities from remote actors!
-		if server != Context::server(&undone_activity_author) {
-			return Err(UpubError::forbidden());
-		};
-
-		let obj = self.fetch_object(&undone_object_uri).await?;
-		let undone_object_id = obj.id;
-
-		match activity_type {
-			apb::ActivityType::Like => {
-				model::like::Entity::delete_many()
-					.filter(
-						Condition::all()
-							.add(model::like::Column::Actor.eq(&uid))
-							.add(model::like::Column::Likes.eq(&undone_object_id))
-					)
-					.exec(self.db())
-					.await?;
-				model::object::Entity::update_many()
-					.filter(model::object::Column::Id.eq(&undone_object_id))
-					.col_expr(model::object::Column::Likes, Expr::col(model::object::Column::Likes).sub(1))
-					.exec(self.db())
-					.await?;
-			},
-			apb::ActivityType::Follow => {
-				model::relation::Entity::delete_many()
-					.filter(
-						Condition::all()
-							.add(model::relation::Column::Follower.eq(&uid))
-							.add(model::relation::Column::Following.eq(&undone_object_id))
-					)
-					.exec(self.db())
-					.await?;
-			},
-			_ => {
-				tracing::error!("received 'Undo' for unimplemented activity: {}", serde_json::to_string_pretty(&activity).unwrap());
-				return Err(StatusCode::NOT_IMPLEMENTED.into());
-			},
-		}
-
-		model::activity::Entity::delete_by_id(undone_aid).exec(self.db()).await?;
-
-		Ok(())
-
-	}
-	
-	async fn announce(&self, _: String, activity: serde_json::Value) -> crate::Result<()> {
-		let activity_model = model::activity::Model::new(&activity)?;
-		let Some(object_uri) = &activity_model.object else {
-			return Err(FieldError("object").into());
-		};
-		let obj = self.fetch_object(object_uri).await?;
-		let oid = obj.id;
-
-		// relays send us activities as Announce, but we don't really want to count those towards the
-		// total shares count of an object, so just fetch the object and be done with it
-		if self.is_relay(&activity_model.actor) {
-			tracing::info!("relay {} broadcasted {}", activity_model.actor, oid);
-			return Ok(())
-		}
-
-		let share = model::share::ActiveModel {
-			id: sea_orm::ActiveValue::NotSet,
-			actor: sea_orm::Set(activity_model.actor.clone()),
-			shares: sea_orm::Set(oid.clone()),
-			date: sea_orm::Set(activity.published().unwrap_or(chrono::Utc::now())),
-		};
-
-		let expanded_addressing = self.expand_addressing(activity.addressed()).await?;
-		self.address_to(Some(&activity_model.id), None, &expanded_addressing).await?;
-		model::share::Entity::insert(share)
-			.exec(self.db()).await?;
-		model::activity::Entity::insert(activity_model.clone().into_active_model())
-			.exec(self.db())
-			.await?;
-		model::object::Entity::update_many()
-			.col_expr(model::object::Column::Shares, Expr::col(model::object::Column::Shares).add(1))
-			.filter(model::object::Column::Id.eq(oid.clone()))
-			.exec(self.db())
-			.await?;
-
-		tracing::info!("{} shared {}", activity_model.actor, oid);
-		Ok(())
-	}
-}

src/server/mod.rs

@@ -1,12 +0,0 @@
-pub mod admin;
-pub mod context;
-pub mod dispatcher;
-pub mod fetcher;
-pub mod inbox;
-pub mod outbox;
-pub mod auth;
-pub mod builders;
-pub mod httpsign;
-pub mod normalizer;
-
-pub use context::Context;

src/server/normalizer.rs

@@ -1,131 +0,0 @@
-use apb::{Node, Base, Object, Document};
-use sea_orm::{sea_query::Expr, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, Set};
-use crate::{errors::UpubError, model, server::Context};
-
-use super::fetcher::Fetcher;
-
-#[axum::async_trait]
-pub trait Normalizer {
-	async fn insert_object(&self, obj: impl apb::Object, server: Option<String>) -> crate::Result<model::object::Model>;
-}
-
-#[axum::async_trait]
-impl Normalizer for super::Context {
-	async fn insert_object(&self, object_node: impl apb::Object, server: Option<String>) -> crate::Result<model::object::Model> {
-		let mut object_model = model::object::Model::new(&object_node)?;
-		let oid = object_model.id.clone();
-		let uid = object_model.attributed_to.clone();
-		if let Some(server) = server {
-			// make sure we're allowed to create this object
-			if let Some(object_author) = &object_model.attributed_to {
-				if server != Context::server(object_author) {
-					return Err(UpubError::forbidden());
-				}
-			} else if server != Context::server(&object_model.id) {
-				return Err(UpubError::forbidden());
-			};
-		}
-
-		// make sure content only contains a safe subset of html
-		if let Some(content) = object_model.content {
-			object_model.content = Some(mdhtml::safe_html(&content));
-		}
-
-		// fix context for remote posts
-		// > note that this will effectively recursively try to fetch the parent object, in order to find
-		// > the context (which is id of topmost object). there's a recursion limit of 16 hidden inside
-		// > btw! also if any link is broken or we get rate limited, the whole insertion fails which is
-		// > kind of dumb. there should be a job system so this can be done in waves. or maybe there's
-		// > some whole other way to do this?? im thinking but misskey aaaa!! TODO
-		if let Some(ref reply) = object_model.in_reply_to {
-			if let Some(o) = model::object::Entity::find_by_id(reply).one(self.db()).await? {
-				object_model.context = o.context;
-			} else {
-				object_model.context = None; // TODO to be filled by some other task
-			}
-		} else {
-			object_model.context = Some(object_model.id.clone());
-		}
-
-		model::object::Entity::insert(object_model.clone().into_active_model()).exec(self.db()).await?;
-
-		// update replies counter
-		if let Some(ref in_reply_to) = object_model.in_reply_to {
-			if self.fetch_object(in_reply_to).await.is_ok() {
-				model::object::Entity::update_many()
-					.filter(model::object::Column::Id.eq(in_reply_to))
-					.col_expr(model::object::Column::Comments, Expr::col(model::object::Column::Comments).add(1))
-					.exec(self.db())
-					.await?;
-			}
-		}
-		// update statuses counter
-		if let Some(object_author) = uid {
-			model::user::Entity::update_many()
-				.col_expr(model::user::Column::StatusesCount, Expr::col(model::user::Column::StatusesCount).add(1))
-				.filter(model::user::Column::Id.eq(&object_author))
-				.exec(self.db())
-				.await?;
-		}
-
-		for attachment in object_node.attachment().flat() {
-			let attachment_model = match attachment {
-				Node::Empty => continue,
-				Node::Array(_) => {
-					tracing::warn!("ignoring array-in-array while processing attachments");
-					continue
-				},
-				Node::Link(l) => model::attachment::ActiveModel {
-					id: sea_orm::ActiveValue::NotSet,
-					url: Set(l.href().to_string()),
-					object: Set(oid.clone()),
-					document_type: Set(apb::DocumentType::Page),
-					name: Set(l.link_name().map(|x| x.to_string())),
-					media_type: Set(l.link_media_type().unwrap_or("link").to_string()),
-					created: Set(chrono::Utc::now()),
-				},
-				Node::Object(o) => model::attachment::ActiveModel {
-					id: sea_orm::ActiveValue::NotSet,
-					url: Set(o.url().id().unwrap_or_else(|| o.id().map(|x| x.to_string()).unwrap_or_default())),
-					object: Set(oid.clone()),
-					document_type: Set(o.as_document().map_or(apb::DocumentType::Document, |x| x.document_type().unwrap_or(apb::DocumentType::Page))),
-					name: Set(o.name().map(|x| x.to_string())),
-					media_type: Set(o.media_type().unwrap_or("link").to_string()),
-					created: Set(o.published().unwrap_or_else(chrono::Utc::now)),
-				},
-			};
-			model::attachment::Entity::insert(attachment_model)
-				.exec(self.db())
-				.await?;
-		}
-		// lemmy sends us an image field in posts, treat it like an attachment i'd say
-		if let Some(img) = object_node.image().get() {
-			// TODO lemmy doesnt tell us the media type but we use it to display the thing...
-			let img_url = img.url().id().unwrap_or_default();
-			let media_type = if img_url.ends_with("png") {
-				Some("image/png".to_string())
-			} else if img_url.ends_with("webp") {
-				Some("image/webp".to_string())
-			} else if img_url.ends_with("jpeg") || img_url.ends_with("jpg") {
-				Some("image/jpeg".to_string())
-			} else {
-				None
-			};
-
-			let attachment_model = model::attachment::ActiveModel {
-				id: sea_orm::ActiveValue::NotSet,
-				url: Set(img.url().id().unwrap_or_else(|| img.id().map(|x| x.to_string()).unwrap_or_default())),
-				object: Set(oid.clone()),
-				document_type: Set(img.as_document().map_or(apb::DocumentType::Document, |x| x.document_type().unwrap_or(apb::DocumentType::Page))),
-				name: Set(img.name().map(|x| x.to_string())),
-				media_type: Set(img.media_type().unwrap_or(media_type.as_deref().unwrap_or("link")).to_string()),
-				created: Set(img.published().unwrap_or_else(chrono::Utc::now)),
-			};
-			model::attachment::Entity::insert(attachment_model)
-				.exec(self.db())
-				.await?;
-		}
-
-		Ok(object_model)
-	}
-}

src/server/outbox.rs

@@ -1,420 +0,0 @@
-use apb::{target::Addressed, Activity, ActivityMut, ActorMut, BaseMut, Node, Object, ObjectMut, PublicKeyMut};
-use reqwest::StatusCode;
-use sea_orm::{sea_query::Expr, ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns, Set};
-
-use crate::{errors::UpubError, model, routes::activitypub::jsonld::LD};
-
-use super::{fetcher::Fetcher, normalizer::Normalizer, Context};
-
-
-#[axum::async_trait]
-impl apb::server::Outbox for Context {
-	type Error = UpubError;
-	type Object = serde_json::Value;
-	type Activity = serde_json::Value;
-
-	async fn create_note(&self, uid: String, object: serde_json::Value) -> crate::Result<String> {
-		// TODO regex hell, here i come...
-		let re = regex::Regex::new(r"@(.+)@([^ ]+)").expect("failed compiling regex pattern");
-		let raw_oid = uuid::Uuid::new_v4().to_string();
-		let oid = self.oid(&raw_oid);
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = object.addressed();
-
-		let mut content = object.content().map(|x| x.to_string());
-		if let Some(c) = content {
-			let mut tmp = mdhtml::safe_markdown(&c);
-			for (full, [user, domain]) in re.captures_iter(&tmp.clone()).map(|x| x.extract()) {
-				if let Ok(Some(uid)) = model::user::Entity::find()
-					.filter(model::user::Column::PreferredUsername.eq(user))
-					.filter(model::user::Column::Domain.eq(domain))
-					.select_only()
-					.select_column(model::user::Column::Id)
-					.into_tuple::<String>()
-					.one(self.db())
-					.await
-				{
-					tmp = tmp.replacen(full, &format!("<a href=\"{uid}\" class=\"u-url mention\">@{user}</a>"), 1);
-				}
-			}
-			content = Some(tmp);
-		}
-
-		let object_model = self.insert_object(
-			object
-				.set_id(Some(&oid))
-				.set_attributed_to(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-				.set_content(content.as_deref())
-				.set_url(Node::maybe_link(self.cfg().instance.frontend.as_ref().map(|x| format!("{x}/objects/{raw_oid}")))),
-			Some(self.domain().to_string()),
-		).await?;
-
-		let activity_model = model::activity::Model {
-			id: aid.clone(),
-			activity_type: apb::ActivityType::Create,
-			actor: uid.clone(),
-			object: Some(oid.clone()),
-			target: None,
-			cc: object_model.cc.clone(),
-			bcc: object_model.bcc.clone(),
-			to: object_model.to.clone(),
-			bto: object_model.bto.clone(),
-			published: object_model.published,
-		};
-
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, Some(&oid)).await?;
-
-		Ok(aid)
-	}
-
-	async fn create(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let Some(object) = activity.object().extract() else {
-			return Err(UpubError::bad_request());
-		};
-
-		let raw_oid = uuid::Uuid::new_v4().to_string();
-		let oid = self.oid(&raw_oid);
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-
-		self.insert_object(
-			object
-				.set_id(Some(&oid))
-				.set_attributed_to(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-				.set_to(activity.to())
-				.set_bto(activity.bto())
-				.set_cc(activity.cc())
-				.set_bcc(activity.bcc()),
-			Some(self.domain().to_string()),
-		).await?;
-
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-				.set_object(Node::link(oid.clone()))
-		)?;
-
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, Some(&oid)).await?;
-		Ok(aid)
-	}
-
-	async fn like(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		self.fetch_object(&oid).await?;
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_published(Some(chrono::Utc::now()))
-				.set_actor(Node::link(uid.clone()))
-		)?;
-
-		let like_model = model::like::ActiveModel {
-			actor: Set(uid.clone()),
-			likes: Set(oid.clone()),
-			date: Set(chrono::Utc::now()),
-			..Default::default()
-		};
-		model::like::Entity::insert(like_model).exec(self.db()).await?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-		model::object::Entity::update_many()
-			.col_expr(model::object::Column::Likes, Expr::col(model::object::Column::Likes).add(1))
-			.filter(model::object::Column::Id.eq(oid))
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn follow(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		if activity.object().id().is_none() {
-			return Err(UpubError::bad_request());
-		}
-
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn accept(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		if activity.object().id().is_none() {
-			return Err(UpubError::bad_request());
-		}
-		let Some(accepted_id) = activity.object().id() else {
-			return Err(UpubError::bad_request());
-		};
-		let Some(accepted_activity) = model::activity::Entity::find_by_id(accepted_id)
-			.one(self.db()).await?
-		else {
-			return Err(UpubError::not_found());
-		};
-
-		match accepted_activity.activity_type {
-			apb::ActivityType::Follow => {
-				model::user::Entity::update_many()
-					.col_expr(
-						model::user::Column::FollowersCount,
-						Expr::col(model::user::Column::FollowersCount).add(1)
-					)
-					.filter(model::user::Column::Id.eq(&uid))
-					.exec(self.db())
-					.await?;
-				model::relation::Entity::insert(
-					model::relation::ActiveModel {
-						follower: Set(accepted_activity.actor), following: Set(uid.clone()),
-						..Default::default()
-					}
-				).exec(self.db()).await?;
-			},
-			t => tracing::warn!("no side effects implemented for accepting {t:?}"),
-		}
-
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn reject(&self, _uid: String, _activity: serde_json::Value) -> crate::Result<String> {
-		todo!()
-	}
-
-	async fn undo(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let old_aid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		let old_activity = model::activity::Entity::find_by_id(old_aid)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-		if old_activity.actor != uid {
-			return Err(UpubError::forbidden());
-		}
-		match old_activity.activity_type {
-			apb::ActivityType::Like => {
-				model::like::Entity::delete_many()
-					.filter(model::like::Column::Actor.eq(old_activity.actor))
-					.filter(model::like::Column::Likes.eq(old_activity.object.unwrap_or("".into())))
-					.exec(self.db())
-					.await?;
-			},
-			apb::ActivityType::Follow => {
-				model::relation::Entity::delete_many()
-					.filter(model::relation::Column::Follower.eq(old_activity.actor))
-					.filter(model::relation::Column::Following.eq(old_activity.object.unwrap_or("".into())))
-					.exec(self.db())
-					.await?;
-			},
-			t => tracing::warn!("extra side effects for activity {t:?} not implemented"),
-		}
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn delete(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-
-		let object = model::object::Entity::find_by_id(&oid)
-			.one(self.db())
-			.await?
-			.ok_or_else(UpubError::not_found)?;
-
-		let Some(author_id) = object.attributed_to else {
-			// can't change local objects attributed to nobody
-			return Err(UpubError::forbidden())
-		};
-
-		if author_id != uid {
-			// can't change objects of others
-			return Err(UpubError::forbidden());
-		}
-
-		let addressed = activity.addressed();
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		model::object::Entity::delete_by_id(&oid)
-			.exec(self.db())
-			.await?;
-
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, addressed, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn update(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let object_node = activity.object().extract().ok_or_else(UpubError::bad_request)?;
-
-		match object_node.object_type() {
-			Some(apb::ObjectType::Actor(_)) => {
-				let mut actor_model = model::user::Model::new(
-					&object_node
-						// TODO must set these, but we will ignore them
-						.set_actor_type(Some(apb::ActorType::Person))
-						.set_public_key(apb::Node::object(
-							serde_json::Value::new_object().set_public_key_pem("")
-						))
-				)?;
-				let old_actor_model = model::user::Entity::find_by_id(&actor_model.id)
-					.one(self.db())
-					.await?
-					.ok_or_else(UpubError::not_found)?;
-
-				if old_actor_model.id != uid {
-					// can't change user fields of others
-					return Err(UpubError::forbidden());
-				}
-
-				if actor_model.name.is_none() { actor_model.name = old_actor_model.name }
-				if actor_model.summary.is_none() { actor_model.summary = old_actor_model.summary }
-				if actor_model.image.is_none() { actor_model.image = old_actor_model.image }
-				if actor_model.icon.is_none() { actor_model.icon = old_actor_model.icon }
-
-				let mut update_model = actor_model.into_active_model();
-				update_model.updated = sea_orm::Set(chrono::Utc::now());
-				update_model.reset(model::user::Column::Name);
-				update_model.reset(model::user::Column::Summary);
-				update_model.reset(model::user::Column::Image);
-				update_model.reset(model::user::Column::Icon);
-
-				model::user::Entity::update(update_model)
-					.exec(self.db()).await?;
-			},
-			Some(apb::ObjectType::Note) => {
-				let mut object_model = model::object::Model::new(
-					&object_node.set_published(Some(chrono::Utc::now()))
-				)?;
-
-				let old_object_model = model::object::Entity::find_by_id(&object_model.id)
-					.one(self.db())
-					.await?
-					.ok_or_else(UpubError::not_found)?;
-
-				// can't change local objects attributed to nobody
-				let author_id = old_object_model.attributed_to.ok_or_else(UpubError::forbidden)?;
-				if author_id != uid {
-					// can't change objects of others
-					return Err(UpubError::forbidden());
-				}
-
-				if object_model.name.is_none() { object_model.name = old_object_model.name }
-				if object_model.summary.is_none() { object_model.summary = old_object_model.summary }
-				if object_model.content.is_none() { object_model.content = old_object_model.content }
-
-				let mut update_model = object_model.into_active_model();
-				update_model.updated = sea_orm::Set(Some(chrono::Utc::now()));
-				update_model.reset(model::object::Column::Name);
-				update_model.reset(model::object::Column::Summary);
-				update_model.reset(model::object::Column::Content);
-				update_model.reset(model::object::Column::Sensitive);
-
-				model::object::Entity::update(update_model)
-					.exec(self.db()).await?;
-			},
-			_ => return Err(UpubError::Status(StatusCode::NOT_IMPLEMENTED)),
-		}
-
-		let addressed = activity.addressed();
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_actor(Node::link(uid.clone()))
-				.set_published(Some(chrono::Utc::now()))
-		)?;
-
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-
-		self.dispatch(&uid, addressed, &aid, None).await?;
-
-		Ok(aid)
-	}
-
-	async fn announce(&self, uid: String, activity: serde_json::Value) -> crate::Result<String> {
-		let aid = self.aid(&uuid::Uuid::new_v4().to_string());
-		let activity_targets = activity.addressed();
-		let oid = activity.object().id().ok_or_else(UpubError::bad_request)?;
-		self.fetch_object(&oid).await?;
-		let activity_model = model::activity::Model::new(
-			&activity
-				.set_id(Some(&aid))
-				.set_published(Some(chrono::Utc::now()))
-				.set_actor(Node::link(uid.clone()))
-		)?;
-
-		let share_model = model::share::ActiveModel {
-			actor: Set(uid.clone()),
-			shares: Set(oid.clone()),
-			date: Set(chrono::Utc::now()),
-			..Default::default()
-		};
-		model::share::Entity::insert(share_model).exec(self.db()).await?;
-		model::activity::Entity::insert(activity_model.into_active_model())
-			.exec(self.db()).await?;
-		model::object::Entity::update_many()
-			.col_expr(model::object::Column::Shares, Expr::col(model::object::Column::Shares).add(1))
-			.filter(model::object::Column::Id.eq(oid))
-			.exec(self.db())
-			.await?;
-
-		self.dispatch(&uid, activity_targets, &aid, None).await?;
-
-		Ok(aid)
-	}
-}

upub/cli/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "upub-cli"
+version = "0.2.0"
+edition = "2021"
+authors = [ "alemi <me@alemi.dev>" ]
+description = "cli maintenance tasks for upub"
+license = "AGPL-3.0"
+repository = "https://git.alemi.dev/upub.git"
+readme = "README.md"
+
+[lib]
+
+[dependencies]
+apb = { path = "../../apb/" }
+upub = { path = "../core" }
+tracing = "0.1"
+serde_json = "1"
+sha256 = "1.5"
+uuid = { version = "1.10", features = ["v4"] }
+chrono = { version = "0.4", features = ["serde"] }
+openssl = "0.10" # TODO handle pubkeys with a smaller crate
+clap = { version = "4.5", features = ["derive"] }
+sea-orm = "1.0"
+futures = "0.3"
+mdhtml = { path = "../../utils/mdhtml/" }
+reqwest = { version = "0.12", features = ["json"] }

upub/cli/README.md

@@ -0,0 +1 @@
+# upub cli

upub/cli/src/cloak.rs

@@ -0,0 +1,115 @@
+use futures::TryStreamExt;
+use sea_orm::{ActiveModelTrait, ActiveValue::{NotSet, Set, Unchanged}, ColumnTrait, Condition, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns};
+use upub::traits::{fetch::RequestError, Cloaker};
+
+pub async fn cloak(ctx: upub::Context, post_contents: bool, objects: bool, actors: bool) -> Result<(), RequestError> {
+	let local_base = format!("{}%", ctx.base());
+	{
+		let mut stream = upub::model::attachment::Entity::find()
+			.filter(upub::model::attachment::Column::Url.not_like(&local_base))
+			.stream(ctx.db())
+			.await?;
+
+		while let Some(attachment) = stream.try_next().await? {
+			tracing::info!("cloaking {}", attachment.url);
+			let (sig, url) = ctx.cloak(&attachment.url);
+			let mut model = attachment.into_active_model();
+			model.url = Set(upub::url!(ctx, "/proxy/{sig}/{url}"));
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if objects {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Image.is_not_null())
+			.filter(upub::model::object::Column::Image.not_like(&local_base))
+			.select_only()
+			.select_column(upub::model::object::Column::Internal)
+			.select_column(upub::model::object::Column::Image)
+			.into_tuple::<(i64, String)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, image)) = stream.try_next().await? {
+			tracing::info!("cloaking object image {image}");
+			let model = upub::model::object::ActiveModel {
+				internal: Unchanged(internal),
+				image: Set(Some(ctx.cloaked(&image))),
+				..Default::default()
+			};
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if actors {
+		let mut stream = upub::model::actor::Entity::find()
+			.filter(
+				Condition::any()
+					.add(upub::model::actor::Column::Image.not_like(&local_base))
+					.add(upub::model::actor::Column::Icon.not_like(&local_base))
+			)
+			.select_only()
+			.select_column(upub::model::actor::Column::Internal)
+			.select_column(upub::model::actor::Column::Image)
+			.select_column(upub::model::actor::Column::Icon)
+			.into_tuple::<(i64, Option<String>, Option<String>)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, image, icon)) = stream.try_next().await? {
+			tracing::info!("cloaking user #{internal}");
+			if image.is_none() && icon.is_none() { continue }
+			// TODO can this if/else/else be made nicer??
+			let image = if let Some(img) = image {
+				if !img.starts_with(ctx.base()) {
+					Set(Some(ctx.cloaked(&img)))
+				} else {
+					NotSet
+				}
+			} else {
+				NotSet
+			};
+			let icon = if let Some(icn) = icon {
+				if !icn.starts_with(ctx.base()) {
+					Set(Some(ctx.cloaked(&icn)))
+				} else {
+					NotSet
+				}
+			} else {
+				NotSet
+			};
+			let model = upub::model::actor::ActiveModel {
+				internal: Unchanged(internal),
+				image, icon,
+				..Default::default()
+			};
+			model.update(ctx.db()).await?;
+		}
+	}
+
+	if post_contents {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Content.like("%<img%"))
+			.select_only()
+			.select_column(upub::model::object::Column::Internal)
+			.select_column(upub::model::object::Column::Content)
+			.into_tuple::<(i64, String)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, content)) = stream.try_next().await? {
+			let sanitized = ctx.sanitize(&content);
+			if sanitized != content {
+				tracing::info!("sanitizing object #{internal}");
+				let model = upub::model::object::ActiveModel {
+					internal: Unchanged(internal),
+					content: Set(Some(sanitized)),
+					..Default::default()
+				};
+				model.update(ctx.db()).await?;
+			}
+		}
+	}
+
+	Ok(())
+}

upub/cli/src/faker.rs

@@ -1,16 +1,17 @@
-use crate::model::{addressing, config, credential, activity, object, user, Audience};
+use upub::{ext::JsonVec, model::{activity, actor, addressing, config, credential, object}};
 use openssl::rsa::Rsa;
-use sea_orm::IntoActiveModel;
+use sea_orm::{ActiveValue::NotSet, IntoActiveModel};
 
-pub async fn faker(ctx: crate::server::Context, count: u64) -> Result<(), sea_orm::DbErr> {
+pub async fn faker(ctx: upub::Context, count: i64) -> Result<(), sea_orm::DbErr> {
 	use sea_orm::{EntityTrait, Set};
 
 	let domain = ctx.domain();
 	let db = ctx.db();
 
 	let key = Rsa::generate(2048).unwrap();
-	let test_user = user::Model {
-		id: format!("{domain}/users/test"),
+	let test_user = actor::Model {
+		internal: 42,
+		id: format!("{domain}/actors/test"),
 		name: Some("μpub".into()),
 		domain: clean_domain(domain),
 		preferred_username: "test".to_string(),
@@ -19,24 +20,28 @@ pub async fn faker(ctx: crate::server::Context, count: u64) -> Result<(), sea_or
 		following_count: 0,
 		followers: None,
 		followers_count: 0,
-		statuses_count: count as i64,
+		statuses_count: count as i32,
+		fields: JsonVec::default(),
+		also_known_as: JsonVec::default(),
+		moved_to: None,
 		icon: Some("https://cdn.alemi.dev/social/circle-square.png".to_string()),
 		image: Some("https://cdn.alemi.dev/social/someriver-xs.jpg".to_string()),
 		inbox: None,
 		shared_inbox: None,
 		outbox: None,
 		actor_type: apb::ActorType::Person,
-		created: chrono::Utc::now(),
+		published: chrono::Utc::now(),
 		updated: chrono::Utc::now(),
 		private_key: Some(std::str::from_utf8(&key.private_key_to_pem().unwrap()).unwrap().to_string()),
 		// TODO generate a fresh one every time
 		public_key: std::str::from_utf8(&key.public_key_to_pem().unwrap()).unwrap().to_string(),
 	};
 
-	user::Entity::insert(test_user.clone().into_active_model()).exec(db).await?;
+	actor::Entity::insert(test_user.clone().into_active_model()).exec(db).await?;
 
 	config::Entity::insert(config::ActiveModel {
-		id: Set(test_user.id.clone()),
+		internal: NotSet,
+		actor: Set(test_user.id.clone()),
 		accept_follow_requests: Set(true),
 		show_followers: Set(true),
 		show_following: Set(true),
@@ -45,9 +50,11 @@ pub async fn faker(ctx: crate::server::Context, count: u64) -> Result<(), sea_or
 	}).exec(db).await?;
 
 	credential::Entity::insert(credential::ActiveModel {
-		id: Set(test_user.id.clone()),
-		email: Set("mail@example.net".to_string()),
+		internal: NotSet,
+		actor: Set(test_user.id.clone()),
+		login: Set("mail@example.net".to_string()),
 		password: Set(sha256::digest("very-strong-password")),
+		active: Set(true),
 	}).exec(db).await?;
 
 	let context = uuid::Uuid::new_v4().to_string();
@@ -57,47 +64,52 @@ pub async fn faker(ctx: crate::server::Context, count: u64) -> Result<(), sea_or
 		let aid = uuid::Uuid::new_v4();
 
 		addressing::Entity::insert(addressing::ActiveModel {
-			actor: Set(apb::target::PUBLIC.to_string()),
-			server: Set("www.w3.org".to_string()),
-			activity: Set(Some(format!("{domain}/activities/{aid}"))),
-			object: Set(Some(format!("{domain}/objects/{oid}"))),
+			actor: Set(None),
+			instance: Set(None),
+			activity: Set(Some(42 + i)),
+			object: Set(Some(42 + i)),
 			published: Set(chrono::Utc::now()),
 			..Default::default()
 		}).exec(db).await?;
 
 		object::Entity::insert(object::ActiveModel {
+			internal: Set(42 + i),
 			id: Set(format!("{domain}/objects/{oid}")),
 			name: Set(None),
 			object_type: Set(apb::ObjectType::Note),
-			attributed_to: Set(Some(format!("{domain}/users/test"))),
+			attributed_to: Set(Some(format!("{domain}/actors/test"))),
 			summary: Set(None),
 			context: Set(Some(context.clone())),
 			in_reply_to: Set(None),
+			quote: Set(None),
 			content: Set(Some(format!("[{i}] Tic(k). Quasiparticle of intensive multiplicity. Tics (or ticks) are intrinsically several components of autonomously numbering anorganic populations, propagating by contagion between segmentary divisions in the order of nature. Ticks - as nonqualitative differentially-decomposable counting marks - each designate a multitude comprehended as a singular variation in tic(k)-density."))),
-			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i)),
-			updated: Set(None),
-			comments: Set(0),
+			image: Set(None),
+			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i as u64)),
+			updated: Set(chrono::Utc::now()),
+			replies: Set(0),
 			likes: Set(0),
-			shares: Set(0),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
-			bto: Set(Audience::default()),
-			cc: Set(Audience(vec![])),
-			bcc: Set(Audience::default()),
+			announces: Set(0),
+			audience: Set(None),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
+			bto: Set(JsonVec::default()),
+			cc: Set(JsonVec(vec![])),
+			bcc: Set(JsonVec::default()),
 			url: Set(None),
 			sensitive: Set(false),
 		}).exec(db).await?;
 
 		activity::Entity::insert(activity::ActiveModel {
+			internal: Set(42 + i),
 			id: Set(format!("{domain}/activities/{aid}")),
 			activity_type: Set(apb::ActivityType::Create),
-			actor: Set(format!("{domain}/users/test")),
+			actor: Set(format!("{domain}/actors/test")),
 			object: Set(Some(format!("{domain}/objects/{oid}"))),
 			target: Set(None),
-			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i)),
-			to: Set(Audience(vec![apb::target::PUBLIC.to_string()])),
-			bto: Set(Audience::default()),
-			cc: Set(Audience(vec![])),
-			bcc: Set(Audience::default()),
+			published: Set(chrono::Utc::now() - std::time::Duration::from_secs(60*i as u64)),
+			to: Set(JsonVec(vec![apb::target::PUBLIC.to_string()])),
+			bto: Set(JsonVec::default()),
+			cc: Set(JsonVec(vec![])),
+			bcc: Set(JsonVec::default()),
 		}).exec(db).await?;
 	}
 

upub/cli/src/fetch.rs

@@ -0,0 +1,65 @@
+use sea_orm::{EntityTrait, TransactionTrait};
+use upub::traits::{fetch::RequestError, Addresser, Fetcher, Normalizer};
+
+pub async fn fetch(ctx: upub::Context, uri: String, save: bool, actor: Option<String>) -> Result<(), RequestError> {
+	use apb::Base;
+
+	let mut pkey = ctx.pkey().to_string();
+	let mut from = ctx.base().to_string();
+	
+	if let Some(actor) = actor {
+		let actor_model = upub::model::actor::Entity::find_by_ap_id(&actor)
+			.one(ctx.db())
+			.await?
+			.ok_or_else(|| sea_orm::DbErr::RecordNotFound(actor.clone()))?;
+
+		match actor_model.private_key {
+			None => tracing::error!("requested actor lacks a private key, fetching with server key instead"),
+			Some(x) => {
+				pkey = x;
+				from = actor.to_string();
+			},
+		}
+	}
+
+	let mut node = apb::Node::link(uri.to_string());
+	if let apb::Node::Link(ref uri) = node {
+		if let Ok(href) = uri.href() {
+			node = upub::Context::request(reqwest::Method::GET, href, None, &from, &pkey, ctx.domain())
+				.await?
+				.json::<serde_json::Value>()
+				.await?
+				.into();
+		}
+	}
+
+
+	let obj = node.extract().expect("node still empty after fetch?");
+
+	println!("{}", serde_json::to_string_pretty(&obj).unwrap());
+
+	if save {
+		let tx = ctx.db().begin().await?;
+		match obj.base_type() {
+			Ok(apb::BaseType::Object(apb::ObjectType::Actor(_))) => {
+				upub::model::actor::Entity::insert(upub::AP::actor_q(&obj, None)?)
+					.exec(&tx)
+					.await?;
+			},
+			Ok(apb::BaseType::Object(apb::ObjectType::Activity(_))) => {
+				let act = ctx.insert_activity(obj, &tx).await?;
+				ctx.address(Some(&act), None, &tx).await?;
+			},
+			Ok(apb::BaseType::Object(apb::ObjectType::Note)) => {
+				let obj = ctx.insert_object(obj, &tx).await?;
+				ctx.address(None, Some(&obj), &tx).await?;
+			},
+			Ok(apb::BaseType::Object(t)) => tracing::warn!("not implemented: {:?}", t),
+			Ok(apb::BaseType::Link(_)) => tracing::error!("fetched another link?"),
+			Err(_) => tracing::error!("no type on object"),
+		}
+		tx.commit().await?;
+	}
+
+	Ok(())
+}

upub/cli/src/fix.rs

@@ -1,7 +1,6 @@
-use sea_orm::EntityTrait;
+use sea_orm::{ActiveModelTrait, EntityTrait};
 
-
-pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies: bool) -> crate::Result<()> {
+pub async fn fix(ctx: upub::Context, likes: bool, shares: bool, replies: bool) -> Result<(), sea_orm::DbErr> {
 	use futures::TryStreamExt;
 	let db = ctx.db();
 
@@ -9,22 +8,19 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing likes...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::like::Entity::find().stream(db).await?;
+			let mut stream = upub::model::like::Entity::find().stream(db).await?;
 			while let Some(like) = stream.try_next().await? {
-				store.insert(like.likes.clone(), store.get(&like.likes).unwrap_or(&0) + 1);
+				store.insert(like.object, store.get(&like.object).unwrap_or(&0) + 1);
 			}
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
-				id: sea_orm::Set(k.clone()),
+			let m = upub::model::object::ActiveModel {
+				internal: sea_orm::Unchanged(k),
 				likes: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
-				.exec(db)
-				.await
-			{
+			if let Err(e) = m.update(db).await {
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}
@@ -34,22 +30,19 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing shares...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::share::Entity::find().stream(db).await?;
+			let mut stream = upub::model::announce::Entity::find().stream(db).await?;
 			while let Some(share) = stream.try_next().await? {
-				store.insert(share.shares.clone(), store.get(&share.shares).unwrap_or(&0) + 1);
+				store.insert(share.object, store.get(&share.object).unwrap_or(&0) + 1);
 			}
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
-				id: sea_orm::Set(k.clone()),
-				shares: sea_orm::Set(v),
+			let m = upub::model::object::ActiveModel {
+				internal: sea_orm::Unchanged(k),
+				announces: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
-				.exec(db)
-				.await
-			{
+			if let Err(e) = m.update(db).await {
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}
@@ -59,7 +52,7 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		tracing::info!("fixing replies...");
 		let mut store = std::collections::HashMap::new();
 		{
-			let mut stream = crate::model::object::Entity::find().stream(db).await?;
+			let mut stream = upub::model::object::Entity::find().stream(db).await?;
 			while let Some(object) = stream.try_next().await? {
 				if let Some(reply) = object.in_reply_to {
 					let before = store.get(&reply).unwrap_or(&0);
@@ -69,15 +62,13 @@ pub async fn fix(ctx: crate::server::Context, likes: bool, shares: bool, replies
 		}
 
 		for (k, v) in store {
-			let m = crate::model::object::ActiveModel {
-				id: sea_orm::Set(k.clone()),
-				comments: sea_orm::Set(v),
+			let m = upub::model::object::ActiveModel {
+				id: sea_orm::Unchanged(k.clone()),
+				replies: sea_orm::Set(v),
 				..Default::default()
 			};
-			if let Err(e) = crate::model::object::Entity::update(m)
-				.exec(db)
-				.await
-			{
+			// TODO will update work with non-primary-key field??
+			if let Err(e) = m.update(db).await {
 				tracing::warn!("record not updated ({k}): {e}");
 			}
 		}

upub/cli/src/lib.rs

@@ -0,0 +1,163 @@
+mod fix;
+pub use fix::*;
+
+mod fetch;
+pub use fetch::*;
+
+mod faker;
+pub use faker::*;
+
+mod relay;
+pub use relay::*;
+
+mod register;
+pub use register::*;
+
+mod update;
+pub use update::*;
+
+mod nuke;
+pub use nuke::*;
+
+mod thread;
+pub use thread::*;
+
+mod cloak;
+pub use cloak::*;
+
+#[derive(Debug, Clone, clap::Subcommand)]
+pub enum CliCommand {
+	/// generate fake user, note and activity
+	Faker{
+		/// how many fake statuses to insert for root user
+		count: u64,
+	},
+
+	/// fetch a single AP object
+	Fetch {
+		/// object id, or uri, to fetch
+		uri: String,
+
+		#[arg(long, default_value_t = false)]
+		/// store fetched object in local db
+		save: bool,
+
+		#[arg(long)]
+		/// use this actor's private key to fetch
+		fetch_as: Option<String>,
+	},
+
+	/// act on remote relay actors at instance level
+	Relay {
+		#[clap(subcommand)]
+		/// action to take against this relay
+		action: RelayCommand,
+	},
+
+	/// run db maintenance tasks
+	Fix {
+		#[arg(long, default_value_t = false)]
+		/// fix likes counts for posts
+		likes: bool,
+
+		#[arg(long, default_value_t = false)]
+		/// fix shares counts for posts
+		shares: bool,
+
+		#[arg(long, default_value_t = false)]
+		/// fix replies counts for posts
+		replies: bool,
+	},
+
+	/// update remote actors
+	Update {
+		#[arg(long, short, default_value_t = 10)]
+		/// number of days after which actors should get updated
+		days: i64,
+
+		#[arg(long)]
+		/// stop after updating this many actors
+		limit: Option<u64>,
+	},
+
+	/// register a new local user
+	Register {
+		/// username for new user, must be unique locally and cannot be changed
+		username: String,
+
+		/// password for new user
+		// TODO get this with getpass rather than argv!!!!
+		password: String,
+
+		/// display name for new user
+		#[arg(long = "name")]
+		display_name: Option<String>,
+
+		/// summary text for new user
+		#[arg(long = "summary")]
+		summary: Option<String>,
+
+		/// url for avatar image of new user
+		#[arg(long = "avatar")]
+		avatar_url: Option<String>,
+
+		/// url for banner image of new user
+		#[arg(long = "banner")]
+		banner_url: Option<String>,
+	},
+
+	/// break all user relations so that instance can be shut down
+	Nuke {
+		/// unless this is set, nuke will be a dry run
+		#[arg(long, default_value_t = false)]
+		for_real: bool,
+
+		/// also send Delete activities for all local objects
+		#[arg(long, default_value_t = false)]
+		delete_objects: bool,
+	},
+
+	/// attempt to fix broken threads and completely gather their context
+	Thread {
+		
+	},
+
+	/// replaces all attachment urls with proxied local versions (only useful for old instances)
+	Cloak {
+		/// also cloak objects image urls
+		#[arg(long, default_value_t = false)]
+		objects: bool,
+
+		/// also cloak actor images
+		#[arg(long, default_value_t = false)]
+		actors: bool,
+
+		/// also replace urls inside post contents
+		#[arg(long, default_value_t = false)]
+		contents: bool,
+	},
+}
+
+pub async fn run(ctx: upub::Context, command: CliCommand) -> Result<(), Box<dyn std::error::Error>> {
+	tracing::info!("running cli task: {command:?}");
+	match command {
+		CliCommand::Faker { count } =>
+			Ok(faker(ctx, count as i64).await?),
+		CliCommand::Fetch { uri, save, fetch_as } =>
+			Ok(fetch(ctx, uri, save, fetch_as).await?),
+		CliCommand::Relay { action } =>
+			Ok(relay(ctx, action).await?),
+		CliCommand::Fix { likes, shares, replies } =>
+			Ok(fix(ctx, likes, shares, replies).await?),
+		CliCommand::Update { days, limit } =>
+			Ok(update_users(ctx, days, limit).await?),
+		CliCommand::Register { username, password, display_name, summary, avatar_url, banner_url } =>
+			Ok(register(ctx, username, password, display_name, summary, avatar_url, banner_url).await?),
+		CliCommand::Nuke { for_real, delete_objects } =>
+			Ok(nuke(ctx, for_real, delete_objects).await?),
+		CliCommand::Thread { } =>
+			Ok(thread(ctx).await?),
+		CliCommand::Cloak { objects, actors, contents } =>
+			Ok(cloak(ctx, contents, objects, actors).await?),
+	}
+}

upub/cli/src/nuke.rs

@@ -0,0 +1,137 @@
+use std::collections::HashSet;
+
+use apb::{ActivityMut, BaseMut, ObjectMut};
+use futures::TryStreamExt;
+use sea_orm::{ActiveValue::{Set, NotSet}, ColumnTrait, EntityTrait, QueryFilter, QuerySelect, SelectColumns};
+
+
+pub async fn nuke(ctx: upub::Context, for_real: bool, delete_posts: bool) -> Result<(), sea_orm::DbErr> {
+	if !for_real {
+		tracing::warn!("THIS IS A DRY RUN! pass --for-real to actually nuke this instance");
+	}
+
+	let mut to_undo = Vec::new();
+
+	// TODO rather expensive to find all local users with a LIKE query, should add an isLocal flag
+	let local_users_vec = upub::model::actor::Entity::find()
+		.filter(upub::model::actor::Column::Id.like(format!("{}%", ctx.base())))
+		.select_only()
+		.select_column(upub::model::actor::Column::Internal)
+		.into_tuple::<i64>()
+		.all(ctx.db())
+		.await?;
+	
+	let local_users : HashSet<i64> = HashSet::from_iter(local_users_vec);
+
+	{
+		let mut stream = upub::model::relation::Entity::find().stream(ctx.db()).await?;
+		while let Some(like) = stream.try_next().await? {
+			if local_users.contains(&like.follower) {
+				to_undo.push(like.activity);
+			} else if local_users.contains(&like.following) {
+				if let Some(accept) = like.accept {
+					to_undo.push(accept);
+				}
+			}
+		}
+	}
+
+	for internal in to_undo {
+		let Some(activity) = upub::model::activity::Entity::find_by_id(internal)
+			.one(ctx.db())
+			.await?
+		else {
+			tracing::error!("could not load activity #{internal}");
+			continue;
+		};
+
+		let Some(ref oid) = activity.object
+		else {
+			tracing::error!("can't undo activity without object");
+			continue;
+		};
+
+		let (target, undone) = if matches!(activity.activity_type, apb::ActivityType::Follow) {
+			(oid.clone(), activity.clone().ap())
+		} else {
+			let follow_activity = upub::model::activity::Entity::find_by_ap_id(oid)
+				.one(ctx.db())
+				.await?
+				.ok_or(sea_orm::DbErr::RecordNotFound(oid.clone()))?;
+			(follow_activity.clone().object.unwrap_or_default(), follow_activity.ap())
+		};
+
+		let aid = ctx.aid(&upub::Context::new_id());
+		let undo_activity = apb::new()
+			.set_id(Some(&aid))
+			.set_activity_type(Some(apb::ActivityType::Undo))
+			.set_actor(apb::Node::link(activity.actor.clone()))
+			.set_object(apb::Node::object(undone))
+			.set_to(apb::Node::links(vec![target]))
+			.set_published(Some(chrono::Utc::now()));
+
+
+		let job = upub::model::job::ActiveModel {
+			internal: NotSet,
+			activity: Set(aid.clone()),
+			job_type: Set(upub::model::job::JobType::Outbound),
+			actor: Set(activity.actor),
+			target: Set(None),
+			published: Set(chrono::Utc::now()),
+			not_before: Set(chrono::Utc::now()),
+			attempt: Set(0),
+			payload: Set(Some(undo_activity)),
+			error: Set(None),
+		};
+
+		tracing::info!("undoing {}", activity.id);
+
+		if for_real {
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		}
+	}
+
+	if delete_posts {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Id.like(format!("{}%", ctx.base())))
+			.stream(ctx.db())
+			.await?;
+
+		while let Some(object) = stream.try_next().await? {
+			let aid = ctx.aid(&upub::Context::new_id());
+			let actor = object.attributed_to.unwrap_or_else(|| ctx.domain().to_string());
+			let undo_activity = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Delete))
+				.set_actor(apb::Node::link(actor.clone()))
+				.set_object(apb::Node::link(object.id.clone()))
+				.set_to(apb::Node::links(object.to.0))
+				.set_cc(apb::Node::links(object.cc.0))
+				.set_bto(apb::Node::links(object.bto.0))
+				.set_bcc(apb::Node::links(object.bcc.0))
+				.set_published(Some(chrono::Utc::now()));
+
+
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(actor),
+				target: Set(None),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				attempt: Set(0),
+				payload: Set(Some(undo_activity)),
+				error: Set(None),
+			};
+
+			tracing::info!("deleting {}", object.id);
+
+			if for_real {
+				upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+			}
+		}
+	}
+
+	Ok(())
+}

upub/cli/src/register.rs

@@ -1,14 +1,14 @@
-use crate::server::admin::Administrable;
+use upub::traits::Administrable;
 
 pub async fn register(
-	ctx: crate::server::Context,
+	ctx: upub::Context,
 	username: String,
 	password: String,
 	display_name: Option<String>,
 	summary: Option<String>,
 	avatar_url: Option<String>,
 	banner_url: Option<String>,
-) -> crate::Result<()> {
+) -> Result<(), sea_orm::DbErr> {
 	ctx.register_user(
 		username.clone(),
 		password,

upub/cli/src/relay.rs

@@ -0,0 +1,204 @@
+use apb::{ActivityMut, BaseMut, ObjectMut};
+use sea_orm::{ActiveValue::{NotSet, Set}, DbErr, EntityTrait, QueryFilter, ColumnTrait};
+use upub::traits::{fetch::RequestError, Fetcher};
+
+#[derive(Debug, Clone, clap::Subcommand)]
+/// available actions to take on relays
+pub enum RelayCommand {
+	/// get all current pending and accepted relays
+	Status,
+	/// request to follow a specific relay
+	Follow {
+		/// relay actor to follow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// accept a pending relay request
+	Accept {
+		/// relay actor to accept (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// retract a follow relation to a relay, stopping receiving content
+	Unfollow {
+		/// relay actor to unfollow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+	/// remove a follow relation from a relay, stopping sending content
+	Remove {
+		/// relay actor to unfollow (must be full AP id, like for pleroma)
+		actor: String,
+	},
+}
+
+pub async fn relay(ctx: upub::Context, action: RelayCommand) -> Result<(), RequestError> {
+	let my_internal = upub::model::actor::Entity::ap_to_internal(ctx.base(), ctx.db())
+		.await?
+		.ok_or_else(|| DbErr::RecordNotFound(ctx.base().to_string()))?;
+
+	let their_internal = match &action {
+		RelayCommand::Status => 0,
+		RelayCommand::Follow { actor }
+		| RelayCommand::Accept { actor }
+		| RelayCommand::Unfollow { actor }
+		| RelayCommand::Remove { actor }
+		=> ctx.fetch_user(actor, ctx.db()).await?.internal,
+	};
+
+	match action {
+		RelayCommand::Status => {
+			tracing::info!("active sinks:");
+			for sink in upub::Query::related(None, Some(my_internal), false)
+				.into_model::<upub::model::actor::Model>()
+				.all(ctx.db())
+				.await?
+			{
+				tracing::info!("[>>] {} {}", sink.name.unwrap_or_default(), sink.id);
+			}
+
+			tracing::info!("active sources:");
+			for source in  upub::Query::related(Some(my_internal), None, false)
+				.into_model::<upub::model::actor::Model>()
+				.all(ctx.db())
+				.await?
+			{
+				tracing::info!("[<<] {} {}", source.name.unwrap_or_default(), source.id);
+			}
+		},
+
+		RelayCommand::Follow { actor } => {
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Follow))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::link(actor.clone()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("following relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Accept { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(their_internal))
+				.filter(upub::model::relation::Column::Following.eq(my_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{their_internal}-{my_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(relation.activity)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", relation.activity)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Accept(apb::AcceptType::Accept)))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::link(activity.id))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("accepting relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Remove { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(their_internal))
+				.filter(upub::model::relation::Column::Following.eq(my_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{their_internal}-{my_internal}")))?;
+			let accept_activity_id = relation.accept.ok_or(DbErr::RecordNotFound(format!("accept-{their_internal}-{my_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(accept_activity_id)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", accept_activity_id)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Undo))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::object(activity.ap()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("unfollowing relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+
+		RelayCommand::Unfollow { actor } => {
+			let relation = upub::model::relation::Entity::find()
+				.filter(upub::model::relation::Column::Follower.eq(my_internal))
+				.filter(upub::model::relation::Column::Following.eq(their_internal))
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("relation-{my_internal}-{their_internal}")))?;
+			let activity = upub::model::activity::Entity::find_by_id(relation.activity)
+				.one(ctx.db())
+				.await?
+				.ok_or_else(|| DbErr::RecordNotFound(format!("activity#{}", relation.activity)))?;
+			let aid = ctx.aid(&upub::Context::new_id());
+			let payload = apb::new()
+				.set_id(Some(&aid))
+				.set_activity_type(Some(apb::ActivityType::Undo))
+				.set_actor(apb::Node::link(ctx.base().to_string()))
+				.set_object(apb::Node::object(activity.ap()))
+				.set_to(apb::Node::links(vec![actor.clone()]))
+				.set_cc(apb::Node::links(vec![apb::target::PUBLIC.to_string()]))
+				.set_published(Some(chrono::Utc::now()));
+			let job = upub::model::job::ActiveModel {
+				internal: NotSet,
+				activity: Set(aid.clone()),
+				job_type: Set(upub::model::job::JobType::Outbound),
+				actor: Set(ctx.base().to_string()),
+				target: Set(None),
+				payload: Set(Some(payload)),
+				attempt: Set(0),
+				published: Set(chrono::Utc::now()),
+				not_before: Set(chrono::Utc::now()),
+				error: Set(None),
+			};
+			tracing::info!("unfollowing relay {actor}");
+			upub::model::job::Entity::insert(job).exec(ctx.db()).await?;
+		},
+	}
+
+	Ok(())
+}

upub/cli/src/thread.rs

@@ -0,0 +1,34 @@
+use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter};
+use upub::traits::{fetch::RequestError, Fetcher};
+
+pub async fn thread(ctx: upub::Context) -> Result<(), RequestError> {
+	use futures::TryStreamExt;
+	let db = ctx.db();
+
+	tracing::info!("fixing contexts...");
+	let mut stream = upub::model::object::Entity::find()
+		.filter(upub::model::object::Column::Context.is_null())
+		.stream(db)
+		.await?;
+
+	while let Some(mut object) = stream.try_next().await? {
+		match object.in_reply_to {
+			None => object.context = Some(object.id.clone()),
+			Some(ref in_reply_to) => {
+				let reply = ctx.fetch_object(in_reply_to, ctx.db()).await?;
+				if let Some(context) = reply.context {
+					object.context = Some(context);
+				} else {
+					continue;
+				}
+			},
+		}
+		tracing::info!("updating context of {}", object.id);
+		upub::model::object::Entity::update(object.into_active_model())
+			.exec(ctx.db())
+			.await?;
+	}
+
+	tracing::info!("done fixing contexts");
+	Ok(())
+}

upub/cli/src/update.rs

@@ -0,0 +1,48 @@
+use futures::TryStreamExt;
+use sea_orm::{ActiveModelTrait, ActiveValue::Set, ColumnTrait, EntityTrait, QueryFilter, ModelTrait};
+use upub::traits::Fetcher;
+
+pub async fn update_users(ctx: upub::Context, days: i64, limit: Option<u64>) -> Result<(), sea_orm::DbErr> {
+	let mut count = 0;
+	let mut stream = upub::model::actor::Entity::find()
+		.filter(upub::model::actor::Column::Updated.lt(chrono::Utc::now() - chrono::Duration::days(days)))
+		.stream(ctx.db())
+		.await?;
+
+
+	while let Some(user) = stream.try_next().await? {
+		if ctx.is_local(&user.id) { continue }
+		if let Some(limit) = limit {
+			if count >= limit { break }
+		}
+		match ctx.pull(&user.id).await.and_then(|x| x.actor()) {
+			Err(upub::traits::fetch::RequestError::Fetch(status, msg)) => {
+				if status.as_u16() == 410 {
+					tracing::info!("user {} has been deleted", user.id);
+					user.delete(ctx.db()).await?;
+				}
+				else if status.as_u16() == 404 {
+					tracing::info!("user {} does not exist anymore", user.id);
+					user.delete(ctx.db()).await?;
+				}
+				else {
+					tracing::warn!("could not fetch user {}: failed with status {status} -- {msg}", user.id);
+				}
+			},
+			Err(e) => tracing::warn!("could not fetch user {}: {e}", user.id),
+			Ok(doc) => match upub::AP::actor_q(&doc, Some(user.internal)) {
+				Ok(mut u) => {
+					tracing::info!("updating user {}", user.id);
+					u.updated = Set(chrono::Utc::now());
+					u.update(ctx.db()).await?;
+					count += 1;
+				},
+				Err(e) => tracing::warn!("failed deserializing user '{}': {e}", user.id),
+			},
+		}
+	}
+
+	tracing::info!("updated {count} users");
+
+	Ok(())
+}

upub/core/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "upub"
+version = "0.3.0"
+edition = "2021"
+authors = [ "alemi <me@alemi.dev>" ]
+description = "core inner workings of upub"
+license = "AGPL-3.0"
+repository = "https://git.alemi.dev/upub.git"
+readme = "README.md"
+
+[lib]
+
+[dependencies]
+thiserror = "1"
+async-recursion = "1.1"
+async-trait = "0.1"
+sha256 = "1.5" # TODO get rid of this and use directly sha2!!
+sha2 = "0.10"
+hmac = "0.12"
+openssl = "0.10" # TODO handle pubkeys with a smaller crate
+base64 = "0.22"
+chrono = { version = "0.4", features = ["serde"] }
+uuid = { version = "1.10", features = ["v4"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+serde_default = "0.2"
+serde-inline-default = "0.2"
+toml = "0.8"
+uriproxy = { path = "../../utils/uriproxy" }
+httpsign = { path = "../../utils/httpsign/" }
+mdhtml = { path = "../../utils/mdhtml/" }
+jrd = "0.1"
+tracing = "0.1"
+sea-orm = { version = "1.0", features = ["macros"] }
+reqwest = { version = "0.12", features = ["json"] }
+apb = { path = "../../apb", features = ["unstructured", "orm", "did-core", "activitypub-miscellaneous-terms", "activitypub-fe", "activitypub-counters", "litepub", "ostatus", "toot"] }
+# nodeinfo = "0.0.2" # the version on crates.io doesn't re-export necessary types to build the struct!!!
+nodeinfo = { git = "https://codeberg.org/thefederationinfo/nodeinfo-rs", rev = "e865094804" }

upub/core/README.md

@@ -0,0 +1 @@
+# upub core

upub/core/src/config.rs

@@ -12,6 +12,9 @@ pub struct Config {
 	#[serde(default)]
 	pub security: SecurityConfig,
 
+	#[serde(default)]
+	pub compat: CompatibilityConfig,
+
 	// TODO should i move app keys here?
 }
 
@@ -40,19 +43,19 @@ pub struct DatasourceConfig {
 	#[serde_inline_default("sqlite://./upub.db".into())]
 	pub connection_string: String,
 
-	#[serde_inline_default(4)]
+	#[serde_inline_default(32)]
 	pub max_connections: u32,
 
 	#[serde_inline_default(1)]
 	pub min_connections: u32,
 
-	#[serde_inline_default(300u64)]
+	#[serde_inline_default(90u64)]
 	pub connect_timeout_seconds: u64,
 
-	#[serde_inline_default(300u64)]
+	#[serde_inline_default(30u64)]
 	pub acquire_timeout_seconds: u64,
 
-	#[serde_inline_default(1u64)]
+	#[serde_inline_default(10u64)]
 	pub slow_query_warn_seconds: u64,
 
 	#[serde_inline_default(true)]
@@ -65,16 +68,55 @@ pub struct SecurityConfig {
 	#[serde(default)]
 	pub allow_registration: bool,
 
+	#[serde(default)] // TODO i don't like the name of this
+	pub require_user_approval: bool,
+
 	#[serde(default)]
 	pub allow_public_debugger: bool,
 
+	#[serde(default)]
+	pub allow_public_search: bool,
+
+	#[serde_inline_default("changeme".to_string())]
+	pub proxy_secret: String,
+
 	#[serde_inline_default(true)]
 	pub show_reply_ids: bool,
+
+	#[serde_inline_default(true)]
+	pub allow_login_refresh: bool,
+
+	#[serde_inline_default(7 * 24)]
+	pub session_duration_hours: i64,
+
+	#[serde_inline_default(2)]
+	pub max_id_redirects: u32, // TODO not sure it fits here
+
+	#[serde_inline_default(20)]
+	pub thread_crawl_depth: u32, // TODO doesn't really fit here
+
+	#[serde_inline_default(30)]
+	pub job_expiration_days: u32, // TODO doesn't really fit here
+
+	#[serde_inline_default(100)]
+	pub reinsertion_attempt_limit: u32, // TODO doesn't really fit here
 }
 
+#[serde_inline_default::serde_inline_default]
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, serde_default::DefaultFromSerde)]
+pub struct CompatibilityConfig {
+	#[serde(default)]
+	pub fix_attachment_images_media_type: bool,
+
+	#[serde(default)]
+	pub add_explicit_target_to_likes_if_local: bool,
+
+	#[serde(default)]
+	pub skip_single_attachment_if_image_is_set: bool,
+}
 
 impl Config {
-	pub fn load(path: Option<std::path::PathBuf>) -> Self {
+	pub fn load(path: Option<&std::path::PathBuf>) -> Self {
 		let Some(cfg_path) = path else { return Config::default() };
 		match std::fs::read_to_string(cfg_path) {
 			Ok(x) => match toml::from_str(&x) {
@@ -85,4 +127,8 @@ impl Config {
 		}
 		Config::default()
 	}
+
+	pub fn frontend_url(&self, url: &str) -> Option<String> {
+		Some(format!("{}{}", self.instance.frontend.as_deref()?, url))
+	}
 }

upub/core/src/context.rs

@@ -0,0 +1,191 @@
+use std::{collections::BTreeSet, sync::Arc};
+
+use sea_orm::{DatabaseConnection, DbErr, QuerySelect, SelectColumns};
+
+use crate::{config::Config, model};
+use uriproxy::UriClass;
+
+#[derive(Clone)]
+pub struct Context(Arc<ContextInner>);
+struct ContextInner {
+	db: DatabaseConnection,
+	config: Config,
+	domain: String,
+	protocol: String,
+	base_url: String,
+	// TODO keep these pre-parsed
+	actor: model::actor::Model,
+	instance: model::instance::Model,
+	pkey: String,
+	waker: Option<Box<dyn WakerToken>>,
+	#[allow(unused)] relay: Relays,
+}
+
+#[allow(unused)]
+pub struct Relays {
+	sources: BTreeSet<String>,
+	sinks: BTreeSet<String>,
+}
+
+#[macro_export]
+macro_rules! url {
+	($ctx:expr, $($args: tt)*) => {
+		format!("{}{}{}", $ctx.protocol(), $ctx.domain(), format!($($args)*))
+	};
+}
+
+pub trait WakerToken: Sync + Send {
+	fn wake(&self);
+}
+
+impl Context {
+
+	// TODO slim constructor down, maybe make a builder?
+	pub async fn new(db: DatabaseConnection, mut domain: String, config: Config, waker: Option<Box<dyn WakerToken>>) -> Result<Self, crate::init::InitError> {
+		let protocol = if domain.starts_with("http://")
+		{ "http://" } else { "https://" }.to_string();
+		if domain.ends_with('/') {
+			domain.replace_range(domain.len()-1.., "");
+		}
+		if domain.starts_with("http") {
+			domain = domain.replace("https://", "").replace("http://", "");
+		}
+		let base_url = format!("{}{}", protocol, domain);
+
+		let (actor, instance) = super::init::application(domain.clone(), base_url.clone(), &db).await?;
+
+		// TODO maybe we could provide a more descriptive error...
+		let pkey = actor.private_key.as_deref().ok_or_else(|| DbErr::RecordNotFound("application private key".into()))?.to_string();
+
+		let relay_sinks = crate::Query::related(None, Some(actor.internal), false)
+			.select_only()
+			.select_column(crate::model::actor::Column::Id)
+			.into_tuple::<String>()
+			.all(&db)
+			.await?;
+
+		let relay_sources = crate::Query::related(Some(actor.internal), None, false)
+			.select_only()
+			.select_column(crate::model::actor::Column::Id)
+			.into_tuple::<String>()
+			.all(&db)
+			.await?;
+
+		let relay = Relays {
+			sources: BTreeSet::from_iter(relay_sources),
+			sinks: BTreeSet::from_iter(relay_sinks),
+		};
+
+		Ok(Context(Arc::new(ContextInner {
+			base_url, db, domain, protocol, actor, instance, config, pkey, relay, waker,
+		})))
+	}
+
+	pub fn actor(&self) -> &model::actor::Model {
+		&self.0.actor
+	}
+
+	#[allow(unused)]
+	pub fn instance(&self) -> &model::instance::Model {
+		&self.0.instance
+	}
+
+	pub fn pkey(&self) -> &str {
+		&self.0.pkey
+	}
+
+	pub fn db(&self) -> &DatabaseConnection {
+		&self.0.db
+	}
+
+	pub fn cfg(&self) -> &Config {
+		&self.0.config
+	}
+
+	pub fn domain(&self) -> &str {
+		&self.0.domain
+	}
+
+	pub fn protocol(&self) -> &str {
+		&self.0.protocol
+	}
+
+	pub fn base(&self) -> &str {
+		&self.0.base_url
+	}
+
+	pub fn new_id() -> String {
+		uuid::Uuid::new_v4().to_string()
+	}
+
+	/// get full user id uri
+	pub fn uid(&self, id: &str) -> String {
+		uriproxy::uri(self.base(), UriClass::Actor, id)
+	}
+
+	/// get full object id uri
+	pub fn oid(&self, id: &str) -> String {
+		uriproxy::uri(self.base(), UriClass::Object, id)
+	}
+
+	/// get full activity id uri
+	pub fn aid(&self, id: &str) -> String {
+		uriproxy::uri(self.base(), UriClass::Activity, id)
+	}
+
+	/// get bare id, which is uuid for local stuff and +{uri|base64} for remote stuff
+	pub fn id(&self, full_id: &str) -> String {
+		if self.is_local(full_id) {
+			uriproxy::decompose(full_id)
+		} else {
+			uriproxy::compact(full_id)
+		}
+	}
+
+	pub fn server(id: &str) -> String {
+		id
+			.replace("https://", "")
+			.replace("http://", "")
+			.split('/')
+			.next()
+			.unwrap_or("")
+			.to_string()
+	}
+
+	pub fn is_local(&self, id: &str) -> bool {
+		id.starts_with(self.base())
+	}
+
+	pub async fn find_internal(&self, id: &str) -> Result<Option<Internal>, DbErr> {
+		if let Some(internal) = model::object::Entity::ap_to_internal(id, self.db()).await? {
+			return Ok(Some(Internal::Object(internal)));
+		}
+
+		if let Some(internal) = model::activity::Entity::ap_to_internal(id, self.db()).await? {
+			return Ok(Some(Internal::Activity(internal)));
+		}
+
+		if let Some(internal) = model::actor::Entity::ap_to_internal(id, self.db()).await? {
+			return Ok(Some(Internal::Actor(internal)));
+		}
+
+		Ok(None)
+	}
+
+	pub fn wake_workers(&self) {
+		if let Some(ref waker) = self.0.waker {
+			waker.wake();
+		}
+	}
+
+	#[allow(unused)]
+	pub fn is_relay(&self, id: &str) -> bool {
+		self.0.relay.sources.contains(id) || self.0.relay.sinks.contains(id)
+	}
+}
+
+pub enum Internal {
+	Object(i64),
+	Activity(i64),
+	Actor(i64),
+}

upub/core/src/ext.rs

@@ -0,0 +1,134 @@
+use sea_orm::{ConnectionTrait, PaginatorTrait};
+
+#[allow(async_fn_in_trait)]
+pub trait AnyQuery {
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr>;
+}
+
+impl<T : sea_orm::EntityTrait> AnyQuery for sea_orm::Select<T>
+where
+	T::Model : Sync,
+{
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr> {
+		// TODO ConnectionTrait became an iterator?? self.count(db) gives error now
+		Ok(PaginatorTrait::count(self, db).await? > 0)
+	}
+}
+
+impl<T : sea_orm::SelectorTrait + Send + Sync> AnyQuery for sea_orm::Selector<T> {
+	async fn any(self, db: &impl ConnectionTrait) -> Result<bool, sea_orm::DbErr> {
+		Ok(self.count(db).await? > 0)
+	}
+}
+
+pub trait LoggableError {
+	fn info_failed(self, msg: &str);
+	fn warn_failed(self, msg: &str);
+	fn err_failed(self, msg: &str);
+}
+
+impl<T, E: std::error::Error> LoggableError for Result<T, E> {
+	fn info_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::info!("{} : {}", msg, e);
+		}
+	}
+
+	fn warn_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::warn!("{} : {}", msg, e);
+		}
+	}
+
+	fn err_failed(self, msg: &str) {
+		if let Err(e) = self {
+			tracing::error!("{} : {}", msg, e);
+		}
+	}
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct JsonVec<T>(pub Vec<T>);
+
+impl<T> From<Vec<T>> for JsonVec<T> {
+	fn from(value: Vec<T>) -> Self {
+		JsonVec(value)
+	}
+}
+
+impl<T> Default for JsonVec<T> {
+	fn default() -> Self {
+		JsonVec(Vec::new())
+	}
+}
+
+// TODO we need this dummy to access the default implementation, which needs to be wrapped to catch
+//      nulls. is there a way to directly call super::try_get_from_json ?? i think this gets
+//      compiled into a lot of variants...
+#[derive(serde::Deserialize)]
+struct DummyVec<T>(pub Vec<T>);
+impl<T: serde::de::DeserializeOwned> sea_orm::TryGetableFromJson for DummyVec<T> {}
+
+impl<T: serde::de::DeserializeOwned> sea_orm::TryGetableFromJson for JsonVec<T> {
+	fn try_get_from_json<I: sea_orm::ColIdx>(res: &sea_orm::QueryResult, idx: I) -> Result<Self, sea_orm::TryGetError> {
+		match DummyVec::try_get_from_json(res, idx) {
+			Ok(DummyVec(x)) => Ok(Self(x)),
+			Err(sea_orm::TryGetError::Null(_)) => Ok(Self::default()),
+			Err(e) => Err(e),
+		}
+	}
+
+	fn from_json_vec(value: serde_json::Value) -> Result<Vec<Self>, sea_orm::TryGetError> {
+		match DummyVec::from_json_vec(value) {
+			Ok(x) => Ok(x.into_iter().map(|x| JsonVec(x.0)).collect()),
+			Err(sea_orm::TryGetError::Null(_)) => Ok(vec![]),
+			Err(e) => Err(e),
+		}
+	}
+}
+
+impl<T: serde::ser::Serialize> std::convert::From<JsonVec<T>> for sea_orm::Value {
+	fn from(source: JsonVec<T>) -> Self {
+		sea_orm::Value::Json(serde_json::to_value(&source).ok().map(std::boxed::Box::new))
+	}
+}
+
+impl<T: serde::de::DeserializeOwned + TypeName> sea_orm::sea_query::ValueType for JsonVec<T> {
+	fn try_from(v: sea_orm::Value) -> Result<Self, sea_orm::sea_query::ValueTypeErr> {
+		match v {
+			sea_orm::Value::Json(Some(json)) => Ok(
+				serde_json::from_value(*json).map_err(|_| sea_orm::sea_query::ValueTypeErr)?,
+			),
+			sea_orm::Value::Json(None) => Ok(JsonVec::default()),
+			_ => Err(sea_orm::sea_query::ValueTypeErr),
+		}
+	}
+
+	fn type_name() -> String {
+		format!("JsonVec_{}", T::type_name())
+	}
+
+	fn array_type() -> sea_orm::sea_query::ArrayType {
+		sea_orm::sea_query::ArrayType::Json
+	}
+
+	fn column_type() -> sea_orm::sea_query::ColumnType {
+		sea_orm::sea_query::ColumnType::Json
+	}
+}
+
+impl<T> sea_orm::sea_query::Nullable for JsonVec<T> {
+	fn null() -> sea_orm::Value {
+		sea_orm::Value::Json(None)
+	}
+}
+
+pub trait TypeName {
+	fn type_name() -> String;
+}
+
+impl TypeName for String {
+	fn type_name() -> String {
+		"String".to_string()
+	}
+}

upub/core/src/init.rs

@@ -0,0 +1,86 @@
+use openssl::rsa::Rsa;
+use sea_orm::{ActiveValue::{NotSet, Set}, DatabaseConnection, EntityTrait};
+
+use crate::{ext::JsonVec, model};
+
+#[derive(Debug, thiserror::Error)]
+pub enum InitError {
+	#[error("database error: {0:?}")]
+	Database(#[from] sea_orm::DbErr),
+
+	#[error("openssl error: {0:?}")]
+	OpenSSL(#[from] openssl::error::ErrorStack),
+
+	#[error("pem format error: {0:?}")]
+	KeyError(#[from] std::str::Utf8Error),
+}
+
+pub async fn application(
+	domain: String,
+	base_url: String,
+	db: &DatabaseConnection
+) -> Result<(model::actor::Model, model::instance::Model), InitError> {
+	Ok((
+		match model::actor::Entity::find_by_ap_id(&base_url).one(db).await? {
+			Some(model) => model,
+			None => {
+				tracing::info!("generating application keys");
+				let rsa = Rsa::generate(2048)?;
+				let privk = std::str::from_utf8(&rsa.private_key_to_pem()?)?.to_string();
+				let pubk = std::str::from_utf8(&rsa.public_key_to_pem()?)?.to_string();
+				let system = model::actor::ActiveModel {
+					internal: NotSet,
+					id: Set(base_url.clone()),
+					domain: Set(domain.clone()),
+					preferred_username: Set(domain.clone()),
+					actor_type: Set(apb::ActorType::Application),
+					also_known_as: Set(JsonVec::default()),
+					moved_to: Set(None),
+					fields: Set(JsonVec::default()), // TODO we could put some useful things here actually
+					private_key: Set(Some(privk)),
+					public_key: Set(pubk),
+					following: Set(None),
+					following_count: Set(0),
+					followers: Set(None),
+					followers_count: Set(0),
+					statuses_count: Set(0),
+					summary: Set(Some("micro social network, federated".to_string())),
+					name: Set(Some("μpub".to_string())),
+					image: Set(None),
+					icon: Set(Some("https://cdn.alemi.dev/social/circle-square.png".to_string())),
+					inbox: Set(Some(format!("{base_url}/inbox"))),
+					shared_inbox: Set(Some(format!("{base_url}/inbox"))),
+					outbox: Set(Some(format!("{base_url}/outbox"))),
+					published: Set(chrono::Utc::now()),
+					updated: Set(chrono::Utc::now()),
+				};
+				model::actor::Entity::insert(system).exec(db).await?;
+				// sqlite doesn't resurn last inserted id so we're better off just querying again, it's just one time
+				model::actor::Entity::find().one(db).await?.expect("could not find app actor just inserted")
+			}
+		},
+
+		match model::instance::Entity::find_by_domain(&domain).one(db).await? {
+			Some(model) => model,
+			None => {
+				tracing::info!("generating instance counters");
+				let system = model::instance::ActiveModel {
+					internal: NotSet,
+					domain: Set(domain.clone()),
+					down_since: Set(None),
+					software: Set(Some("upub".to_string())),
+					version: Set(Some(crate::VERSION.to_string())),
+					name: Set(None),
+					icon: Set(None),
+					users: Set(Some(0)),
+					posts: Set(Some(0)),
+					published: Set(chrono::Utc::now()),
+					updated: Set(chrono::Utc::now()),
+				};
+				model::instance::Entity::insert(system).exec(db).await?;
+				// sqlite doesn't resurn last inserted id so we're better off just querying again, it's just one time
+				model::instance::Entity::find().one(db).await?.expect("could not find app instance just inserted")
+			}
+		}
+	))
+}

upub/core/src/lib.rs

@@ -0,0 +1,18 @@
+pub mod model;
+pub mod traits;
+
+pub mod context;
+pub use context::Context;
+
+pub mod config;
+pub use config::Config;
+
+pub mod init;
+pub mod ext;
+
+pub mod selector;
+pub use selector::Query;
+
+pub use traits::normalize::AP;
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");

upub/core/src/model/activity.rs

@@ -1,23 +1,23 @@
 use apb::{ActivityMut, ActivityType, BaseMut, ObjectMut};
-use sea_orm::entity::prelude::*;
+use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::routes::activitypub::jsonld::LD;
+use crate::ext::JsonVec;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "activities")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
+	pub internal: i64,
 	#[sea_orm(unique)]
-	pub ap_id: String,
+	pub id: String,
 	pub activity_type: ActivityType,
-	pub actor: i32,
-	pub object: Option<i32>,
+	pub actor: String,
+	pub object: Option<String>,
 	pub target: Option<String>,
-	pub to: Option<Json>,
-	pub bto: Option<Json>,
-	pub cc: Option<Json>,
-	pub bcc: Option<Json>,
+	pub to: JsonVec<String>,
+	pub bto: JsonVec<String>,
+	pub cc: JsonVec<String>,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 }
 
@@ -33,8 +33,8 @@ pub enum Relation {
 	Actors,
 	#[sea_orm(has_many = "super::addressing::Entity")]
 	Addressing,
-	#[sea_orm(has_many = "super::delivery::Entity")]
-	Deliveries,
+	#[sea_orm(has_many = "super::notification::Entity")]
+	Notifications,
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
 		from = "Column::Object",
@@ -57,9 +57,9 @@ impl Related<super::addressing::Entity> for Entity {
 	}
 }
 
-impl Related<super::delivery::Entity> for Entity {
+impl Related<super::notification::Entity> for Entity {
 	fn to() -> RelationDef {
-		Relation::Deliveries.def()
+		Relation::Notifications.def()
 	}
 }
 
@@ -72,32 +72,24 @@ impl Related<super::object::Entity> for Entity {
 impl ActiveModelBehavior for ActiveModel {}
 
 impl Entity {
-	pub fn find_by_ap_id(ap_id: &str) -> Select<Entity> {
-		Entity::find().filter(Column::ApId.eq(ap_id))
+	pub fn find_by_ap_id(id: &str) -> Select<Entity> {
+		Entity::find().filter(Column::Id.eq(id))
 	}
-}
 
-impl ActiveModel {
-	pub fn new(activity: &impl apb::Activity) -> Result<Self, super::FieldError> {
-		Ok(ActiveModel {
-			id: sea_orm::ActiveValue::NotSet,
-			ap_id: sea_orm::ActiveValue::Set(activity.id().ok_or(super::FieldError("id"))?.to_string()),
-			activity_type: sea_orm::ActiveValue::Set(activity.activity_type().ok_or(super::FieldError("type"))?),
-			actor: sea_orm::ActiveValue::Set(activity.actor().id().ok_or(super::FieldError("actor"))?),
-			object: sea_orm::ActiveValue::Set(activity.object().id()),
-			target: sea_orm::ActiveValue::Set(activity.target().id()),
-			published: sea_orm::ActiveValue::Set(activity.published().unwrap_or(chrono::Utc::now())),
-			to: sea_orm::ActiveValue::Set(activity.to().into()),
-			bto: sea_orm::ActiveValue::Set(activity.bto().into()),
-			cc: sea_orm::ActiveValue::Set(activity.cc().into()),
-			bcc: sea_orm::ActiveValue::Set(activity.bcc().into()),
-		})
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
+		Entity::find()
+			.filter(Column::Id.eq(id))
+			.select_only()
+			.select_column(Column::Internal)
+			.into_tuple::<i64>()
+			.one(db)
+			.await
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_activity_type(Some(self.activity_type))
 			.set_actor(apb::Node::link(self.actor))
@@ -119,4 +111,10 @@ impl apb::target::Addressed for Model {
 		to.append(&mut self.bcc.0.clone());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to = self.to.0.clone();
+		to.append(&mut self.bto.0.clone());
+		to
+	}
 }

upub/core/src/model/actor.rs

@@ -1,24 +1,53 @@
-use sea_orm::entity::prelude::*;
+use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use apb::{Actor, ActorMut, ActorType, BaseMut, DocumentMut, Endpoints, EndpointsMut, Object, ObjectMut, PublicKey, PublicKeyMut};
+use apb::{field::OptionalString, ActorMut, ActorType, BaseMut, DocumentMut, EndpointsMut, ObjectMut, PublicKeyMut};
 
-use crate::routes::activitypub::jsonld::LD;
+use crate::ext::{JsonVec, TypeName};
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct Field {
+	#[serde(default)]
+	pub name: String,
+	#[serde(default)]
+	pub value: String,
+	#[serde(default, skip_serializing_if = "Option::is_none")]
+	pub verified_at: Option<ChronoDateTimeUtc>,
+	#[serde(default, rename = "type")]
+	pub field_type: String,
+}
+
+impl TypeName for Field {
+	fn type_name() -> String {
+		"Field".to_string()
+	}
+}
+
+impl<T: apb::Object> From<T> for Field {
+	fn from(value: T) -> Self {
+		Field {
+			name: value.name().str().unwrap_or_default(),
+			value: mdhtml::safe_html(value.value().unwrap_or_default()),
+			field_type: "PropertyValue".to_string(), // TODO can we try parsing this instead??
+			verified_at: None, // TODO where does verified_at come from? extend apb maybe
+		}
+	}
+}
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "actors")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
+	pub internal: i64,
 	#[sea_orm(unique)]
-	pub ap_id: String,
+	pub id: String,
 	pub actor_type: ActorType,
-	pub instance: i32,
+	pub domain: String,
 	pub name: Option<String>,
 	pub summary: Option<String>,
 	pub image: Option<String>,
 	pub icon: Option<String>,
-	#[sea_orm(unique)]
 	pub preferred_username: String,
+	pub fields: JsonVec<Field>,
 	pub inbox: Option<String>,
 	pub shared_inbox: Option<String>,
 	pub outbox: Option<String>,
@@ -29,8 +58,10 @@ pub struct Model {
 	pub statuses_count: i32,
 	pub public_key: String,
 	pub private_key: Option<String>,
-	pub created: ChronoDateTimeUtc,
+	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
+	pub also_known_as: JsonVec<String>,
+	pub moved_to: Option<String>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -41,26 +72,30 @@ pub enum Relation {
 	Addressing,
 	#[sea_orm(has_many = "super::announce::Entity")]
 	Announces,
-	#[sea_orm(has_one = "super::config::Entity")]
+	#[sea_orm(has_many = "super::config::Entity")]
 	Configs,
-	#[sea_orm(has_one = "super::credential::Entity")]
+	#[sea_orm(has_many = "super::credential::Entity")]
 	Credentials,
-	#[sea_orm(has_many = "super::delivery::Entity")]
-	Deliveries,
 	#[sea_orm(
 		belongs_to = "super::instance::Entity",
-		from = "Column::Instance",
-		to = "super::instance::Column::Id",
+		from = "Column::Domain",
+		to = "super::instance::Column::Domain",
 		on_update = "Cascade",
 		on_delete = "NoAction"
 	)]
 	Instances,
+	#[sea_orm(has_many = "super::dislike::Entity")]
+	Dislikes,
 	#[sea_orm(has_many = "super::like::Entity")]
 	Likes,
 	#[sea_orm(has_many = "super::mention::Entity")]
 	Mentions,
+	#[sea_orm(has_many = "super::notification::Entity")]
+	Notifications,
 	#[sea_orm(has_many = "super::object::Entity")]
 	Objects,
+	#[sea_orm(has_many = "super::relation::Entity")]
+	Relations,
 	#[sea_orm(has_many = "super::session::Entity")]
 	Sessions,
 }
@@ -95,18 +130,18 @@ impl Related<super::credential::Entity> for Entity {
 	}
 }
 
-impl Related<super::delivery::Entity> for Entity {
-	fn to() -> RelationDef {
-		Relation::Deliveries.def()
-	}
-}
-
 impl Related<super::instance::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Instances.def()
 	}
 }
 
+impl Related<super::dislike::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Dislikes.def()
+	}
+}
+
 impl Related<super::like::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Likes.def()
@@ -119,12 +154,24 @@ impl Related<super::mention::Entity> for Entity {
 	}
 }
 
+impl Related<super::notification::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Notifications.def()
+	}
+}
+
 impl Related<super::object::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Objects.def()
 	}
 }
 
+impl Related<super::relation::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Relations.def()
+	}
+}
+
 impl Related<super::session::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Sessions.def()
@@ -134,64 +181,50 @@ impl Related<super::session::Entity> for Entity {
 impl ActiveModelBehavior for ActiveModel {}
 
 impl Entity {
-	pub fn find_by_ap_id(ap_id: &str) -> Select<Entity> {
-		Entity::find().filter(Column::ApId.eq(ap_id))
+	pub fn find_by_ap_id(id: &str) -> Select<Entity> {
+		Entity::find().filter(Column::Id.eq(id))
 	}
 
-	pub fn find_with_instance() -> Select<Entity> {
+	pub fn delete_by_ap_id(id: &str) -> sea_orm::DeleteMany<Entity> {
+		Entity::delete_many().filter(Column::Id.eq(id))
+	}
+
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
 		Entity::find()
-			.left_join(Relation::Instances.def())
-	}
-}
-
-impl ActiveModel {
-	pub fn new(object: &impl Actor, instance: i32) -> Result<Self, super::FieldError> {
-		let ap_id = object.id().ok_or(super::FieldError("id"))?.to_string();
-		let (_domain, fallback_preferred_username) = split_user_id(&ap_id);
-		Ok(ActiveModel {
-			instance: sea_orm::ActiveValue::Set(instance), // TODO receiving it from outside is cheap
-			id: sea_orm::ActiveValue::NotSet,
-			ap_id: sea_orm::ActiveValue::Set(ap_id),
-			preferred_username: sea_orm::ActiveValue::Set(object.preferred_username().unwrap_or(&fallback_preferred_username).to_string()),
-			actor_type: sea_orm::ActiveValue::Set(object.actor_type().ok_or(super::FieldError("type"))?),
-			name: sea_orm::ActiveValue::Set(object.name().map(|x| x.to_string())),
-			summary: sea_orm::ActiveValue::Set(object.summary().map(|x| x.to_string())),
-			icon: sea_orm::ActiveValue::Set(object.icon().get().and_then(|x| x.url().id())),
-			image: sea_orm::ActiveValue::Set(object.image().get().and_then(|x| x.url().id())),
-			inbox: sea_orm::ActiveValue::Set(object.inbox().id()),
-			outbox: sea_orm::ActiveValue::Set(object.outbox().id()),
-			shared_inbox: sea_orm::ActiveValue::Set(object.endpoints().get().and_then(|x| Some(x.shared_inbox()?.to_string()))),
-			followers: sea_orm::ActiveValue::Set(object.followers().id()),
-			following: sea_orm::ActiveValue::Set(object.following().id()),
-			created: sea_orm::ActiveValue::Set(object.published().unwrap_or(chrono::Utc::now())),
-			updated: sea_orm::ActiveValue::Set(chrono::Utc::now()),
-			following_count: sea_orm::ActiveValue::Set(object.following_count().unwrap_or(0) as i64),
-			followers_count: sea_orm::ActiveValue::Set(object.followers_count().unwrap_or(0) as i64),
-			statuses_count: sea_orm::ActiveValue::Set(object.statuses_count().unwrap_or(0) as i64),
-			public_key: sea_orm::ActiveValue::Set(object.public_key().get().ok_or(super::FieldError("publicKey"))?.public_key_pem().to_string()),
-			private_key: sea_orm::ActiveValue::Set(None), // there's no way to transport privkey over AP json, must come from DB
-		})
+			.filter(Column::Id.eq(id))
+			.select_only()
+			.select_column(Column::Internal)
+			.into_tuple::<i64>()
+			.one(db)
+			.await
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_actor_type(Some(self.actor_type))
 			.set_name(self.name.as_deref())
 			.set_summary(self.summary.as_deref())
 			.set_icon(apb::Node::maybe_object(self.icon.map(|i|
-				serde_json::Value::new_object()
+				apb::new()
 					.set_document_type(Some(apb::DocumentType::Image))
 					.set_url(apb::Node::link(i.clone()))
 			)))
 			.set_image(apb::Node::maybe_object(self.image.map(|i|
-				serde_json::Value::new_object()
+				apb::new()
 					.set_document_type(Some(apb::DocumentType::Image))
 					.set_url(apb::Node::link(i.clone()))
 			)))
-			.set_published(Some(self.created))
+			.set_attachment(apb::Node::array(
+				self.fields.0
+					.into_iter()
+					.filter_map(|x| serde_json::to_value(x).ok())
+					.collect()
+			))
+			.set_published(Some(self.published))
+			.set_updated(if self.updated != self.published { Some(self.updated) } else { None })
 			.set_preferred_username(Some(&self.preferred_username))
 			.set_statuses_count(Some(self.statuses_count as u64))
 			.set_followers_count(Some(self.followers_count as u64))
@@ -201,25 +234,17 @@ impl Model {
 			.set_following(apb::Node::maybe_link(self.following))
 			.set_followers(apb::Node::maybe_link(self.followers))
 			.set_public_key(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_id(Some(&format!("{}#main-key", self.id)))
 					.set_owner(Some(&self.id))
 					.set_public_key_pem(&self.public_key)
 			))
 			.set_endpoints(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_shared_inbox(self.shared_inbox.as_deref())
 			))
+			.set_also_known_as(apb::Node::links(self.also_known_as.0))
+			.set_moved_to(apb::Node::maybe_link(self.moved_to))
 			.set_discoverable(Some(true))
 	}
 }
-
-fn split_user_id(id: &str) -> (String, String) {
-	let clean = id
-		.replace("http://", "")
-		.replace("https://", "");
-	let mut splits = clean.split('/');
-	let first = splits.next().unwrap_or("");
-	let last = splits.last().unwrap_or(first);
-	(first.to_string(), last.to_string())
-}

upub/core/src/model/addressing.rs

@@ -0,0 +1,75 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "addressing")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub actor: Option<i64>,
+	pub instance: Option<i64>,
+	pub activity: Option<i64>,
+	pub object: Option<i64>,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Activity",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Activities,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+	#[sea_orm(
+		belongs_to = "super::instance::Entity",
+		from = "Column::Instance",
+		to = "super::instance::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "NoAction"
+	)]
+	Instances,
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::activity::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Activities.def()
+	}
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::instance::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Instances.def()
+	}
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}

upub/core/src/model/announce.rs

@@ -4,9 +4,9 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "announces")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
-	pub announces: i32,
+	pub internal: i64,
+	pub actor: i64,
+	pub object: i64,
 	pub published: ChronoDateTimeUtc,
 }
 
@@ -15,15 +15,15 @@ pub enum Relation {
 	#[sea_orm(
 		belongs_to = "super::actor::Entity",
 		from = "Column::Actor",
-		to = "super::actor::Column::Id",
+		to = "super::actor::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]
 	Actors,
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
-		from = "Column::Announces",
-		to = "super::object::Column::Id",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]
@@ -43,3 +43,9 @@ impl Related<super::object::Entity> for Entity {
 }
 
 impl ActiveModelBehavior for ActiveModel {}
+
+impl Entity {
+	pub fn find_by_uid_oid(uid: i64, oid: i64) -> Select<Entity> {
+		Entity::find().filter(Column::Actor.eq(uid)).filter(Column::Object.eq(oid))
+	}
+}

upub/core/src/model/attachment.rs

@@ -0,0 +1,45 @@
+use apb::{DocumentMut, DocumentType, ObjectMut};
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "attachments")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	#[sea_orm(unique)]
+	pub url: String,
+	pub object: i64,
+	pub document_type: DocumentType,
+	pub name: Option<String>,
+	pub media_type: String,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Model {
+	pub fn ap(self) -> serde_json::Value {
+		apb::new()
+			.set_url(apb::Node::link(self.url))
+			.set_document_type(Some(self.document_type))
+			.set_media_type(Some(&self.media_type))
+			.set_name(self.name.as_deref())
+	}
+}

upub/core/src/model/config.rs

@@ -4,8 +4,9 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "configs")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
+	pub internal: i64,
+	#[sea_orm(unique)]
+	pub actor: String,
 	pub accept_follow_requests: bool,
 	pub show_followers_count: bool,
 	pub show_following_count: bool,
@@ -16,7 +17,7 @@ pub struct Model {
 impl Default for Model {
 	fn default() -> Self {
 		Model {
-			id: 0, actor: 0,
+			internal: 0, actor: "".into(),
 			accept_follow_requests: true,
 			show_following_count: true,
 			show_following: true,

upub/core/src/model/credential.rs

@@ -4,10 +4,12 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "credentials")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
+	pub internal: i64,
+	#[sea_orm(unique)]
+	pub actor: String,
 	pub login: String,
 	pub password: String,
+	pub active: bool,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

upub/core/src/model/dislike.rs

@@ -0,0 +1,51 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "dislikes")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub actor: i64,
+	pub object: i64,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+	#[sea_orm(
+		belongs_to = "super::object::Entity",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Objects,
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::object::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Objects.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Entity {
+	pub fn find_by_uid_oid(uid: i64, oid: i64) -> Select<Entity> {
+		Entity::find().filter(Column::Actor.eq(uid)).filter(Column::Object.eq(oid))
+	}
+}

upub/core/src/model/hashtag.rs

@@ -4,10 +4,9 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "hashtags")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub object: i32,
+	pub internal: i64,
+	pub object: i64,
 	pub name: String,
-	pub published: ChronoDateTimeUtc,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -15,7 +14,7 @@ pub enum Relation {
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
 		from = "Column::Object",
-		to = "super::object::Column::Id",
+		to = "super::object::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]

upub/core/src/model/instance.rs

@@ -1,17 +1,20 @@
-use sea_orm::entity::prelude::*;
+use nodeinfo::NodeInfoOwned;
+use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "instances")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub name: Option<String>,
+	pub internal: i64,
+	#[sea_orm(unique)]
 	pub domain: String,
+	pub name: Option<String>,
 	pub software: Option<String>,
 	pub version: Option<String>,
+	pub icon: Option<String>,
 	pub down_since: Option<ChronoDateTimeUtc>,
-	pub users: Option<i32>,
-	pub posts: Option<i32>,
+	pub users: Option<i64>,
+	pub posts: Option<i64>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
 }
@@ -42,4 +45,25 @@ impl Entity {
 	pub fn find_by_domain(domain: &str) -> Select<Entity> {
 		Entity::find().filter(Column::Domain.eq(domain))
 	}
+
+	pub async fn domain_to_internal(domain: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
+		Entity::find()
+			.filter(Column::Domain.eq(domain))
+			.select_only()
+			.select_column(Column::Internal)
+			.into_tuple::<i64>()
+			.one(db)
+			.await
+	}
+
+	pub async fn nodeinfo(domain: &str) -> reqwest::Result<NodeInfoOwned> {
+		match reqwest::get(format!("https://{domain}/nodeinfo/2.0.json")).await {
+			Ok(res) => res.json().await,
+			// ughhh pleroma wants with json, key without
+			Err(_) => reqwest::get(format!("https://{domain}/nodeinfo/2.0.json"))
+				.await?
+				.json()
+				.await,
+		}
+	}
 }

upub/core/src/model/job.rs

@@ -0,0 +1,59 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[sea_orm(rs_type = "i16", db_type = "SmallInteger")]
+pub enum JobType {
+	Inbound = 1,
+	Outbound = 2,
+	Delivery = 3,
+}
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "jobs")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub job_type: JobType,
+	pub actor: String,
+	pub target: Option<String>,
+	pub activity: String,
+	pub payload: Option<serde_json::Value>,
+	pub published: ChronoDateTimeUtc,
+	pub not_before: ChronoDateTimeUtc,
+	pub attempt: i16,
+	pub error: Option<String>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl Model {
+	pub fn next_attempt(&self) -> ChronoDateTimeUtc {
+		match self.attempt {
+			0 => chrono::Utc::now() + std::time::Duration::from_secs(10),
+			1 => chrono::Utc::now() + std::time::Duration::from_secs(60),
+			2 => chrono::Utc::now() + std::time::Duration::from_secs(5 * 60),
+			3 => chrono::Utc::now() + std::time::Duration::from_secs(20 * 60),
+			4 => chrono::Utc::now() + std::time::Duration::from_secs(60 * 60),
+			5 => chrono::Utc::now() + std::time::Duration::from_secs(12 * 60 * 60),
+			_ => chrono::Utc::now() + std::time::Duration::from_secs(24 * 60 * 60),
+		}
+	}
+
+	pub fn repeat(self, error: Option<String>) -> ActiveModel {
+		ActiveModel {
+			internal: sea_orm::ActiveValue::NotSet,
+			job_type: sea_orm::ActiveValue::Set(self.job_type),
+			not_before: sea_orm::ActiveValue::Set(self.next_attempt()),
+			actor: sea_orm::ActiveValue::Set(self.actor),
+			target: sea_orm::ActiveValue::Set(self.target),
+			payload: sea_orm::ActiveValue::Set(self.payload),
+			activity: sea_orm::ActiveValue::Set(self.activity),
+			published: sea_orm::ActiveValue::Set(self.published),
+			attempt: sea_orm::ActiveValue::Set(self.attempt + 1),
+			error: sea_orm::ActiveValue::Set(error),
+		}
+	}
+}

upub/core/src/model/like.rs

@@ -4,9 +4,9 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "likes")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub actor: i32,
-	pub likes: i32,
+	pub internal: i64,
+	pub actor: i64,
+	pub object: i64,
 	pub published: ChronoDateTimeUtc,
 }
 
@@ -15,15 +15,15 @@ pub enum Relation {
 	#[sea_orm(
 		belongs_to = "super::actor::Entity",
 		from = "Column::Actor",
-		to = "super::actor::Column::Id",
+		to = "super::actor::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]
 	Actors,
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
-		from = "Column::Likes",
-		to = "super::object::Column::Id",
+		from = "Column::Object",
+		to = "super::object::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]
@@ -43,3 +43,9 @@ impl Related<super::object::Entity> for Entity {
 }
 
 impl ActiveModelBehavior for ActiveModel {}
+
+impl Entity {
+	pub fn find_by_uid_oid(uid: i64, oid: i64) -> Select<Entity> {
+		Entity::find().filter(Column::Actor.eq(uid)).filter(Column::Object.eq(oid))
+	}
+}

upub/core/src/model/mention.rs

@@ -4,10 +4,9 @@ use sea_orm::entity::prelude::*;
 #[sea_orm(table_name = "mentions")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
-	pub object: i32,
-	pub actor: i32,
-	pub published: ChronoDateTimeUtc,
+	pub internal: i64,
+	pub object: i64,
+	pub actor: i64,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -23,7 +22,7 @@ pub enum Relation {
 	#[sea_orm(
 		belongs_to = "super::object::Entity",
 		from = "Column::Object",
-		to = "super::object::Column::Id",
+		to = "super::object::Column::Internal",
 		on_update = "Cascade",
 		on_delete = "Cascade"
 	)]

upub/core/src/model/mod.rs

@@ -0,0 +1,21 @@
+pub mod actor;
+pub mod object;
+pub mod activity;
+
+pub mod config;
+pub mod credential;
+pub mod session;
+
+pub mod instance;
+pub mod job;
+pub mod addressing;
+pub mod notification;
+pub mod relation;
+
+pub mod announce;
+pub mod like;
+pub mod dislike;
+
+pub mod hashtag;
+pub mod mention;
+pub mod attachment;

upub/core/src/model/notification.rs

@@ -0,0 +1,46 @@
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "notifications")]
+pub struct Model {
+	#[sea_orm(primary_key)]
+	pub internal: i64,
+	pub activity: i64,
+	pub actor: i64,
+	pub seen: bool,
+	pub published: ChronoDateTimeUtc,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+	#[sea_orm(
+		belongs_to = "super::activity::Entity",
+		from = "Column::Activity",
+		to = "super::activity::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Activities,
+	#[sea_orm(
+		belongs_to = "super::actor::Entity",
+		from = "Column::Actor",
+		to = "super::actor::Column::Internal",
+		on_update = "Cascade",
+		on_delete = "Cascade"
+	)]
+	Actors,
+}
+
+impl Related<super::actor::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Actors.def()
+	}
+}
+
+impl Related<super::activity::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Activities.def()
+	}
+}
+
+impl ActiveModelBehavior for ActiveModel {}

upub/core/src/model/object.rs

@@ -1,22 +1,22 @@
-use apb::{BaseMut, Collection, CollectionMut, ObjectMut};
-use sea_orm::entity::prelude::*;
+use apb::{BaseMut, CollectionMut, DocumentMut, ObjectMut, ObjectType};
+use sea_orm::{entity::prelude::*, QuerySelect, SelectColumns};
 
-use crate::routes::activitypub::jsonld::LD;
-
-use super::Audience;
+use crate::ext::JsonVec;
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
 #[sea_orm(table_name = "objects")]
 pub struct Model {
 	#[sea_orm(primary_key)]
-	pub id: i32,
+	pub internal: i64,
 	#[sea_orm(unique)]
-	pub ap_id: String,
-	pub object_type: String,
+	pub id: String,
+	pub object_type: ObjectType,
 	pub attributed_to: Option<String>,
 	pub name: Option<String>,
 	pub summary: Option<String>,
 	pub content: Option<String>,
+	pub image: Option<String>,
+	pub quote: Option<String>,
 	pub sensitive: bool,
 	pub in_reply_to: Option<String>,
 	pub url: Option<String>,
@@ -24,12 +24,14 @@ pub struct Model {
 	pub announces: i32,
 	pub replies: i32,
 	pub context: Option<String>,
-	pub to: Option<Json>,
-	pub bto: Option<Json>,
-	pub cc: Option<Json>,
-	pub bcc: Option<Json>,
+	pub to: JsonVec<String>,
+	pub bto: JsonVec<String>,
+	pub cc: JsonVec<String>,
+	pub bcc: JsonVec<String>,
 	pub published: ChronoDateTimeUtc,
 	pub updated: ChronoDateTimeUtc,
+
+	pub audience: Option<String>, // added with migration m20240606_000001
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -50,12 +52,30 @@ pub enum Relation {
 	Announces,
 	#[sea_orm(has_many = "super::attachment::Entity")]
 	Attachments,
+	#[sea_orm(has_many = "super::dislike::Entity")]
+	Dislikes,
 	#[sea_orm(has_many = "super::hashtag::Entity")]
 	Hashtags,
 	#[sea_orm(has_many = "super::like::Entity")]
 	Likes,
 	#[sea_orm(has_many = "super::mention::Entity")]
 	Mentions,
+	#[sea_orm(
+		belongs_to = "Entity",
+		from = "Column::InReplyTo",
+		to = "Column::Id",
+		on_update = "NoAction",
+		on_delete = "NoAction"
+	)]
+	ObjectsReply,
+	#[sea_orm(
+		belongs_to = "Entity",
+		from = "Column::Quote",
+		to = "Column::Id",
+		on_update = "NoAction",
+		on_delete = "NoAction"
+	)]
+	ObjectsQuote,
 }
 
 impl Related<super::activity::Entity> for Entity {
@@ -88,6 +108,12 @@ impl Related<super::attachment::Entity> for Entity {
 	}
 }
 
+impl Related<super::dislike::Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::Dislikes.def()
+	}
+}
+
 impl Related<super::hashtag::Entity> for Entity {
 	fn to() -> RelationDef {
 		Relation::Hashtags.def()
@@ -106,59 +132,55 @@ impl Related<super::mention::Entity> for Entity {
 	}
 }
 
-impl ActiveModelBehavior for ActiveModel {}
-
-impl Entity {
-	pub fn find_by_ap_id(ap_id: &str) -> Select<Entity> {
-		Entity::find().filter(Column::ApId.eq(ap_id))
+impl Related<Entity> for Entity {
+	fn to() -> RelationDef {
+		Relation::ObjectsReply.def()
 	}
 }
 
-impl ActiveModel {
-	pub fn new(object: &impl apb::Object) -> Result<Self, super::FieldError> {
-		Ok(ActiveModel {
-			id: sea_orm::ActiveValue::NotSet,
-			ap_id: sea_orm::ActiveValue::Set(object.id().ok_or(super::FieldError("id"))?.to_string()),
-			object_type: sea_orm::ActiveValue::Set(object.object_type().ok_or(super::FieldError("type"))?),
-			attributed_to: sea_orm::ActiveValue::Set(object.attributed_to().id()),
-			name: sea_orm::ActiveValue::Set(object.name().map(|x| x.to_string())),
-			summary: sea_orm::ActiveValue::Set(object.summary().map(|x| x.to_string())),
-			content: sea_orm::ActiveValue::Set(object.content().map(|x| x.to_string())),
-			context: sea_orm::ActiveValue::Set(object.context().id()),
-			in_reply_to: sea_orm::ActiveValue::Set(object.in_reply_to().id()),
-			published: sea_orm::ActiveValue::Set(object.published().ok_or(super::FieldError("published"))?),
-			updated: sea_orm::ActiveValue::Set(object.updated()),
-			url: sea_orm::ActiveValue::Set(object.url().id()),
-			replies: sea_orm::ActiveValue::Set(object.replies().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i64),
-			likes: sea_orm::ActiveValue::Set(object.likes().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i64),
-			announces: sea_orm::ActiveValue::Set(object.shares().get()
-				.map_or(0, |x| x.total_items().unwrap_or(0)) as i64),
-			to: sea_orm::ActiveValue::Set(object.to().into()),
-			bto: sea_orm::ActiveValue::Set(object.bto().into()),
-			cc: sea_orm::ActiveValue::Set(object.cc().into()),
-			bcc: sea_orm::ActiveValue::Set(object.bcc().into()),
+impl ActiveModelBehavior for ActiveModel {}
 
-			sensitive: sea_orm::ActiveValue::Set(object.sensitive().unwrap_or(false)),
-		})
+impl Entity {
+	pub fn find_by_ap_id(id: &str) -> Select<Entity> {
+		Entity::find().filter(Column::Id.eq(id))
+	}
+
+	pub fn delete_by_ap_id(id: &str) -> sea_orm::DeleteMany<Entity> {
+		Entity::delete_many().filter(Column::Id.eq(id))
+	}
+
+	pub async fn ap_to_internal(id: &str, db: &impl ConnectionTrait) -> Result<Option<i64>, DbErr> {
+		Entity::find()
+			.filter(Column::Id.eq(id))
+			.select_only()
+			.select_column(Column::Internal)
+			.into_tuple::<i64>()
+			.one(db)
+			.await
 	}
 }
 
 impl Model {
 	pub fn ap(self) -> serde_json::Value {
-		serde_json::Value::new_object()
+		apb::new()
 			.set_id(Some(&self.id))
 			.set_object_type(Some(self.object_type))
 			.set_attributed_to(apb::Node::maybe_link(self.attributed_to))
 			.set_name(self.name.as_deref())
 			.set_summary(self.summary.as_deref())
 			.set_content(self.content.as_deref())
+			.set_image(apb::Node::maybe_object(self.image.map(|x| 
+				apb::new()
+					.set_document_type(Some(apb::DocumentType::Image))
+					.set_url(apb::Node::link(x))
+			)))
 			.set_context(apb::Node::maybe_link(self.context.clone()))
 			.set_conversation(apb::Node::maybe_link(self.context.clone())) // duplicate context for mastodon
 			.set_in_reply_to(apb::Node::maybe_link(self.in_reply_to.clone()))
+			.set_quote_url(apb::Node::maybe_link(self.quote.clone()))
 			.set_published(Some(self.published))
-			.set_updated(Some(self.updated))
+			.set_updated(if self.updated != self.published { Some(self.updated) } else { None })
+			.set_audience(apb::Node::maybe_link(self.audience))
 			.set_to(apb::Node::links(self.to.0.clone()))
 			.set_bto(apb::Node::Empty)
 			.set_cc(apb::Node::links(self.cc.0.clone()))
@@ -166,19 +188,19 @@ impl Model {
 			.set_url(apb::Node::maybe_link(self.url))
 			.set_sensitive(Some(self.sensitive))
 			.set_shares(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
-					.set_total_items(Some(self.shares as u64))
+					.set_total_items(Some(self.announces as u64))
 			))
 			.set_likes(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
 					.set_total_items(Some(self.likes as u64))
 			))
 			.set_replies(apb::Node::object(
-				serde_json::Value::new_object()
+				apb::new()
 					.set_collection_type(Some(apb::CollectionType::OrderedCollection))
-					.set_total_items(Some(self.comments as u64))
+					.set_total_items(Some(self.replies as u64))
 			))
 	}
 }
@@ -191,4 +213,10 @@ impl apb::target::Addressed for Model {
 		to.append(&mut self.bcc.0.clone());
 		to
 	}
+
+	fn mentioning(&self) -> Vec<String> {
+		let mut to = self.to.0.clone();
+		to.append(&mut self.bto.0.clone());
+		to
+	}
 }