feat: process, store and fill obj hashtags/mentions

upub/core/src/selector.rs

@@ -1,5 +1,7 @@
-use apb::{ActivityMut, ObjectMut};
+use std::collections::{hash_map::Entry, HashMap};
-use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, LoaderTrait, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns};
+
+use apb::{ActivityMut, LinkMut, ObjectMut};
+use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns};
 
 use crate::model;
 
@@ -76,18 +78,13 @@ impl Query {
 
 
 
-#[derive(Debug, Clone)]
-pub struct EmbeddedActivity {
-	pub activity: model::activity::Model,
-	pub object: model::object::Model,
-	pub liked: Option<i64>,
-}
-
 pub struct RichActivity {
 	pub activity: model::activity::Model,
 	pub object: Option<model::object::Model>,
 	pub liked: Option<i64>,
 	pub attachments: Option<Vec<model::attachment::Model>>,
+	pub hashtags: Option<Vec<model::hashtag::Model>>,
+	pub mentions: Option<Vec<model::mention::Model>>,
 }
 
 impl FromQueryResult for RichActivity {
@@ -96,7 +93,7 @@ impl FromQueryResult for RichActivity {
 			activity: model::activity::Model::from_query_result(res, model::activity::Entity.table_name())?,
 			object: model::object::Model::from_query_result(res, model::object::Entity.table_name()).ok(),
 			liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(),
-			attachments: None,
+			attachments: None, hashtags: None, mentions: None,
 		})
 	}
 }
@@ -105,16 +102,42 @@ impl RichActivity {
 	pub fn ap(self) -> serde_json::Value {
 		let object = match self.object {
 			None => apb::Node::maybe_link(self.activity.object.clone()),
-			Some(o) => apb::Node::object(
+			Some(o) => {
+				// TODO can we avoid repeating this tags code?
+				let mut tags = Vec::new();
+				if let Some(mentions) = self.mentions {
+					for mention in mentions {
+						tags.push(
+							apb::new()
+								.set_link_type(Some(apb::LinkType::Mention))
+								.set_href(&mention.actor)
+								// TODO do i need to set name? i could join while batch loading or put the @name in
+								// each mention object...
+						);
+					}
+				}
+				if let Some(hashtags) = self.hashtags {
+					for hash in hashtags {
+						tags.push(
+							// TODO ewwww set_name clash and cant use builder, wtf is this
+							LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name)))
+								.set_link_type(Some(apb::LinkType::Hashtag))
+								// TODO do we need to set href too? we can't access context here, quite an issue!
+						);
+					}
+				}
+				apb::Node::object(
 					o.ap()
 						.set_liked_by_me(if self.liked.is_some() { Some(true) } else { None })
+						.set_tag(apb::Node::array(tags))
 						.set_attachment(match self.attachments {
 							None => apb::Node::Empty,
 							Some(vec) => apb::Node::array(
 								vec.into_iter().map(|x| x.ap()).collect()
 							),
 						})
-			),
+				)
+			},
 		};
 		self.activity.ap().set_object(object)
 	}
@@ -124,6 +147,8 @@ pub struct RichObject {
 	pub object: model::object::Model,
 	pub liked: Option<i64>,
 	pub attachments: Option<Vec<model::attachment::Model>>,
+	pub hashtags: Option<Vec<model::hashtag::Model>>,
+	pub mentions: Option<Vec<model::mention::Model>>,
 }
 
 impl FromQueryResult for RichObject {
@@ -131,15 +156,39 @@ impl FromQueryResult for RichObject {
 		Ok(RichObject {
 			object: model::object::Model::from_query_result(res, model::object::Entity.table_name())?,
 			liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(),
-			attachments: None,
+			attachments: None, hashtags: None, mentions: None,
 		})
 	}
 }
 
 impl RichObject {
 	pub fn ap(self) -> serde_json::Value {
+		// TODO can we avoid repeating this tags code?
+		let mut tags = Vec::new();
+		if let Some(mentions) = self.mentions {
+			for mention in mentions {
+				tags.push(
+					apb::new()
+						.set_link_type(Some(apb::LinkType::Mention))
+						.set_href(&mention.actor)
+						// TODO do i need to set name? i could join while batch loading or put the @name in
+						// each mention object...
+				);
+			}
+		}
+		if let Some(hashtags) = self.hashtags {
+			for hash in hashtags {
+				tags.push(
+					// TODO ewwww set_name clash and cant use builder, wtf is this
+					LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name)))
+						.set_link_type(Some(apb::LinkType::Hashtag))
+						// TODO do we need to set href too? we can't access context here, quite an issue!
+				);
+			}
+		}
 		self.object.ap()
 			.set_liked_by_me(if self.liked.is_some() { Some(true) } else { None })
+			.set_tag(apb::Node::array(tags))
 			.set_attachment(match self.attachments {
 				None => apb::Node::Empty,
 				Some(vec) => apb::Node::array(
@@ -151,34 +200,43 @@ impl RichObject {
 
 #[async_trait::async_trait]
 pub trait BatchFillable: Sized {
-	async fn with_attachments(self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>;
+	async fn with_batched<E>(self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
+	where
+		E: BatchFillableComparison + EntityTrait,
+		E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
+		RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
+		RichObject: BatchFillableAcceptor<Vec<E::Model>>;
 }
 
 #[async_trait::async_trait]
 impl BatchFillable for Vec<RichActivity> {
 	// TODO 3 iterations... can we make it in less passes?
-	async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
+	async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
-		let objects : Vec<model::object::Model> = self
+	where
-			.iter()
+		E: BatchFillableComparison + EntityTrait,
-			.filter_map(|x| x.object.as_ref().cloned())
+		E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
-			.collect();
+		RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
-
+		RichObject: BatchFillableAcceptor<Vec<E::Model>>
-		let attachments = objects.load_many(model::attachment::Entity, tx).await?;
+	{
-
+		let ids : Vec<i64> = self.iter().filter_map(|x| Some(x.object.as_ref()?.internal)).collect();
-		let mut out : std::collections::BTreeMap<i64, Vec<model::attachment::Model>> = std::collections::BTreeMap::new();
+		let batch = E::find()
-		for attach in attachments.into_iter().flatten() {
+			.filter(E::comparison(ids))
-			match out.entry(attach.object) {
+			.all(tx)
-				std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); },
+			.await?;
-				std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); },
+		let mut map : HashMap<i64, Vec<E::Model>> = HashMap::new();
+		for element in batch {
+			match map.entry(element.key()) {
+				Entry::Occupied(mut x) => { x.get_mut().push(element); },
+				Entry::Vacant(x) => { x.insert(vec![element]); },
 			}
 		}
-
+		for element in self.iter_mut() {
-		for activity in self.iter_mut() {
+			if let Some(ref object) = element.object {
-			if let Some(ref object) = activity.object {
+				if let Some(v) = map.remove(&object.internal) {
-				activity.attachments = out.remove(&object.internal);
+					element.accept(v);
+				}
 			}
 		}
-
 		Ok(self)
 	}
 }
@@ -186,73 +244,159 @@ impl BatchFillable for Vec<RichActivity> {
 #[async_trait::async_trait]
 impl BatchFillable for Vec<RichObject> {
 	// TODO 3 iterations... can we make it in less passes?
-	async fn with_attachments(mut self, db: &impl ConnectionTrait) -> Result<Self, DbErr> {
+	async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
-		let objects : Vec<model::object::Model> = self
+	where
-			.iter()
+		E: BatchFillableComparison + EntityTrait,
-			.map(|o| o.object.clone())
+		E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
-			.collect();
+		RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
-
+		RichObject: BatchFillableAcceptor<Vec<E::Model>>
-		let attachments = objects.load_many(model::attachment::Entity, db).await?;
+	{
-
+		let ids : Vec<i64> = self.iter().map(|x| x.object.internal).collect();
-		let mut out : std::collections::BTreeMap<i64, Vec<model::attachment::Model>> = std::collections::BTreeMap::new();
+		let batch = E::find()
-		for attach in attachments.into_iter().flatten() {
+			.filter(E::comparison(ids))
-			match out.entry(attach.object) {
+			.all(tx)
-				std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); },
+			.await?;
-				std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); },
+		let mut map : HashMap<i64, Vec<E::Model>> = HashMap::new();
+		for element in batch {
+			match map.entry(element.key()) {
+				Entry::Occupied(mut x) => { x.get_mut().push(element); },
+				Entry::Vacant(x) => { x.insert(vec![element]); },
 			}
 		}
-
+		for element in self.iter_mut() {
-		for obj in self.iter_mut() {
+			if let Some(v) = map.remove(&element.object.internal) {
-			obj.attachments = out.remove(&obj.object.internal);
+				element.accept(v);
+			}
 		}
-
 		Ok(self)
 	}
 }
 
 #[async_trait::async_trait]
 impl BatchFillable for RichActivity {
-	async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
+	async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
+	where
+		E: BatchFillableComparison + EntityTrait,
+		E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
+		RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
+		RichObject: BatchFillableAcceptor<Vec<E::Model>>
+	{
 		if let Some(ref obj) = self.object {
-			self.attachments = Some(
+			let batch =E::find()
-				obj.find_related(model::attachment::Entity)
+				.filter(E::comparison(vec![obj.internal]))
 				.all(tx)
-					.await?
+				.await?;
-			);
+			self.accept(batch);
 		}
-
 		Ok(self)
 	}
 }
 
 #[async_trait::async_trait]
 impl BatchFillable for RichObject {
-	async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
+	async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
-		self.attachments = Some(
+	where
-			self.object.find_related(model::attachment::Entity)
+		E: BatchFillableComparison + EntityTrait,
+		E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
+		RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
+		RichObject: BatchFillableAcceptor<Vec<E::Model>>
+	{
+		let batch = E::find()
+			.filter(E::comparison(vec![self.object.internal]))
 			.all(tx)
-				.await?
+			.await?;
-		);
+		self.accept(batch);
-
 		Ok(self)
 	}
 }
 
-#[cfg(test)]
-mod test {
-	use sea_orm::{ColumnTrait, Condition, QueryFilter, QueryTrait};
 
-	#[test]
+// welcome to interlocking trait hell, enjoy your stay
-	fn wtf_postgres() {
+mod hell {
-		panic!("{}", crate::Query::objects(None)
+	use sea_orm::{sea_query::IntoCondition, ColumnTrait};
-			.filter(
+
-				Condition::any()
+	pub trait BatchFillableComparison {
-					.add(crate::model::addressing::Column::Actor.is_null())
+		fn comparison(ids: Vec<i64>) -> sea_orm::Condition;
-					.add(crate::model::addressing::Column::Actor.eq(2))
+	}
-					.add(crate::model::object::Column::AttributedTo.eq("https://upub.alemi.dev/actors/alemi"))
+
-			)
+	impl BatchFillableComparison for crate::model::attachment::Entity {
-			.build(sea_orm::DatabaseBackend::Postgres)
+		fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
-			.to_string()
+			crate::model::attachment::Column::Object.is_in(ids).into_condition()
-		)
+		}
+	}
+
+	impl BatchFillableComparison for crate::model::mention::Entity {
+		fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
+			crate::model::mention::Column::Object.is_in(ids).into_condition()
+		}
+	}
+
+	impl BatchFillableComparison for crate::model::hashtag::Entity {
+		fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
+			crate::model::hashtag::Column::Object.is_in(ids).into_condition()
+		}
+	}
+	
+	pub trait BatchFillableKey {
+		fn key(&self) -> i64;
+	}
+
+	impl BatchFillableKey for crate::model::attachment::Model {
+		fn key(&self) -> i64 {
+			self.internal
+		}
+	}
+
+	impl BatchFillableKey for crate::model::mention::Model {
+		fn key(&self) -> i64 {
+			self.internal
+		}
+	}
+
+	impl BatchFillableKey for crate::model::hashtag::Model {
+		fn key(&self) -> i64 {
+			self.internal
+		}
+	}
+	
+	pub trait BatchFillableAcceptor<B> {
+		fn accept(&mut self, batch: B);
+	}
+	
+	impl BatchFillableAcceptor<Vec<crate::model::attachment::Model>> for super::RichActivity {
+		fn accept(&mut self, batch: Vec<crate::model::attachment::Model>) {
+			self.attachments = Some(batch);
+		}
+	}
+	
+	impl BatchFillableAcceptor<Vec<crate::model::hashtag::Model>> for super::RichActivity {
+		fn accept(&mut self, batch: Vec<crate::model::hashtag::Model>) {
+			self.hashtags = Some(batch);
+		}
+	}
+	
+	impl BatchFillableAcceptor<Vec<crate::model::mention::Model>> for super::RichActivity {
+		fn accept(&mut self, batch: Vec<crate::model::mention::Model>) {
+			self.mentions = Some(batch);
+		}
+	}
+
+	impl BatchFillableAcceptor<Vec<crate::model::attachment::Model>> for super::RichObject {
+		fn accept(&mut self, batch: Vec<crate::model::attachment::Model>) {
+			self.attachments = Some(batch);
+		}
+	}
+	
+	impl BatchFillableAcceptor<Vec<crate::model::hashtag::Model>> for super::RichObject {
+		fn accept(&mut self, batch: Vec<crate::model::hashtag::Model>) {
+			self.hashtags = Some(batch);
+		}
+	}
+	
+	impl BatchFillableAcceptor<Vec<crate::model::mention::Model>> for super::RichObject {
+		fn accept(&mut self, batch: Vec<crate::model::mention::Model>) {
+			self.mentions = Some(batch);
+		}
 	}
 }
+use hell::*;

upub/core/src/traits/normalize.rs

@@ -1,4 +1,4 @@
-use apb::{field::OptionalString, Collection, Document, Endpoints, Node, Object, PublicKey};
+use apb::{field::OptionalString, Link, Collection, Document, Endpoints, Node, Object, PublicKey};
 use sea_orm::{sea_query::Expr, ActiveModelTrait, ActiveValue::{Unchanged, NotSet, Set}, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, QueryFilter};
 
 use super::Addresser;
@@ -25,6 +25,7 @@ pub trait Normalizer {
 impl Normalizer for crate::Context {
 
 	async fn insert_object(&self, object: impl apb::Object, tx: &impl ConnectionTrait) -> Result<crate::model::object::Model, NormalizerError> {
+		let now = chrono::Utc::now();
 		let mut object_model = AP::object(&object)?;
 
 		// make sure content only contains a safe subset of html
@@ -86,7 +87,7 @@ impl Normalizer for crate::Context {
 					document_type: Set(apb::DocumentType::Page),
 					name: Set(l.name().str()),
 					media_type: Set(l.media_type().unwrap_or("link").to_string()),
-					published: Set(chrono::Utc::now()),
+					published: Set(now),
 				},
 				Node::Object(o) =>
 					AP::attachment_q(o.as_document()?, object_model.internal, None)?,
@@ -123,6 +124,37 @@ impl Normalizer for crate::Context {
 				.await?;
 		}
 
+		for tag in object.tag() {
+			match tag.link_type() {
+				Ok(apb::LinkType::Mention) => {
+					let model = crate::model::mention::ActiveModel {
+						internal: NotSet,
+						object: Set(object_model.internal),
+						actor: Set(tag.href().to_string()),
+						published: Set(now),
+					};
+					crate::model::mention::Entity::insert(model)
+						.exec(tx)
+						.await?;
+				},
+				Ok(apb::LinkType::Hashtag) => {
+					let hashtag = tag.name()
+						.unwrap_or_else(|_| tag.href().split('/').last().unwrap_or_default())
+						.replace('#', "");
+					let model = crate::model::hashtag::ActiveModel {
+						internal: NotSet,
+						object: Set(object_model.internal),
+						name: Set(hashtag),
+						published: Set(now),
+					};
+					crate::model::hashtag::Entity::insert(model)
+						.exec(tx)
+						.await?;
+				},
+				_ => {},
+			}
+		}
+
 		Ok(object_model)
 	}
 

upub/routes/src/activitypub/activity.rs

@@ -30,7 +30,11 @@ pub async fn view(
 		.one(ctx.db())
 		.await?
 		.ok_or_else(crate::ApiError::not_found)?
-		.with_attachments(ctx.db())
+		.with_batched::<upub::model::attachment::Entity>(ctx.db())
+		.await?
+		.with_batched::<upub::model::mention::Entity>(ctx.db())
+		.await?
+		.with_batched::<upub::model::hashtag::Entity>(ctx.db())
 		.await?;
 
 	Ok(JsonLD(row.ap().ld_context()))

upub/routes/src/activitypub/object/mod.rs

@@ -3,8 +3,8 @@ pub mod context;
 
 use apb::{BaseMut, CollectionMut, ObjectMut, LD};
 use axum::extract::{Path, Query, State};
-use sea_orm::{ColumnTrait, ModelTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait};
+use sea_orm::{ColumnTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait};
-use upub::{model, selector::RichObject, traits::Fetcher, Context};
+use upub::{model, selector::{BatchFillable, RichObject}, traits::Fetcher, Context};
 
 use crate::{builders::JsonLD, AuthIdentity};
 
@@ -33,14 +33,14 @@ pub async fn view(
 		.into_model::<RichObject>()
 		.one(ctx.db())
 		.await?
-		.ok_or_else(crate::ApiError::not_found)?;
+		.ok_or_else(crate::ApiError::not_found)?
-
+		.with_batched::<upub::model::attachment::Entity>(ctx.db())
-	let attachments = item.object.find_related(model::attachment::Entity)
-		.all(ctx.db())
 		.await?
-		.into_iter()
+		.with_batched::<upub::model::mention::Entity>(ctx.db())
-		.map(|x| x.ap())
+		.await?
-		.collect::<Vec<serde_json::Value>>();
+		.with_batched::<upub::model::hashtag::Entity>(ctx.db())
+		.await?;
+
 
 	let mut replies = apb::Node::Empty;
 	
@@ -66,7 +66,6 @@ pub async fn view(
 	
 	Ok(JsonLD(
 		item.ap()
-			.set_attachment(apb::Node::array(attachments))
 			.set_replies(replies)
 			.ld_context()
 	))

upub/routes/src/builders.rs

@@ -31,7 +31,11 @@ pub async fn paginate_activities(
 		.into_model::<RichActivity>()
 		.all(db)
 		.await?
-		.with_attachments(db)
+		.with_batched::<upub::model::attachment::Entity>(db)
+		.await?
+		.with_batched::<upub::model::mention::Entity>(db)
+		.await?
+		.with_batched::<upub::model::hashtag::Entity>(db)
 		.await?;
 
 	let items : Vec<serde_json::Value> = items
@@ -73,7 +77,11 @@ pub async fn paginate_objects(
 		.into_model::<RichObject>() // <--- difference three
 		.all(db)
 		.await?
-		.with_attachments(db)
+		.with_batched::<upub::model::attachment::Entity>(db)
+		.await?
+		.with_batched::<upub::model::mention::Entity>(db)
+		.await?
+		.with_batched::<upub::model::hashtag::Entity>(db)
 		.await?;
 
 	let items : Vec<serde_json::Value> = items