diff --git a/upub/core/src/selector.rs b/upub/core/src/selector.rs index eed2be8..fed618c 100644 --- a/upub/core/src/selector.rs +++ b/upub/core/src/selector.rs @@ -1,5 +1,7 @@ -use apb::{ActivityMut, ObjectMut}; -use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, LoaderTrait, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns}; +use std::collections::{hash_map::Entry, HashMap}; + +use apb::{ActivityMut, LinkMut, ObjectMut}; +use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns}; use crate::model; @@ -76,18 +78,13 @@ impl Query { -#[derive(Debug, Clone)] -pub struct EmbeddedActivity { - pub activity: model::activity::Model, - pub object: model::object::Model, - pub liked: Option, -} - pub struct RichActivity { pub activity: model::activity::Model, pub object: Option, pub liked: Option, pub attachments: Option>, + pub hashtags: Option>, + pub mentions: Option>, } impl FromQueryResult for RichActivity { @@ -96,7 +93,7 @@ impl FromQueryResult for RichActivity { activity: model::activity::Model::from_query_result(res, model::activity::Entity.table_name())?, object: model::object::Model::from_query_result(res, model::object::Entity.table_name()).ok(), liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(), - attachments: None, + attachments: None, hashtags: None, mentions: None, }) } } @@ -105,16 +102,42 @@ impl RichActivity { pub fn ap(self) -> serde_json::Value { let object = match self.object { None => apb::Node::maybe_link(self.activity.object.clone()), - Some(o) => apb::Node::object( - o.ap() - .set_liked_by_me(if self.liked.is_some() { Some(true) } else { None }) - .set_attachment(match self.attachments { - None => apb::Node::Empty, - Some(vec) => apb::Node::array( - vec.into_iter().map(|x| x.ap()).collect() - ), - }) - ), + Some(o) => { + // TODO can we avoid repeating this tags code? + let mut tags = Vec::new(); + if let Some(mentions) = self.mentions { + for mention in mentions { + tags.push( + apb::new() + .set_link_type(Some(apb::LinkType::Mention)) + .set_href(&mention.actor) + // TODO do i need to set name? i could join while batch loading or put the @name in + // each mention object... + ); + } + } + if let Some(hashtags) = self.hashtags { + for hash in hashtags { + tags.push( + // TODO ewwww set_name clash and cant use builder, wtf is this + LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name))) + .set_link_type(Some(apb::LinkType::Hashtag)) + // TODO do we need to set href too? we can't access context here, quite an issue! + ); + } + } + apb::Node::object( + o.ap() + .set_liked_by_me(if self.liked.is_some() { Some(true) } else { None }) + .set_tag(apb::Node::array(tags)) + .set_attachment(match self.attachments { + None => apb::Node::Empty, + Some(vec) => apb::Node::array( + vec.into_iter().map(|x| x.ap()).collect() + ), + }) + ) + }, }; self.activity.ap().set_object(object) } @@ -124,6 +147,8 @@ pub struct RichObject { pub object: model::object::Model, pub liked: Option, pub attachments: Option>, + pub hashtags: Option>, + pub mentions: Option>, } impl FromQueryResult for RichObject { @@ -131,15 +156,39 @@ impl FromQueryResult for RichObject { Ok(RichObject { object: model::object::Model::from_query_result(res, model::object::Entity.table_name())?, liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(), - attachments: None, + attachments: None, hashtags: None, mentions: None, }) } } impl RichObject { pub fn ap(self) -> serde_json::Value { + // TODO can we avoid repeating this tags code? + let mut tags = Vec::new(); + if let Some(mentions) = self.mentions { + for mention in mentions { + tags.push( + apb::new() + .set_link_type(Some(apb::LinkType::Mention)) + .set_href(&mention.actor) + // TODO do i need to set name? i could join while batch loading or put the @name in + // each mention object... + ); + } + } + if let Some(hashtags) = self.hashtags { + for hash in hashtags { + tags.push( + // TODO ewwww set_name clash and cant use builder, wtf is this + LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name))) + .set_link_type(Some(apb::LinkType::Hashtag)) + // TODO do we need to set href too? we can't access context here, quite an issue! + ); + } + } self.object.ap() .set_liked_by_me(if self.liked.is_some() { Some(true) } else { None }) + .set_tag(apb::Node::array(tags)) .set_attachment(match self.attachments { None => apb::Node::Empty, Some(vec) => apb::Node::array( @@ -151,34 +200,43 @@ impl RichObject { #[async_trait::async_trait] pub trait BatchFillable: Sized { - async fn with_attachments(self, tx: &impl ConnectionTrait) -> Result; + async fn with_batched(self, tx: &impl ConnectionTrait) -> Result + where + E: BatchFillableComparison + EntityTrait, + E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait, + RichActivity: BatchFillableAcceptor>, + RichObject: BatchFillableAcceptor>; } #[async_trait::async_trait] impl BatchFillable for Vec { // TODO 3 iterations... can we make it in less passes? - async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result { - let objects : Vec = self - .iter() - .filter_map(|x| x.object.as_ref().cloned()) - .collect(); - - let attachments = objects.load_many(model::attachment::Entity, tx).await?; - - let mut out : std::collections::BTreeMap> = std::collections::BTreeMap::new(); - for attach in attachments.into_iter().flatten() { - match out.entry(attach.object) { - std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); }, - std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); }, + async fn with_batched(mut self, tx: &impl ConnectionTrait) -> Result + where + E: BatchFillableComparison + EntityTrait, + E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait, + RichActivity: BatchFillableAcceptor>, + RichObject: BatchFillableAcceptor> + { + let ids : Vec = self.iter().filter_map(|x| Some(x.object.as_ref()?.internal)).collect(); + let batch = E::find() + .filter(E::comparison(ids)) + .all(tx) + .await?; + let mut map : HashMap> = HashMap::new(); + for element in batch { + match map.entry(element.key()) { + Entry::Occupied(mut x) => { x.get_mut().push(element); }, + Entry::Vacant(x) => { x.insert(vec![element]); }, } } - - for activity in self.iter_mut() { - if let Some(ref object) = activity.object { - activity.attachments = out.remove(&object.internal); + for element in self.iter_mut() { + if let Some(ref object) = element.object { + if let Some(v) = map.remove(&object.internal) { + element.accept(v); + } } } - Ok(self) } } @@ -186,73 +244,159 @@ impl BatchFillable for Vec { #[async_trait::async_trait] impl BatchFillable for Vec { // TODO 3 iterations... can we make it in less passes? - async fn with_attachments(mut self, db: &impl ConnectionTrait) -> Result { - let objects : Vec = self - .iter() - .map(|o| o.object.clone()) - .collect(); - - let attachments = objects.load_many(model::attachment::Entity, db).await?; - - let mut out : std::collections::BTreeMap> = std::collections::BTreeMap::new(); - for attach in attachments.into_iter().flatten() { - match out.entry(attach.object) { - std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); }, - std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); }, + async fn with_batched(mut self, tx: &impl ConnectionTrait) -> Result + where + E: BatchFillableComparison + EntityTrait, + E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait, + RichActivity: BatchFillableAcceptor>, + RichObject: BatchFillableAcceptor> + { + let ids : Vec = self.iter().map(|x| x.object.internal).collect(); + let batch = E::find() + .filter(E::comparison(ids)) + .all(tx) + .await?; + let mut map : HashMap> = HashMap::new(); + for element in batch { + match map.entry(element.key()) { + Entry::Occupied(mut x) => { x.get_mut().push(element); }, + Entry::Vacant(x) => { x.insert(vec![element]); }, } } - - for obj in self.iter_mut() { - obj.attachments = out.remove(&obj.object.internal); + for element in self.iter_mut() { + if let Some(v) = map.remove(&element.object.internal) { + element.accept(v); + } } - Ok(self) } } #[async_trait::async_trait] impl BatchFillable for RichActivity { - async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result { + async fn with_batched(mut self, tx: &impl ConnectionTrait) -> Result + where + E: BatchFillableComparison + EntityTrait, + E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait, + RichActivity: BatchFillableAcceptor>, + RichObject: BatchFillableAcceptor> + { if let Some(ref obj) = self.object { - self.attachments = Some( - obj.find_related(model::attachment::Entity) - .all(tx) - .await? - ); + let batch =E::find() + .filter(E::comparison(vec![obj.internal])) + .all(tx) + .await?; + self.accept(batch); } - Ok(self) } } #[async_trait::async_trait] impl BatchFillable for RichObject { - async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result { - self.attachments = Some( - self.object.find_related(model::attachment::Entity) - .all(tx) - .await? - ); - + async fn with_batched(mut self, tx: &impl ConnectionTrait) -> Result + where + E: BatchFillableComparison + EntityTrait, + E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait, + RichActivity: BatchFillableAcceptor>, + RichObject: BatchFillableAcceptor> + { + let batch = E::find() + .filter(E::comparison(vec![self.object.internal])) + .all(tx) + .await?; + self.accept(batch); Ok(self) } } -#[cfg(test)] -mod test { - use sea_orm::{ColumnTrait, Condition, QueryFilter, QueryTrait}; - #[test] - fn wtf_postgres() { - panic!("{}", crate::Query::objects(None) - .filter( - Condition::any() - .add(crate::model::addressing::Column::Actor.is_null()) - .add(crate::model::addressing::Column::Actor.eq(2)) - .add(crate::model::object::Column::AttributedTo.eq("https://upub.alemi.dev/actors/alemi")) - ) - .build(sea_orm::DatabaseBackend::Postgres) - .to_string() - ) +// welcome to interlocking trait hell, enjoy your stay +mod hell { + use sea_orm::{sea_query::IntoCondition, ColumnTrait}; + + pub trait BatchFillableComparison { + fn comparison(ids: Vec) -> sea_orm::Condition; + } + + impl BatchFillableComparison for crate::model::attachment::Entity { + fn comparison(ids: Vec) -> sea_orm::Condition { + crate::model::attachment::Column::Object.is_in(ids).into_condition() + } + } + + impl BatchFillableComparison for crate::model::mention::Entity { + fn comparison(ids: Vec) -> sea_orm::Condition { + crate::model::mention::Column::Object.is_in(ids).into_condition() + } + } + + impl BatchFillableComparison for crate::model::hashtag::Entity { + fn comparison(ids: Vec) -> sea_orm::Condition { + crate::model::hashtag::Column::Object.is_in(ids).into_condition() + } + } + + pub trait BatchFillableKey { + fn key(&self) -> i64; + } + + impl BatchFillableKey for crate::model::attachment::Model { + fn key(&self) -> i64 { + self.internal + } + } + + impl BatchFillableKey for crate::model::mention::Model { + fn key(&self) -> i64 { + self.internal + } + } + + impl BatchFillableKey for crate::model::hashtag::Model { + fn key(&self) -> i64 { + self.internal + } + } + + pub trait BatchFillableAcceptor { + fn accept(&mut self, batch: B); + } + + impl BatchFillableAcceptor> for super::RichActivity { + fn accept(&mut self, batch: Vec) { + self.attachments = Some(batch); + } + } + + impl BatchFillableAcceptor> for super::RichActivity { + fn accept(&mut self, batch: Vec) { + self.hashtags = Some(batch); + } + } + + impl BatchFillableAcceptor> for super::RichActivity { + fn accept(&mut self, batch: Vec) { + self.mentions = Some(batch); + } + } + + impl BatchFillableAcceptor> for super::RichObject { + fn accept(&mut self, batch: Vec) { + self.attachments = Some(batch); + } + } + + impl BatchFillableAcceptor> for super::RichObject { + fn accept(&mut self, batch: Vec) { + self.hashtags = Some(batch); + } + } + + impl BatchFillableAcceptor> for super::RichObject { + fn accept(&mut self, batch: Vec) { + self.mentions = Some(batch); + } } } +use hell::*; diff --git a/upub/core/src/traits/normalize.rs b/upub/core/src/traits/normalize.rs index 8238eab..1fc9edd 100644 --- a/upub/core/src/traits/normalize.rs +++ b/upub/core/src/traits/normalize.rs @@ -1,4 +1,4 @@ -use apb::{field::OptionalString, Collection, Document, Endpoints, Node, Object, PublicKey}; +use apb::{field::OptionalString, Link, Collection, Document, Endpoints, Node, Object, PublicKey}; use sea_orm::{sea_query::Expr, ActiveModelTrait, ActiveValue::{Unchanged, NotSet, Set}, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, QueryFilter}; use super::Addresser; @@ -25,6 +25,7 @@ pub trait Normalizer { impl Normalizer for crate::Context { async fn insert_object(&self, object: impl apb::Object, tx: &impl ConnectionTrait) -> Result { + let now = chrono::Utc::now(); let mut object_model = AP::object(&object)?; // make sure content only contains a safe subset of html @@ -86,7 +87,7 @@ impl Normalizer for crate::Context { document_type: Set(apb::DocumentType::Page), name: Set(l.name().str()), media_type: Set(l.media_type().unwrap_or("link").to_string()), - published: Set(chrono::Utc::now()), + published: Set(now), }, Node::Object(o) => AP::attachment_q(o.as_document()?, object_model.internal, None)?, @@ -123,6 +124,37 @@ impl Normalizer for crate::Context { .await?; } + for tag in object.tag() { + match tag.link_type() { + Ok(apb::LinkType::Mention) => { + let model = crate::model::mention::ActiveModel { + internal: NotSet, + object: Set(object_model.internal), + actor: Set(tag.href().to_string()), + published: Set(now), + }; + crate::model::mention::Entity::insert(model) + .exec(tx) + .await?; + }, + Ok(apb::LinkType::Hashtag) => { + let hashtag = tag.name() + .unwrap_or_else(|_| tag.href().split('/').last().unwrap_or_default()) + .replace('#', ""); + let model = crate::model::hashtag::ActiveModel { + internal: NotSet, + object: Set(object_model.internal), + name: Set(hashtag), + published: Set(now), + }; + crate::model::hashtag::Entity::insert(model) + .exec(tx) + .await?; + }, + _ => {}, + } + } + Ok(object_model) } diff --git a/upub/routes/src/activitypub/activity.rs b/upub/routes/src/activitypub/activity.rs index 66f29b6..af6d003 100644 --- a/upub/routes/src/activitypub/activity.rs +++ b/upub/routes/src/activitypub/activity.rs @@ -30,7 +30,11 @@ pub async fn view( .one(ctx.db()) .await? .ok_or_else(crate::ApiError::not_found)? - .with_attachments(ctx.db()) + .with_batched::(ctx.db()) + .await? + .with_batched::(ctx.db()) + .await? + .with_batched::(ctx.db()) .await?; Ok(JsonLD(row.ap().ld_context())) diff --git a/upub/routes/src/activitypub/object/mod.rs b/upub/routes/src/activitypub/object/mod.rs index 406e6d1..1770ff9 100644 --- a/upub/routes/src/activitypub/object/mod.rs +++ b/upub/routes/src/activitypub/object/mod.rs @@ -3,8 +3,8 @@ pub mod context; use apb::{BaseMut, CollectionMut, ObjectMut, LD}; use axum::extract::{Path, Query, State}; -use sea_orm::{ColumnTrait, ModelTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait}; -use upub::{model, selector::RichObject, traits::Fetcher, Context}; +use sea_orm::{ColumnTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait}; +use upub::{model, selector::{BatchFillable, RichObject}, traits::Fetcher, Context}; use crate::{builders::JsonLD, AuthIdentity}; @@ -33,14 +33,14 @@ pub async fn view( .into_model::() .one(ctx.db()) .await? - .ok_or_else(crate::ApiError::not_found)?; - - let attachments = item.object.find_related(model::attachment::Entity) - .all(ctx.db()) + .ok_or_else(crate::ApiError::not_found)? + .with_batched::(ctx.db()) .await? - .into_iter() - .map(|x| x.ap()) - .collect::>(); + .with_batched::(ctx.db()) + .await? + .with_batched::(ctx.db()) + .await?; + let mut replies = apb::Node::Empty; @@ -66,7 +66,6 @@ pub async fn view( Ok(JsonLD( item.ap() - .set_attachment(apb::Node::array(attachments)) .set_replies(replies) .ld_context() )) diff --git a/upub/routes/src/builders.rs b/upub/routes/src/builders.rs index 35e96e0..47d73cc 100644 --- a/upub/routes/src/builders.rs +++ b/upub/routes/src/builders.rs @@ -31,7 +31,11 @@ pub async fn paginate_activities( .into_model::() .all(db) .await? - .with_attachments(db) + .with_batched::(db) + .await? + .with_batched::(db) + .await? + .with_batched::(db) .await?; let items : Vec = items @@ -73,7 +77,11 @@ pub async fn paginate_objects( .into_model::() // <--- difference three .all(db) .await? - .with_attachments(db) + .with_batched::(db) + .await? + .with_batched::(db) + .await? + .with_batched::(db) .await?; let items : Vec = items