alemi/upub
alemi/upub
1
0
Fork 1
Code Issues 4 Pull requests Projects Releases Packages 1 Wiki Activity Actions

feat: process, store and fill obj hashtags/mentions

Browse source
This commit is contained in:
əlemi 2024-06-22 03:40:17 +02:00
parent e7e1a926c1
commit b80eb03373
Signed by: alemi
GPG key ID: A4895B84D311642C
5 changed files with 288 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

316
upub/core/src/selector.rs
View file

@ -1,5 +1,7 @@
use apb::{ActivityMut, ObjectMut};
use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, LoaderTrait, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns};
use std::collections::{hash_map::Entry, HashMap};
use apb::{ActivityMut, LinkMut, ObjectMut};
use sea_orm::{sea_query::{IntoColumnRef, IntoCondition}, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityName, EntityTrait, FromQueryResult, Iden, Iterable, ModelTrait, Order, QueryFilter, QueryOrder, QueryResult, QuerySelect, RelationTrait, Select, SelectColumns};
use crate::model;
@ -76,18 +78,13 @@ impl Query {
#[derive(Debug, Clone)]
pub struct EmbeddedActivity {
pub activity: model::activity::Model,
pub object: model::object::Model,
pub liked: Option<i64>,
}
pub struct RichActivity {
pub activity: model::activity::Model,
pub object: Option<model::object::Model>,
pub liked: Option<i64>,
pub attachments: Option<Vec<model::attachment::Model>>,
pub hashtags: Option<Vec<model::hashtag::Model>>,
pub mentions: Option<Vec<model::mention::Model>>,
}
impl FromQueryResult for RichActivity {
@ -96,7 +93,7 @@ impl FromQueryResult for RichActivity {
activity: model::activity::Model::from_query_result(res, model::activity::Entity.table_name())?,
object: model::object::Model::from_query_result(res, model::object::Entity.table_name()).ok(),
liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(),
attachments: None,
attachments: None, hashtags: None, mentions: None,
})
}
}
@ -105,16 +102,42 @@ impl RichActivity {
pub fn ap(self) -> serde_json::Value {
let object = match self.object {
None => apb::Node::maybe_link(self.activity.object.clone()),
Some(o) => apb::Node::object(
o.ap()
.set_liked_by_me(if self.liked.is_some() { Some(true) } else { None })
.set_attachment(match self.attachments {
None => apb::Node::Empty,
Some(vec) => apb::Node::array(
vec.into_iter().map(|x| x.ap()).collect()
),
})
),
Some(o) => {
// TODO can we avoid repeating this tags code?
let mut tags = Vec::new();
if let Some(mentions) = self.mentions {
for mention in mentions {
tags.push(
apb::new()
.set_link_type(Some(apb::LinkType::Mention))
.set_href(&mention.actor)
// TODO do i need to set name? i could join while batch loading or put the @name in
// each mention object...
);
}
}
if let Some(hashtags) = self.hashtags {
for hash in hashtags {
tags.push(
// TODO ewwww set_name clash and cant use builder, wtf is this
LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name)))
.set_link_type(Some(apb::LinkType::Hashtag))
// TODO do we need to set href too? we can't access context here, quite an issue!
);
}
}
apb::Node::object(
o.ap()
.set_liked_by_me(if self.liked.is_some() { Some(true) } else { None })
.set_tag(apb::Node::array(tags))
.set_attachment(match self.attachments {
None => apb::Node::Empty,
Some(vec) => apb::Node::array(
vec.into_iter().map(|x| x.ap()).collect()
),
})
)
},
};
self.activity.ap().set_object(object)
}
@ -124,6 +147,8 @@ pub struct RichObject {
pub object: model::object::Model,
pub liked: Option<i64>,
pub attachments: Option<Vec<model::attachment::Model>>,
pub hashtags: Option<Vec<model::hashtag::Model>>,
pub mentions: Option<Vec<model::mention::Model>>,
}
impl FromQueryResult for RichObject {
@ -131,15 +156,39 @@ impl FromQueryResult for RichObject {
Ok(RichObject {
object: model::object::Model::from_query_result(res, model::object::Entity.table_name())?,
liked: res.try_get(model::like::Entity.table_name(), &model::like::Column::Actor.to_string()).ok(),
attachments: None,
attachments: None, hashtags: None, mentions: None,
})
}
}
impl RichObject {
pub fn ap(self) -> serde_json::Value {
// TODO can we avoid repeating this tags code?
let mut tags = Vec::new();
if let Some(mentions) = self.mentions {
for mention in mentions {
tags.push(
apb::new()
.set_link_type(Some(apb::LinkType::Mention))
.set_href(&mention.actor)
// TODO do i need to set name? i could join while batch loading or put the @name in
// each mention object...
);
}
}
if let Some(hashtags) = self.hashtags {
for hash in hashtags {
tags.push(
// TODO ewwww set_name clash and cant use builder, wtf is this
LinkMut::set_name(apb::new(), Some(&format!("#{}", hash.name)))
.set_link_type(Some(apb::LinkType::Hashtag))
// TODO do we need to set href too? we can't access context here, quite an issue!
);
}
}
self.object.ap()
.set_liked_by_me(if self.liked.is_some() { Some(true) } else { None })
.set_tag(apb::Node::array(tags))
.set_attachment(match self.attachments {
None => apb::Node::Empty,
Some(vec) => apb::Node::array(
@ -151,34 +200,43 @@ impl RichObject {
#[async_trait::async_trait]
pub trait BatchFillable: Sized {
async fn with_attachments(self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>;
async fn with_batched<E>(self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
where
E: BatchFillableComparison + EntityTrait,
E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
RichObject: BatchFillableAcceptor<Vec<E::Model>>;
}
#[async_trait::async_trait]
impl BatchFillable for Vec<RichActivity> {
// TODO 3 iterations... can we make it in less passes?
async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
let objects : Vec<model::object::Model> = self
.iter()
.filter_map(|x| x.object.as_ref().cloned())
.collect();
let attachments = objects.load_many(model::attachment::Entity, tx).await?;
let mut out : std::collections::BTreeMap<i64, Vec<model::attachment::Model>> = std::collections::BTreeMap::new();
for attach in attachments.into_iter().flatten() {
match out.entry(attach.object) {
std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); },
std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); },
async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
where
E: BatchFillableComparison + EntityTrait,
E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
RichObject: BatchFillableAcceptor<Vec<E::Model>>
{
let ids : Vec<i64> = self.iter().filter_map(|x| Some(x.object.as_ref()?.internal)).collect();
let batch = E::find()
.filter(E::comparison(ids))
.all(tx)
.await?;
let mut map : HashMap<i64, Vec<E::Model>> = HashMap::new();
for element in batch {
match map.entry(element.key()) {
Entry::Occupied(mut x) => { x.get_mut().push(element); },
Entry::Vacant(x) => { x.insert(vec![element]); },
}
}
for activity in self.iter_mut() {
if let Some(ref object) = activity.object {
activity.attachments = out.remove(&object.internal);
for element in self.iter_mut() {
if let Some(ref object) = element.object {
if let Some(v) = map.remove(&object.internal) {
element.accept(v);
}
}
}
Ok(self)
}
}
@ -186,73 +244,159 @@ impl BatchFillable for Vec<RichActivity> {
#[async_trait::async_trait]
impl BatchFillable for Vec<RichObject> {
// TODO 3 iterations... can we make it in less passes?
async fn with_attachments(mut self, db: &impl ConnectionTrait) -> Result<Self, DbErr> {
let objects : Vec<model::object::Model> = self
.iter()
.map(|o| o.object.clone())
.collect();
let attachments = objects.load_many(model::attachment::Entity, db).await?;
let mut out : std::collections::BTreeMap<i64, Vec<model::attachment::Model>> = std::collections::BTreeMap::new();
for attach in attachments.into_iter().flatten() {
match out.entry(attach.object) {
std::collections::btree_map::Entry::Vacant(a) => { a.insert(vec![attach]); },
std::collections::btree_map::Entry::Occupied(mut e) => { e.get_mut().push(attach); },
async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
where
E: BatchFillableComparison + EntityTrait,
E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
RichObject: BatchFillableAcceptor<Vec<E::Model>>
{
let ids : Vec<i64> = self.iter().map(|x| x.object.internal).collect();
let batch = E::find()
.filter(E::comparison(ids))
.all(tx)
.await?;
let mut map : HashMap<i64, Vec<E::Model>> = HashMap::new();
for element in batch {
match map.entry(element.key()) {
Entry::Occupied(mut x) => { x.get_mut().push(element); },
Entry::Vacant(x) => { x.insert(vec![element]); },
}
}
for obj in self.iter_mut() {
obj.attachments = out.remove(&obj.object.internal);
for element in self.iter_mut() {
if let Some(v) = map.remove(&element.object.internal) {
element.accept(v);
}
}
Ok(self)
}
}
#[async_trait::async_trait]
impl BatchFillable for RichActivity {
async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
where
E: BatchFillableComparison + EntityTrait,
E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
RichObject: BatchFillableAcceptor<Vec<E::Model>>
{
if let Some(ref obj) = self.object {
self.attachments = Some(
obj.find_related(model::attachment::Entity)
.all(tx)
.await?
);
let batch =E::find()
.filter(E::comparison(vec![obj.internal]))
.all(tx)
.await?;
self.accept(batch);
}
Ok(self)
}
}
#[async_trait::async_trait]
impl BatchFillable for RichObject {
async fn with_attachments(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr> {
self.attachments = Some(
self.object.find_related(model::attachment::Entity)
.all(tx)
.await?
);
async fn with_batched<E>(mut self, tx: &impl ConnectionTrait) -> Result<Self, DbErr>
where
E: BatchFillableComparison + EntityTrait,
E::Model: BatchFillableKey + Send + FromQueryResult + ModelTrait<Entity = E>,
RichActivity: BatchFillableAcceptor<Vec<E::Model>>,
RichObject: BatchFillableAcceptor<Vec<E::Model>>
{
let batch = E::find()
.filter(E::comparison(vec![self.object.internal]))
.all(tx)
.await?;
self.accept(batch);
Ok(self)
}
}
#[cfg(test)]
mod test {
use sea_orm::{ColumnTrait, Condition, QueryFilter, QueryTrait};
#[test]
fn wtf_postgres() {
panic!("{}", crate::Query::objects(None)
.filter(
Condition::any()
.add(crate::model::addressing::Column::Actor.is_null())
.add(crate::model::addressing::Column::Actor.eq(2))
.add(crate::model::object::Column::AttributedTo.eq("https://upub.alemi.dev/actors/alemi"))
)
.build(sea_orm::DatabaseBackend::Postgres)
.to_string()
)
// welcome to interlocking trait hell, enjoy your stay
mod hell {
use sea_orm::{sea_query::IntoCondition, ColumnTrait};
pub trait BatchFillableComparison {
fn comparison(ids: Vec<i64>) -> sea_orm::Condition;
}
impl BatchFillableComparison for crate::model::attachment::Entity {
fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
crate::model::attachment::Column::Object.is_in(ids).into_condition()
}
}
impl BatchFillableComparison for crate::model::mention::Entity {
fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
crate::model::mention::Column::Object.is_in(ids).into_condition()
}
}
impl BatchFillableComparison for crate::model::hashtag::Entity {
fn comparison(ids: Vec<i64>) -> sea_orm::Condition {
crate::model::hashtag::Column::Object.is_in(ids).into_condition()
}
}
pub trait BatchFillableKey {
fn key(&self) -> i64;
}
impl BatchFillableKey for crate::model::attachment::Model {
fn key(&self) -> i64 {
self.internal
}
}
impl BatchFillableKey for crate::model::mention::Model {
fn key(&self) -> i64 {
self.internal
}
}
impl BatchFillableKey for crate::model::hashtag::Model {
fn key(&self) -> i64 {
self.internal
}
}
pub trait BatchFillableAcceptor<B> {
fn accept(&mut self, batch: B);
}
impl BatchFillableAcceptor<Vec<crate::model::attachment::Model>> for super::RichActivity {
fn accept(&mut self, batch: Vec<crate::model::attachment::Model>) {
self.attachments = Some(batch);
}
}
impl BatchFillableAcceptor<Vec<crate::model::hashtag::Model>> for super::RichActivity {
fn accept(&mut self, batch: Vec<crate::model::hashtag::Model>) {
self.hashtags = Some(batch);
}
}
impl BatchFillableAcceptor<Vec<crate::model::mention::Model>> for super::RichActivity {
fn accept(&mut self, batch: Vec<crate::model::mention::Model>) {
self.mentions = Some(batch);
}
}
impl BatchFillableAcceptor<Vec<crate::model::attachment::Model>> for super::RichObject {
fn accept(&mut self, batch: Vec<crate::model::attachment::Model>) {
self.attachments = Some(batch);
}
}
impl BatchFillableAcceptor<Vec<crate::model::hashtag::Model>> for super::RichObject {
fn accept(&mut self, batch: Vec<crate::model::hashtag::Model>) {
self.hashtags = Some(batch);
}
}
impl BatchFillableAcceptor<Vec<crate::model::mention::Model>> for super::RichObject {
fn accept(&mut self, batch: Vec<crate::model::mention::Model>) {
self.mentions = Some(batch);
}
}
}
use hell::*;

36
upub/core/src/traits/normalize.rs
View file

@ -1,4 +1,4 @@
use apb::{field::OptionalString, Collection, Document, Endpoints, Node, Object, PublicKey};
use apb::{field::OptionalString, Link, Collection, Document, Endpoints, Node, Object, PublicKey};
use sea_orm::{sea_query::Expr, ActiveModelTrait, ActiveValue::{Unchanged, NotSet, Set}, ColumnTrait, ConnectionTrait, DbErr, EntityTrait, IntoActiveModel, QueryFilter};
use super::Addresser;
@ -25,6 +25,7 @@ pub trait Normalizer {
impl Normalizer for crate::Context {
async fn insert_object(&self, object: impl apb::Object, tx: &impl ConnectionTrait) -> Result<crate::model::object::Model, NormalizerError> {
let now = chrono::Utc::now();
let mut object_model = AP::object(&object)?;
// make sure content only contains a safe subset of html
@ -86,7 +87,7 @@ impl Normalizer for crate::Context {
document_type: Set(apb::DocumentType::Page),
name: Set(l.name().str()),
media_type: Set(l.media_type().unwrap_or("link").to_string()),
published: Set(chrono::Utc::now()),
published: Set(now),
},
Node::Object(o) =>
AP::attachment_q(o.as_document()?, object_model.internal, None)?,
@ -123,6 +124,37 @@ impl Normalizer for crate::Context {
.await?;
}
for tag in object.tag() {
match tag.link_type() {
Ok(apb::LinkType::Mention) => {
let model = crate::model::mention::ActiveModel {
internal: NotSet,
object: Set(object_model.internal),
actor: Set(tag.href().to_string()),
published: Set(now),
};
crate::model::mention::Entity::insert(model)
.exec(tx)
.await?;
},
Ok(apb::LinkType::Hashtag) => {
let hashtag = tag.name()
.unwrap_or_else(|_| tag.href().split('/').last().unwrap_or_default())
.replace('#', "");
let model = crate::model::hashtag::ActiveModel {
internal: NotSet,
object: Set(object_model.internal),
name: Set(hashtag),
published: Set(now),
};
crate::model::hashtag::Entity::insert(model)
.exec(tx)
.await?;
},
_ => {},
}
}
Ok(object_model)
}

6
upub/routes/src/activitypub/activity.rs
View file

@ -30,7 +30,11 @@ pub async fn view(
.one(ctx.db())
.await?
.ok_or_else(crate::ApiError::not_found)?
.with_attachments(ctx.db())
.with_batched::<upub::model::attachment::Entity>(ctx.db())
.await?
.with_batched::<upub::model::mention::Entity>(ctx.db())
.await?
.with_batched::<upub::model::hashtag::Entity>(ctx.db())
.await?;
Ok(JsonLD(row.ap().ld_context()))

19
upub/routes/src/activitypub/object/mod.rs
View file

@ -3,8 +3,8 @@ pub mod context;
use apb::{BaseMut, CollectionMut, ObjectMut, LD};
use axum::extract::{Path, Query, State};
use sea_orm::{ColumnTrait, ModelTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait};
use upub::{model, selector::RichObject, traits::Fetcher, Context};
use sea_orm::{ColumnTrait, QueryFilter, QuerySelect, SelectColumns, TransactionTrait};
use upub::{model, selector::{BatchFillable, RichObject}, traits::Fetcher, Context};
use crate::{builders::JsonLD, AuthIdentity};
@ -33,14 +33,14 @@ pub async fn view(
.into_model::<RichObject>()
.one(ctx.db())
.await?
.ok_or_else(crate::ApiError::not_found)?;
let attachments = item.object.find_related(model::attachment::Entity)
.all(ctx.db())
.ok_or_else(crate::ApiError::not_found)?
.with_batched::<upub::model::attachment::Entity>(ctx.db())
.await?
.into_iter()
.map(|x| x.ap())
.collect::<Vec<serde_json::Value>>();
.with_batched::<upub::model::mention::Entity>(ctx.db())
.await?
.with_batched::<upub::model::hashtag::Entity>(ctx.db())
.await?;
let mut replies = apb::Node::Empty;
@ -66,7 +66,6 @@ pub async fn view(
Ok(JsonLD(
item.ap()
.set_attachment(apb::Node::array(attachments))
.set_replies(replies)
.ld_context()
))

12
upub/routes/src/builders.rs
View file

@ -31,7 +31,11 @@ pub async fn paginate_activities(
.into_model::<RichActivity>()
.all(db)
.await?
.with_attachments(db)
.with_batched::<upub::model::attachment::Entity>(db)
.await?
.with_batched::<upub::model::mention::Entity>(db)
.await?
.with_batched::<upub::model::hashtag::Entity>(db)
.await?;
let items : Vec<serde_json::Value> = items
@ -73,7 +77,11 @@ pub async fn paginate_objects(
.into_model::<RichObject>() // <--- difference three
.all(db)
.await?
.with_attachments(db)
.with_batched::<upub::model::attachment::Entity>(db)
.await?
.with_batched::<upub::model::mention::Entity>(db)
.await?
.with_batched::<upub::model::hashtag::Entity>(db)
.await?;
let items : Vec<serde_json::Value> = items