fix: also cloak object intrinsic image

This commit is contained in:
əlemi 2024-07-16 01:19:47 +02:00
parent 6eb964275e
commit 7ae1d02c02
Signed by: alemi
GPG key ID: A4895B84D311642C
3 changed files with 49 additions and 18 deletions

View file

@ -2,7 +2,7 @@ use futures::TryStreamExt;
use sea_orm::{ActiveModelTrait, ActiveValue::{NotSet, Set, Unchanged}, ColumnTrait, Condition, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns}; use sea_orm::{ActiveModelTrait, ActiveValue::{NotSet, Set, Unchanged}, ColumnTrait, Condition, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns};
use upub::traits::{fetch::RequestError, Cloaker}; use upub::traits::{fetch::RequestError, Cloaker};
pub async fn cloak(ctx: upub::Context, post_contents: bool, actors: bool) -> Result<(), RequestError> { pub async fn cloak(ctx: upub::Context, post_contents: bool, objects: bool, actors: bool) -> Result<(), RequestError> {
let local_base = format!("{}%", ctx.base()); let local_base = format!("{}%", ctx.base());
{ {
let mut stream = upub::model::attachment::Entity::find() let mut stream = upub::model::attachment::Entity::find()
@ -21,25 +21,23 @@ pub async fn cloak(ctx: upub::Context, post_contents: bool, actors: bool) -> Res
if post_contents { if post_contents {
let mut stream = upub::model::object::Entity::find() let mut stream = upub::model::object::Entity::find()
.filter(upub::model::object::Column::Content.like("%<img%")) .filter(upub::model::object::Column::Image.is_not_null())
.filter(upub::model::object::Column::Image.not_like(&local_base))
.select_only() .select_only()
.select_column(upub::model::object::Column::Internal) .select_column(upub::model::object::Column::Internal)
.select_column(upub::model::object::Column::Content) .select_column(upub::model::object::Column::Image)
.into_tuple::<(i64, String)>() .into_tuple::<(i64, String)>()
.stream(ctx.db()) .stream(ctx.db())
.await?; .await?;
while let Some((internal, content)) = stream.try_next().await? { while let Some((internal, image)) = stream.try_next().await? {
let sanitized = ctx.sanitize(&content); tracing::info!("cloaking object image {image}");
if sanitized != content { let model = upub::model::object::ActiveModel {
tracing::info!("sanitizing object #{internal}"); internal: Unchanged(internal),
let model = upub::model::object::ActiveModel { image: Set(Some(ctx.cloaked(&image))),
internal: Unchanged(internal), ..Default::default()
content: Set(Some(sanitized)), };
..Default::default() model.update(ctx.db()).await?;
};
model.update(ctx.db()).await?;
}
} }
} }
@ -59,6 +57,7 @@ pub async fn cloak(ctx: upub::Context, post_contents: bool, actors: bool) -> Res
.await?; .await?;
while let Some((internal, image, icon)) = stream.try_next().await? { while let Some((internal, image, icon)) = stream.try_next().await? {
tracing::info!("cloaking user #{internal}");
if image.is_none() && icon.is_none() { continue } if image.is_none() && icon.is_none() { continue }
// TODO can this if/else/else be made nicer?? // TODO can this if/else/else be made nicer??
let image = if let Some(img) = image { let image = if let Some(img) = image {
@ -88,5 +87,29 @@ pub async fn cloak(ctx: upub::Context, post_contents: bool, actors: bool) -> Res
} }
} }
if post_contents {
let mut stream = upub::model::object::Entity::find()
.filter(upub::model::object::Column::Content.like("%<img%"))
.select_only()
.select_column(upub::model::object::Column::Internal)
.select_column(upub::model::object::Column::Content)
.into_tuple::<(i64, String)>()
.stream(ctx.db())
.await?;
while let Some((internal, content)) = stream.try_next().await? {
let sanitized = ctx.sanitize(&content);
if sanitized != content {
tracing::info!("sanitizing object #{internal}");
let model = upub::model::object::ActiveModel {
internal: Unchanged(internal),
content: Set(Some(sanitized)),
..Default::default()
};
model.update(ctx.db()).await?;
}
}
}
Ok(()) Ok(())
} }

View file

@ -116,13 +116,17 @@ pub enum CliCommand {
/// replaces all attachment urls with proxied local versions (only useful for old instances) /// replaces all attachment urls with proxied local versions (only useful for old instances)
Cloak { Cloak {
/// also replace urls inside post contents /// also cloak objects image urls
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
post_contents: bool, objects: bool,
/// also cloak actor images /// also cloak actor images
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
actors: bool, actors: bool,
/// also replace urls inside post contents
#[arg(long, default_value_t = false)]
contents: bool,
}, },
} }
@ -145,7 +149,7 @@ pub async fn run(ctx: upub::Context, command: CliCommand) -> Result<(), Box<dyn
Ok(nuke(ctx, for_real, delete_objects).await?), Ok(nuke(ctx, for_real, delete_objects).await?),
CliCommand::Thread { } => CliCommand::Thread { } =>
Ok(thread(ctx).await?), Ok(thread(ctx).await?),
CliCommand::Cloak { post_contents, actors } => CliCommand::Cloak { objects, actors, contents } =>
Ok(cloak(ctx, post_contents, actors).await?), Ok(cloak(ctx, contents, objects, actors).await?),
} }
} }

View file

@ -31,6 +31,10 @@ impl Normalizer for crate::Context {
object_model.content = Some(self.sanitize(&content)); object_model.content = Some(self.sanitize(&content));
} }
if let Some(image) = object_model.image {
object_model.image = Some(self.cloaked(&image));
}
// fix context for remote posts // fix context for remote posts
// > if any link is broken or we get rate limited, the whole insertion fails which is // > if any link is broken or we get rate limited, the whole insertion fails which is
// > kind of dumb. there should be a job system so this can be done in waves. or maybe there's // > kind of dumb. there should be a job system so this can be done in waves. or maybe there's