feat(cli): added cloak command to fix previous urls

upub/cli/Cargo.toml

@@ -22,3 +22,4 @@ openssl = "0.10" # TODO handle pubkeys with a smaller crate
 clap = { version = "4.5", features = ["derive"] }
 sea-orm = { version = "0.12", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls"] }
 futures = "0.3"
+mdhtml = { path = "../../utils/mdhtml/" }

upub/cli/src/cloak.rs

@@ -0,0 +1,46 @@
+use futures::TryStreamExt;
+use sea_orm::{ActiveModelTrait, ActiveValue::{Set, Unchanged}, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, QuerySelect, SelectColumns, TransactionTrait};
+use upub::traits::{fetch::RequestError, Cloaker};
+
+pub async fn cloak(ctx: upub::Context, post_contents: bool) -> Result<(), RequestError> {
+	let tx = ctx.db().begin().await?;
+
+	{
+		let mut stream = upub::model::attachment::Entity::find()
+			.filter(upub::model::attachment::Column::Url.not_like(format!("{}%", ctx.base())))
+			.stream(ctx.db())
+			.await?;
+
+		while let Some(attachment) = stream.try_next().await? {
+			let (sig, url) = ctx.cloak(&attachment.url);
+			let mut model = attachment.into_active_model();
+			model.url = Set(upub::url!(ctx, "/proxy/{sig}/{url}"));
+			model.update(&tx).await?;
+		}
+	}
+
+	if post_contents {
+		let mut stream = upub::model::object::Entity::find()
+			.filter(upub::model::object::Column::Content.is_not_null())
+			.select_only()
+			.select_column(upub::model::object::Column::Internal)
+			.select_column(upub::model::object::Column::Content)
+			.into_tuple::<(i64, String)>()
+			.stream(ctx.db())
+			.await?;
+
+		while let Some((internal, content)) = stream.try_next().await? {
+			let sanitized = mdhtml::safe_html(&content);
+			if sanitized != content {
+				let model = upub::model::object::ActiveModel {
+					internal: Unchanged(internal),
+					content: Set(Some(sanitized)),
+					..Default::default()
+				};
+				model.update(&tx).await?;
+			}
+		}
+	}
+
+	Ok(())
+}

upub/cli/src/lib.rs

@@ -22,6 +22,9 @@ pub use nuke::*;
 mod thread;
 pub use thread::*;
 
+mod cloak;
+pub use cloak::*;
+
 #[derive(Debug, Clone, clap::Subcommand)]
 pub enum CliCommand {
 	/// generate fake user, note and activity
@@ -109,7 +112,14 @@ pub enum CliCommand {
 	/// attempt to fix broken threads and completely gather their context
 	Thread {
 		
-	}
+	},
+
+	/// replaces all attachment urls with proxied local versions (only useful for old instances)
+	Cloak {
+		/// also replace urls inside post contents
+		#[arg(long, default_value_t = false)]
+		post_contents: bool,
+	},
 }
 
 pub async fn run(ctx: upub::Context, command: CliCommand) -> Result<(), Box<dyn std::error::Error>> {
@@ -131,5 +141,7 @@ pub async fn run(ctx: upub::Context, command: CliCommand) -> Result<(), Box<dyn
 			Ok(nuke(ctx, for_real, delete_objects).await?),
 		CliCommand::Thread { } =>
 			Ok(thread(ctx).await?),
+		CliCommand::Cloak { post_contents } =>
+			Ok(cloak(ctx, post_contents).await?),
 	}
 }

upub/core/Cargo.toml

@@ -14,7 +14,9 @@ readme = "README.md"
 thiserror = "1"
 async-recursion = "1.1"
 async-trait = "0.1"
-sha256 = "1.5"
+sha256 = "1.5" # TODO get rid of this and use directly sha2!!
+sha2 = "0.10"
+hmac = "0.12"
 openssl = "0.10" # TODO handle pubkeys with a smaller crate
 base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }

upub/core/src/traits/cloak.rs

@@ -0,0 +1,37 @@
+use base64::{Engine, prelude::BASE64_URL_SAFE};
+use hmac::Mac;
+
+
+pub type Signature = hmac::Hmac<sha2::Sha256>;
+
+pub trait Cloaker {
+	fn secret(&self) -> &str;
+
+	fn cloak(&self, url: &str) -> (String, String) {
+		let mut hmac = Signature::new_from_slice(self.secret().as_bytes())
+			.expect("invalid length for hmac key, cannot cloak");
+		hmac.update(url.as_bytes());
+		let sig = BASE64_URL_SAFE.encode(hmac.finalize().into_bytes());
+		let url = BASE64_URL_SAFE.encode(url);
+		(sig, url)
+	}
+
+	fn uncloak(&self, signature: &str, url: &str) -> Option<String> {
+		let mut hmac = Signature::new_from_slice(self.secret().as_bytes())
+			.expect("invalid length for hmac key, cannot cloak");
+
+		let sig = BASE64_URL_SAFE.decode(signature).ok()?;
+		let url = std::str::from_utf8(&BASE64_URL_SAFE.decode(url).ok()?).ok()?.to_string();
+
+		hmac.update(url.as_bytes());
+		hmac.verify_slice(&sig).ok()?;
+
+		Some(url)
+	}
+}
+
+impl Cloaker for crate::Context {
+	fn secret(&self) -> &str {
+		&self.cfg().security.proxy_secret
+	}
+}

upub/core/src/traits/mod.rs

@@ -3,9 +3,11 @@ pub mod fetch;
 pub mod normalize;
 pub mod process;
 pub mod admin;
+pub mod cloak;
 
 pub use admin::Administrable;
 pub use address::Addresser;
 pub use normalize::Normalizer;
 pub use process::Processor;
 pub use fetch::Fetcher;
+pub use cloak::Cloaker;