feat: add session and local id vars

i want to throw errors when vars are missing, so that its easier to find
and fill them when running pw

Cargo.lock

@@ -815,7 +815,7 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "postwoman"
-version = "0.4.1"
+version = "0.4.2"
 dependencies = [
  "base64",
  "chrono",
@@ -832,6 +832,7 @@ dependencies = [
  "tokio",
  "toml",
  "toml_edit",
+ "uuid",
 ]
 
 [[package]]
@@ -1385,6 +1386,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "uuid"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "postwoman"
 description = "API tester and debugger for your CLI "
-version = "0.4.1"
+version = "0.4.2"
 repository = "https://moonlit.technology/alemi/postwoman"
 authors = [ "alemi <me@alemi.dev>" ]
 license = "GPL-3.0-only"
@@ -25,6 +25,7 @@ thiserror = "1.0.64"
 tokio = { version = "1.40", features = ["rt-multi-thread"] }
 toml = { version = "0.8", features = ["preserve_order"] }
 toml_edit = { version = "0.22", features = ["serde"] } # only to pretty print tables ...
+uuid = { version = "1.11", features = ["v4"] }
 
 [profile.release]
 opt-level = "z"

src/errors.rs

@@ -42,6 +42,9 @@ pub enum PostWomanError {
 
 	#[error("regex failed matching in content: {0}")]
 	NoMatch(String),
+
+	#[error("missing environment variable: {0}")]
+	MissingVar(#[from] crate::ext::FillError),
 }
 
 #[derive(Debug, thiserror::Error)]

src/ext.rs

@@ -1,3 +1,10 @@
+use std::{collections::HashMap, sync::OnceLock};
+
+pub fn sid() -> &'static str {
+	static SID: std::sync::OnceLock<String> = std::sync::OnceLock::new();
+	SID.get_or_init(|| uuid::Uuid::new_v4().to_string())
+}
+
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 #[serde(untagged)]
 pub enum StringOr<T> {
@@ -34,18 +41,47 @@ pub fn stringify_json(v: &serde_json::Value) -> String {
 	}
 }
 
+pub fn var_matcher() -> &'static regex::Regex {
+	static MATCHER : OnceLock<regex::Regex> = OnceLock::new();
+	MATCHER.get_or_init(|| regex::Regex::new(r"\$\{(\w+)\}").expect("wrong matcher regex"))
+}
+
 // keep it as separate fn so we can change it everywhere easily
 pub fn full_name(namespace: &str, name: &str) -> String {
 	format!("{namespace}:{name}")
 }
 
-pub trait FillableFromEnvironment {
-	fn fill(self, env: &toml::Table) -> Self;
+#[derive(Debug, thiserror::Error)]
+#[error("could not fill {0}")]
+pub struct FillError(pub String);
+
+pub trait FillableFromEnvironment: Sized {
+	fn fill(self, env: &toml::Table) -> Result<Self, FillError>;
+
+	fn replace(mut from: String, env: &HashMap<String, String>) -> Result<String, FillError> {
+		let placeholders: Vec<(String, String)> = var_matcher()
+			.captures_iter(&from)
+			.map(|m| m.extract())
+			.map(|(txt, [var])| (txt.to_string(), var.to_string()))
+			.collect();
+
+		// TODO can we avoid cloning all matches??? can't mutate `from` as captures_iter holds an
+		// immutable reference to original string
+
+		for (txt, var) in placeholders {
+			let value = env.get(&var).ok_or(FillError(var.to_string()))?;
+			from = from.replace(&txt, value);
+		}
+
+		Ok(from)
+	}
 
 	fn default_vars(env: &toml::Table) -> std::collections::HashMap<String, String> {
 		let mut vars: std::collections::HashMap<String, String> = std::collections::HashMap::default();
 
 		vars.insert("POSTWOMAN_TIMESTAMP".to_string(), chrono::Local::now().timestamp().to_string());
+		vars.insert("POSTWOMAN_LOCAL_ID".to_string(), uuid::Uuid::new_v4().to_string());
+		vars.insert("POSTWOMAN_SESSION_ID".to_string(), sid().to_string());
 
 		for (k, v) in env {
 			vars.insert(k.to_string(), stringify_toml(v));

src/main.rs

@@ -70,14 +70,6 @@ fn main() {
 	let args = PostWomanArgs::parse();
 	let multi_thread = args.multi_thread;
 
-	// if we got a regex, test it early to avoid wasting work when invalid
-	if let Some(PostWomanActions::Run { ref query, .. }) = args.action {
-		// note that if you remove this test, there's another .expect() below you need to manage too!
-		if let Err(e) = regex::Regex::new(query) {
-			return eprintln!("! invalid regex filter: {e}");
-		}
-	}
-
 	let mut collections = IndexMap::new();
 
 	if !load_collections(&mut collections, args.collection.clone(), &toml::Table::default()) {
@@ -94,6 +86,11 @@ fn main() {
 		},
 
 		PostWomanActions::Run { query, parallel, debug, dry_run } => {
+			// note that if you remove this test, there's another .expect() below you need to manage too!
+			let filter = match regex::Regex::new(query) {
+				Ok(regex) => regex,
+				Err(e) => return eprintln!("! invalid regex filter: {e}"),
+			};
 			let task = async move {
 				let mut pool = tokio::task::JoinSet::new();
 
@@ -101,7 +98,7 @@ fn main() {
 					run_collection_endpoints(
 						collection_name,
 						collection,
-						query.clone(),
+						filter.clone(),
 						*parallel,
 						*debug,
 						*dry_run,
@@ -138,22 +135,39 @@ fn main() {
 // TODO too many arguments
 async fn run_collection_endpoints(
 	namespace: String,
-	collection: PostWomanCollection,
-	query: String,
+	mut collection: PostWomanCollection,
+	filter: regex::Regex,
 	parallel: bool,
 	debug: bool,
 	dry_run: bool,
 	report: bool,
 	pool: &mut tokio::task::JoinSet<()>
 ) {
-	// this is always safe to compile because we tested it beforehand
-	let pattern = regex::Regex::new(&query).expect("tested it before and still failed here???");
-	let env = std::sync::Arc::new(collection.env);
-	let client = std::sync::Arc::new(collection.client.fill(&env));
+	let mut matched_endpoints = Vec::new();
+	for name in collection.route.keys() {
+		let full_name = ext::full_name(&namespace, name);
+		if filter.find(&full_name).is_some() {
+			matched_endpoints.push(name.clone());
+		};
+	}
 
-	for (name, mut endpoint) in collection.route {
+	if matched_endpoints.is_empty() { return } // nothing to do for this collection
+
+	let env = std::sync::Arc::new(collection.env);
+	let client = match collection.client.fill(&env) {
+		Ok(c) => std::sync::Arc::new(c),
+		Err(e) => return eprintln!(
+			"<!>[{}] {namespace}:* \terror constructing client\n ! missing environment variable: {e}",
+			chrono::Local::now().format(fmt::TIMESTAMP_FMT)
+		),
+	};
+
+	for name in matched_endpoints {
+		let mut endpoint = collection.route
+			.swap_remove(&name)
+			.expect("endpoint removed while running collection?");
 		let full_name = ext::full_name(&namespace, &name);
-		if pattern.find(&full_name).is_none() { continue };
+		if filter.find(&full_name).is_none() { continue };
 
 		if debug { endpoint.extract = Some(ext::StringOr::T(model::ExtractorConfig::Debug)) };
 		let _client = client.clone();
@@ -162,15 +176,17 @@ async fn run_collection_endpoints(
 
 		let task = async move {
 			let before = chrono::Local::now();
-			eprintln!(" : [{}] {full_name} \tsending request...", before.format(fmt::TIMESTAMP_FMT));
 
-			let res = if dry_run {
-				Ok("".to_string())
-			} else {
-				endpoint
-					.fill(&_env)
-					.execute(&_client)
-					.await
+			let res = match endpoint.fill(&_env) {
+				Err(e) => Err(e.into()),
+				Ok(e) => {
+					eprintln!(" : [{}] {full_name} \tsending request...", before.format(fmt::TIMESTAMP_FMT));
+					if dry_run {
+						Ok("".to_string())
+					} else {
+						e.execute(&_client).await
+					}
+				},
 			};
 
 			let after = chrono::Local::now();
@@ -179,7 +195,7 @@ async fn run_collection_endpoints(
 			let timestamp = after.format(fmt::TIMESTAMP_FMT);
 			let symbol = if res.is_ok() { " + " } else { "<!>" };
 			let verb = if res.is_ok() { "done in" } else { "failed after" };
-			eprintln!("{symbol}[{timestamp}] {_namespace}::{name} \t{verb} {elapsed}ms", );
+			eprintln!("{symbol}[{timestamp}] {_namespace}:{name} \t{verb} {elapsed}ms", );
 
 			if report {
 				(res, _namespace, name, elapsed).report();

src/model/client.rs

@@ -1,4 +1,4 @@
-use crate::ext::FillableFromEnvironment;
+use crate::ext::{FillError, FillableFromEnvironment};
 
 
 #[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)]
@@ -16,21 +16,17 @@ pub struct ClientConfig {
 }
 
 impl FillableFromEnvironment for ClientConfig {
-	fn fill(mut self, env: &toml::Table) -> Self {
+	fn fill(mut self, env: &toml::Table) -> Result<Self, FillError> {
 		let vars = Self::default_vars(env);
 
-		for (k, v) in vars {
-			let k_var = format!("${{{k}}}");
-
-			if let Some(base) = self.base {
-				self.base = Some(base.replace(&k_var, &v));
-			}
-
-			if let Some(user_agent) = self.user_agent {
-				self.user_agent = Some(user_agent.replace(&k_var, &v));
-			}
+		if let Some(ref base) = self.base {
+			self.base = Some(Self::replace(base.clone(), &vars)?);
 		}
 
-		self
+		if let Some(ref user_agent) = self.user_agent {
+			self.user_agent = Some(Self::replace(user_agent.clone(), &vars)?);
+		}
+
+		Ok(self)
 	}
 }

src/model/endpoint.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::str::FromStr;
 
 use base64::{prelude::BASE64_STANDARD, Engine};
@@ -8,7 +9,7 @@ use jaq_interpret::FilterT;
 use crate::errors::InvalidHeaderError;
 use crate::{PostWomanError, APP_USER_AGENT};
 
-use crate::ext::{stringify_json, FillableFromEnvironment, StringOr};
+use crate::ext::{stringify_json, FillError, FillableFromEnvironment, StringOr};
 use super::{ExtractorConfig, ClientConfig};
 
 
@@ -84,6 +85,8 @@ impl EndpointConfig {
 		let client = reqwest::Client::builder()
 			.user_agent(opts.user_agent.as_deref().unwrap_or(APP_USER_AGENT))
 			.timeout(std::time::Duration::from_secs(opts.timeout.unwrap_or(30)))
+			.read_timeout(std::time::Duration::from_secs(opts.timeout.unwrap_or(30)))
+			.connect_timeout(std::time::Duration::from_secs(opts.timeout.unwrap_or(30)))
 			.redirect(opts.redirects.map(reqwest::redirect::Policy::limited).unwrap_or(reqwest::redirect::Policy::none()))
 			.danger_accept_invalid_certs(opts.accept_invalid_certs.unwrap_or(false))
 			.build()?;
@@ -146,84 +149,89 @@ impl EndpointConfig {
 }
 
 impl FillableFromEnvironment for EndpointConfig {
-	fn fill(mut self, env: &toml::Table) -> Self {
+	fn fill(mut self, env: &toml::Table) -> Result<Self, FillError> {
 		let vars = Self::default_vars(env);
 
-		for (k, v) in vars {
-			let k_var = format!("${{{k}}}");
-			self.path = self.path.replace(&k_var, &v);
-			if let Some(method) = self.method {
-				self.method = Some(method.replace(&k_var, &v));
-			}
-			if let Some(b) = self.body {
-				match b {
-					StringOr::Str(body) => {
-						self.body = Some(StringOr::Str(body.replace(&k_var, &v)));
-					},
-					StringOr::T(json) => {
-						let wrap = toml::Value::Table(json.clone());
-						let toml::Value::Table(out) = replace_recursive(wrap, &k_var, &v)
-						else { unreachable!("we put in a table, we get out a table") };
-						self.body = Some(StringOr::T(out));
-					},
-				}
-			}
-			if let Some(query) = self.query {
-				self.query = Some(
-					query.into_iter()
-						.map(|x| x.replace(&k_var, &v))
-						.collect()
-				);
-			}
-			if let Some(headers) = self.headers {
-				self.headers = Some(
-					headers.into_iter()
-						.map(|x| x.replace(&k_var, &v))
-						.collect()
-				);
+		self.path = Self::replace(self.path, &vars)?;
+		if let Some(method) = self.method {
+			self.method = Some(Self::replace(method, &vars)?);
+		}
+		if let Some(b) = self.body {
+			match b {
+				StringOr::Str(body) => {
+					self.body = Some(StringOr::Str(Self::replace(body, &vars)?));
+				},
+				StringOr::T(json) => {
+					let wrap = toml::Value::Table(json);
+					let toml::Value::Table(out) = replace_recursive(wrap, &vars)?
+					else { unreachable!("we put in a table, we get out a table") };
+					self.body = Some(StringOr::T(out));
+				},
 			}
 		}
+		if let Some(query) = self.query {
+			let mut out = Vec::new();
+			for q in query {
+				out.push(Self::replace(q, &vars)?);
+			}
+			self.query = Some(out);
+		}
+		if let Some(headers) = self.headers {
+			let mut out = Vec::new();
+			for h in headers {
+				out.push(Self::replace(h.clone(), &vars)?);
+			}
+			self.headers = Some(out);
+		}
 		
-		self
+		Ok(self)
 	}
 }
 
-fn replace_recursive(element: toml::Value, from: &str, to: &str) -> toml::Value {
-	match element {
+fn replace_recursive(element: toml::Value, env: &HashMap<String, String>) -> Result<toml::Value, FillError> {
+	Ok(match element {
 		toml::Value::Float(x) => toml::Value::Float(x),
 		toml::Value::Integer(x) => toml::Value::Integer(x),
 		toml::Value::Boolean(x) => toml::Value::Boolean(x),
 		toml::Value::Datetime(x) => toml::Value::Datetime(x),
-		toml::Value::String(x) => toml::Value::String(x.replace(from, to)),
-		toml::Value::Array(x) => toml::Value::Array(
-			x.into_iter().map(|x| replace_recursive(x, from, to)).collect()
-		),
+		toml::Value::String(x) => toml::Value::String(EndpointConfig::replace(x, env)?),
+		toml::Value::Array(mut arr) => {
+			for v in arr.iter_mut() {
+				*v = replace_recursive(v.clone(), env)?;
+			}
+			toml::Value::Array(arr)
+		},
 		toml::Value::Table(map) => {
 			let mut out = toml::map::Map::new();
 			for (k, v) in map {
-				let new_v = replace_recursive(v.clone(), from, to);
-				if k.contains(from) {
-					out.insert(k.replace(from, to), new_v);
-				} else {
-					out.insert(k.to_string(), new_v);
-				}
+				let new_v = replace_recursive(v.clone(), env)?;
+				let new_k = EndpointConfig::replace(k, env)?;
+				out.insert(new_k, new_v);
 			}
 			toml::Value::Table(out)
 		},
-	}
+	})
 }
 
 async fn format_body(res: reqwest::Response) -> Result<String, PostWomanError> {
-	match res.headers().get("Content-Type") {
-		None => Ok(res.text().await?),
+	let content_type = res.headers().get("Content-Type").cloned();
+	let raw = res.bytes().await?;
+	match content_type {
+		None => Ok(String::from_utf8_lossy(&raw).to_string()),
 		Some(v) => {
 			let content_type = v.to_str()?;
 			if content_type.starts_with("application/json") {
-				Ok(serde_json::to_string_pretty(&res.json::<serde_json::Value>().await?)?)
+				match serde_json::from_slice::<serde_json::Value>(&raw) {
+					Ok(x) => Ok(serde_json::to_string_pretty(&x)?),
+					Err(e) => {
+						eprintln!(" ? content-type is 'json' but content is not valid json: {e}");
+						Ok(String::from_utf8_lossy(&raw).to_string())
+					},
+				}
 			} else if content_type.starts_with("text/plain") || content_type.starts_with("text/html") {
-				Ok(res.text().await?)
+				Ok(String::from_utf8_lossy(&raw).to_string())
 			} else {
-				Ok(format!("base64({})\n", BASE64_STANDARD.encode(res.bytes().await?)))
+				Ok(format!("base64({})\n", BASE64_STANDARD.encode(raw)))
 			}
 		},
 	}