docs: updated readme.md

also fix by-value/by-reference stuff

README.md

@@ -29,104 +29,58 @@ A collection can be super simple
 url = "https://api.alemi.dev/debug"
 ```
 
-But more complex options are available
+But more complex options are available, check out provided `postwoman.toml` for some (ready to run!) examples.
+
+More complex collection trees can be achieved with `include` top level field.
+Includes are idempotent and always resolved relative from parent collection's directory.
 
 ```toml
-[client] # HTTP client configuration
+include = [
-user_agent = "postwoman@sample/0.3.1"
+	"other.toml",
-timeout = 60 # max time for each request to complete, in seconds
+	"even/more.toml"
-redirects = 5 # allow up to five redirects, defaults to none
-
-[env] # these will be replaced in routes options. environment vars overrule these
-PW_TOKEN = "set-me-as-and-environment-variable!"
-
-
-
-[route.healthcheck] # the simplest possible route: just name and url
-url = "https://api.alemi.dev/"
-
-[route.debug]
-url = "https://api.alemi.dev/debug"
-method = "PUT" # specify request method
-query = [ # specify query parameters in a more friendly way
-	"body=json",
-	"cache=0"
 ]
-headers = [ # add custom headers to request
-	"Content-Type: application/json",
-	"Authorization: Bearer ${PW_TOKEN}",
-]
-body = { hello = "world!", success = true } # body can be a bare string, or an inline table (will be converted to json)
-extract = ".path" # extract from json responses with JQ syntax
-# note that a bare extractor string is equivalent to `{ type = "jq", query = ".path" }`
-
-[route.benchmark]
-url = "https://api.alemi.dev/look/into/the/void"
-extract = { type = "discard" } # if you don't care about the output, discard it!
-
-[route.notfound]
-url = "https://api.alemi.dev/not-found"
-expect = 404 # it's possible to specify expected status code, will fail if doesn't match
-extract = { type = "regex", pattern = 'nginx/[0-9\.]+' } # extract from response with regex
-
-[route.payload]
-url = "https://api.alemi.dev/debug"
-method = "POST"
-body = '''{
-	"complex": {
-		"json": "payloads",
-		"can": "be",
-		"expressed": "this",
-		"way": true
-	}
-}'''
-extract = { type = "body" } # get the whole response body, this is the default extractor
-
-[route.cookie]
-url = "https://api.alemi.dev/getcookie"
-method = "GET"
-extract = { type = "header", key = "Set-Cookie" } # get a specific response header, ignoring body
 ```
 
 ### Running
 Show collection summary
 ```
 $ postwoman
-> postwoman@sample/0.2.0
+~@ postwoman/0.3.1
-+ PW_TOKEN: set-me-as-and-environment-variable!
+-> postwoman.toml
-
+ + PW_TOKEN=set-me-as-and-environment-variable!
-- healthcheck: 	GET 	https://api.alemi.dev/
+ - healthcheck 	GET 	https://api.alemi.dev/
-- debug: 	PUT 	https://api.alemi.dev/debug
+ - debug 	PUT 	https://api.alemi.dev/debug
-- benchmark: 	GET 	https://api.alemi.dev/look/into/the/void
+ - benchmark 	GET 	https://api.alemi.dev/look/into/the/void
-- notfound: 	GET 	https://api.alemi.dev/not-found
+ - notfound 	GET 	https://api.alemi.dev/not-found
-- payload: 	POST 	https://api.alemi.dev/debug
+ - payload 	POST 	https://api.alemi.dev/debug
-- cookie: 	GET 	https://api.alemi.dev/getcookie
+ - cookie 	GET 	https://api.alemi.dev/getcookie
 ```
 
 Run all endpoints matching `.` (aka all of them)
 ```
 $ postwoman run .
- : [22:27:17.960461] sending healthcheck ...
+~@ postwoman/0.3.1
- + [22:27:18.109843] healthcheck done in 149ms
+ : [05:14:47.241122] postwoman.toml::healthcheck 	sending request...
+ + [05:14:47.411708] postwoman.toml::healthcheck 	done in 170ms
 {
   "example": [
     "https://api.alemi.dev/debug",
     "https://api.alemi.dev/msg",
     "https://api.alemi.dev/mumble/ping"
   ],
-  "time": "Saturday, 19-Oct-2024 20:27:18 GMT",
+  "time": "Sunday, 20-Oct-2024 03:14:47 GMT",
   "up": true
 }
- : [22:27:18.109924] sending debug ...
+ : [05:14:47.411807] postwoman.toml::debug 	sending request...
- + [22:27:18.268383] debug done in 158ms
+ + [05:14:47.574391] postwoman.toml::debug 	done in 162ms
 /debug?body=json&cache=0
- : [22:27:18.268477] sending benchmark ...
+ : [05:14:47.574474] postwoman.toml::benchmark 	sending request...
- + [22:27:18.422707] benchmark done in 154ms
+ + [05:14:47.726527] postwoman.toml::benchmark 	done in 152ms
- : [22:27:18.422775] sending notfound ...
+ : [05:14:47.726605] postwoman.toml::notfound 	sending request...
- + [22:27:18.575942] notfound done in 153ms
+ + [05:14:47.878922] postwoman.toml::notfound 	done in 152ms
 nginx/1.26.2
- : [22:27:18.576010] sending payload ...
+ : [05:14:47.879000] postwoman.toml::payload 	sending request...
- + [22:27:18.732582] payload done in 156ms
+ + [05:14:48.039053] postwoman.toml::payload 	done in 160ms
 {
   "body": "{\n\t\"complex\": {\n\t\t\"json\": \"payloads\",\n\t\t\"can\": \"be\",\n\t\t\"expressed\": \"this\",\n\t\t\"way\": true\n\t}\n}",
   "headers": {
@@ -135,33 +89,34 @@ nginx/1.26.2
     ],
     "connection": "close",
     "content-length": "94",
-    "user-agent": "postwoman@sample/0.2.0",
+    "user-agent": "postwoman@sample/0.3.1",
     "x-forwarded-proto": "https",
     "x-real-ip": "93.34.149.115",
-    "x-real-port": 46695,
+    "x-real-port": 46945,
-    "x-user-agent": "postwoman@sample/0.2.0"
+    "x-user-agent": "postwoman@sample/0.3.1"
   },
   "method": "POST",
   "path": "/debug",
-  "time": 1729369638.7079802,
+  "time": 1729394088.0156112,
   "version": "HTTP/1.0"
 }
- : [22:27:18.732676] sending cookie ...
+ : [05:14:48.039099] postwoman.toml::cookie 	sending request...
- + [22:27:18.886862] cookie done in 154ms
+ + [05:14:48.168725] postwoman.toml::cookie 	done in 129ms
 SGF2ZSBhIENvb2tpZSE=
 ```
 
 Debug a specific route passing `--debug`:
 ```
 $ postwoman run notfound --debug
- : [22:26:59.045642] sending notfound ...
+~@ postwoman/0.3.1
- + [22:26:59.220103] notfound done in 174ms
+ : [05:15:16.960147] postwoman.toml::notfound 	sending request...
+ + [05:15:17.120647] postwoman.toml::notfound 	done in 160ms
 Response {
     url: "https://api.alemi.dev/not-found",
     status: 404,
     headers: {
         "server": "nginx/1.26.2",
-        "date": "Sat, 19 Oct 2024 20:26:59 GMT",
+        "date": "Sun, 20 Oct 2024 03:15:17 GMT",
         "content-type": "text/html",
         "content-length": "153",
         "connection": "keep-alive",

src/main.rs

@@ -2,6 +2,8 @@ mod model;
 mod errors;
 mod ext;
 
+use std::{collections::HashMap, str::FromStr};
+
 use clap::{Parser, Subcommand};
 
 pub use model::PostWomanCollection;
@@ -15,7 +17,7 @@ pub static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CAR
 struct PostWomanArgs {
 	/// collection file to use
 	#[arg(short, long, default_value = "postwoman.toml")]
-	collection: String,
+	collection: std::path::PathBuf,
 
 	/// action to run
 	#[clap(subcommand)]
@@ -34,13 +36,9 @@ pub enum PostWomanActions {
 		query: String,
 
 		/// run requests in parallel
-		#[arg(long, default_value_t = false)]
+		#[arg(short, long, default_value_t = false)]
 		parallel: bool,
 
-		/// repeat request N times
-		#[arg(long, default_value_t = 1)]
-		repeat: u32,
-
 		/// force debug extractor on all routes
 		#[arg(long, default_value_t = false)]
 		debug: bool,
@@ -48,124 +46,189 @@ pub enum PostWomanActions {
 
 	/// show all registered routes in current collection
 	List {
-		/// show verbose details for each route
+		/// show only limited details for each route
 		#[arg(short, long, default_value_t = false)]
-		verbose: bool,
+		compact: bool,
 	},
 }
 
 const TIMESTAMP_FMT: &str = "%H:%M:%S%.6f"; 
 
-fn main() -> Result<(), PostWomanError> {
+fn main() {
 	let args = PostWomanArgs::parse();
+	let multi_thread = args.multi_threaded;
 
-	let collection_raw = std::fs::read_to_string(&args.collection)?;
+	// if we got a regex, test it early to avoid wasting work when invalid
-	let collection: PostWomanCollection = toml::from_str(&collection_raw)?;
+	if let Some(PostWomanActions::Run { ref query, .. }) = args.action {
+		// note that if you remove this test, there's another .expect() below you need to manage too!
+		if let Err(e) = regex::Regex::new(query) {
+			return eprintln!("! invalid regex filter: {e}");
+		}
+	}
 
-	if args.multi_threaded {
+	let mut collections = HashMap::new();
+
+	if !load_collections(&mut collections, args.collection.clone()) {
+		return;
+	}
+
+	let task = async move {
+
+		let mut pool = tokio::task::JoinSet::new();
+
+		for (collection_name, collection) in collections {
+			run_postwoman(&args, collection_name, collection, &mut pool).await;
+		}
+
+		while let Some(j) = pool.join_next().await {
+			match j {
+				Err(e) => eprintln!("! error joining task: {e}"),
+				Ok(res) => res.print(),
+			}
+		}
+	};
+
+	eprintln!("~@ {APP_USER_AGENT}");
+	if multi_thread {
 		tokio::runtime::Builder::new_multi_thread()
 			.enable_all()
 			.build()
 			.expect("failed creating tokio multi-thread runtime")
-			.block_on(async { run_postwoman(args, collection).await })
+			.block_on(task)
 	} else {
 		tokio::runtime::Builder::new_current_thread()
 			.enable_all()
 			.build()
 			.expect("failed creating tokio current-thread runtime")
-			.block_on(async { run_postwoman(args, collection).await })
+			.block_on(task)
 	}
 }
 
-async fn run_postwoman(args: PostWomanArgs, collection: PostWomanCollection) -> Result<(), PostWomanError> {
+fn load_collections(store: &mut HashMap<String, PostWomanCollection>, mut path: std::path::PathBuf) -> bool {
-	let action = args.action.unwrap_or(PostWomanActions::List { verbose: false });
+	let collection_raw = match std::fs::read_to_string(&path) {
+		Ok(x) => x,
+		Err(e) => {
+			eprintln!("! error loading collection {path:?}: {e}");
+			return false;
+		},
+	};
+
+	let collection: PostWomanCollection = match toml::from_str(&collection_raw) {
+		Ok(x) => x,
+		Err(e) => {
+			eprintln!("! error parsing collection {path:?}: {e}");
+			return false;
+		},
+	};
+
+	let name = path.to_string_lossy().to_string();
+
+	if let Some(ref includes) = collection.include {
+		path.pop();
+		for include in includes {
+			let mut base = path.clone();
+			let new = std::path::PathBuf::from_str(include).expect("infallible");
+			base.push(new);
+			if !load_collections(store, base) {
+				return false;
+			}
+		}
+	}
+
+	store.insert(name, collection);
+
+	true
+}
+
+const DEFAULT_ACTION: PostWomanActions = PostWomanActions::List { compact: true };
+type RunResult = (Result<String, PostWomanError>, String, String, chrono::DateTime<chrono::Local>);
+
+async fn run_postwoman(args: &PostWomanArgs, namespace: String, collection: PostWomanCollection, pool: &mut tokio::task::JoinSet<RunResult>) {
+	let action = args.action.as_ref().unwrap_or(&DEFAULT_ACTION);
 
 	match action {
-		PostWomanActions::List { verbose } => {
+		PostWomanActions::List { compact } => {
-			let ua = collection.client.user_agent.unwrap_or(APP_USER_AGENT.to_string());
+			println!("-> {namespace}");
-			println!("> {ua}");
 
-			for (key, value) in collection.env {
+			for (key, value) in collection.env.unwrap_or_default() {
-				println!("+ {key}: {}", ext::stringify_toml(&value));
+				println!(" + {key}={}", ext::stringify_toml(&value));
 			}
 
-			println!();
-
 			for (name, mut endpoint) in collection.route {
-				println!("- {name}: \t{} \t{}", endpoint.method.as_deref().unwrap_or("GET"), endpoint.url);
+				println!(" - {name} \t{} \t{}", endpoint.method.as_deref().unwrap_or("GET"), endpoint.url);
-				if verbose {
+				if ! *compact {
 					if let Some(ref query) = endpoint.query {
 						for query in query {
-							println!(" |? {query}");
+							println!("   |? {query}");
 						}
 					}
 					if let Some(ref headers) = endpoint.headers {
 						for header in headers {
-							println!(" |: {header}");
+							println!("   |: {header}");
 						}
 					}
 					if let Some(ref _x) = endpoint.body {
 						if let Ok(body) = endpoint.body() {
-							println!(" |> {body}");
+							println!("   |> {}", body.replace("\n", "\n   |> "));
 						} else {
-							println!(" |> [!] invalid body");
+							println!("   |> [!] invalid body");
 						}
 					}
 				}
 			}
+
+			println!();
 		},
-		PostWomanActions::Run { query, parallel, repeat, debug  } => {
+		PostWomanActions::Run { query, parallel, debug  } => {
-			let pattern = regex::Regex::new(&query)?;
+			// this is always safe to compile because we tested it beforehand
-			let mut joinset = tokio::task::JoinSet::new();
+			let pattern = regex::Regex::new(query).expect("tested it before and still failed here???");
-			let client = std::sync::Arc::new(collection.client);
+			let client = std::sync::Arc::new(collection.client.unwrap_or_default());
-			let env = std::sync::Arc::new(collection.env);
+			let env = std::sync::Arc::new(collection.env.unwrap_or_default());
 			for (name, mut endpoint) in collection.route {
 				if pattern.find(&name).is_some() {
-					if debug { endpoint.extract = Some(ext::StringOr::T(model::ExtractorConfig::Debug)) };
+					if *debug { endpoint.extract = Some(ext::StringOr::T(model::ExtractorConfig::Debug)) };
-					for i in 0..repeat {
+					let _client = client.clone();
-						let suffix = if repeat > 1 {
+					let _env = env.clone();
-							format!("#{} ", i+1)
+					let _endpoint = endpoint.clone();
-						} else {
+					let _name = name.clone();
-							"".to_string()
+					let _namespace = namespace.clone();
-						};
+					let task = async move {
-						let _client = client.clone();
+						let before = chrono::Local::now();
-						let _env = env.clone();
+						eprintln!(" : [{}] {_namespace}::{_name} \tsending request...", before.format(TIMESTAMP_FMT));
-						let _endpoint = endpoint.clone();
+						let res = _endpoint
-						let _name = name.clone();
+							.fill(&_env)
-						let task = async move {
+							.execute(&_client)
-							let before = chrono::Local::now();
+							.await;
-							eprintln!(" : [{}] sending {_name} {suffix}...", before.format(TIMESTAMP_FMT));
+						(res, _namespace, _name, before)
-							let res = _endpoint
+					};
-								.fill(&_env)
+					if *parallel {
-								.execute(&_client)
+						pool.spawn(task);
-								.await;
+					} else {
-							(res, _name, before, suffix)
+						task.await.print();
-						};
-						if parallel {
-							joinset.spawn(task);
-						} else {
-							let (res, name, before, num) = task.await;
-							print_results(res?, name, before, num);
-						}
 					}
 				}
 			}
-			while let Some(j) = joinset.join_next().await {
-				match j {
-					Ok((res, name, before, num)) => print_results(res?, name, before, num),
-					Err(e) => eprintln!("! error joining task: {e}"),
-				}
-			}
 		},
 	}
-
-	Ok(())
 }
 
-fn print_results(res: String, name: String, before: chrono::DateTime<chrono::Local>, suffix: String) {
+trait PrintableResult {
-	let after = chrono::Local::now();
+	fn print(self);
-	let elapsed = (after - before).num_milliseconds();
+}
-	let timestamp = after.format(TIMESTAMP_FMT);
+
-	eprintln!(" + [{timestamp}] {name} {suffix}done in {elapsed}ms", );
+impl PrintableResult for RunResult {
-	print!("{}", res);
+	fn print(self) {
+		let (result, namespace, name, before) = self;
+		let success = result.is_ok();
+		let after = chrono::Local::now();
+		let elapsed = (after - before).num_milliseconds();
+		let timestamp = after.format(TIMESTAMP_FMT);
+		let symbol = if success { " + " } else { "<!>" };
+		let verb = if success { "done in" } else { "failed after" };
+		eprintln!("{symbol}[{timestamp}] {namespace}::{name} \t{verb} {elapsed}ms", );
+		match result {
+			Ok(x) => print!("{x}"),
+			Err(e) => eprintln!(" ! {e}"),
+		}
+	}
 }

src/model/mod.rs

@@ -8,8 +8,9 @@ pub use extractor::ExtractorConfig;
 
 #[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize)]
 pub struct PostWomanCollection {
-	pub client: ClientConfig,
+	pub client: Option<ClientConfig>,
-	pub env: toml::Table,
+	pub env: Option<toml::Table>,
+	pub include: Option<Vec<String>>,
 	// it's weird to name it singular but makes more sense in config
 	pub route: indexmap::IndexMap<String, EndpointConfig>,
 }