commit 93292667f3ab48d7fa8b09e5398b3bd79b991e64 Author: paimon Date: Thu Jun 18 15:03:00 2026 +0200 Release: V-1.0 diff --git a/Cargo.toml b/Cargo.toml new file mode 100755 index 0000000..02ff64b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "ROFMAD" +version = "1.0.0" +edition = "2021" +license = "Apache-2.0" +description = "ROFMAD (Rust Orchestrator For Modular Automated Discovery) — an automated internal network reconnaissance and enumeration framework for authorized penetration testing" +authors = ["P.aïmon"] +repository = "https://github.com/P-Aimon-Pen/ROFMAD" + +[dependencies] +chrono = "0.4.40" +clap = { version = "4.5.30", features = ["derive"] } +tokio = { version = "1.43.0", features = ["rt-multi-thread", "macros", "process", "signal"] } +sqlx = { version = "0.8.3", features = ["sqlite", "runtime-tokio-rustls"] } +uuid = { version = "1", features = ["v4"] } +regex = "1.11.1" +serde = "1.0.219" +serde_yaml = "0.9.34" +colored = "3.0.0" +ipnetwork = "0.21.1" +once_cell = "1.21.3" +futures = "0.3.31" +lazy_static = "1.5.0" +serde_json = "1.0.140" +which = "7.0.3" +num_cpus = "1.16.0" +nix = "0.30.1" +if-addrs = "0.13.4" +tokio-util = "0.7.15" + +# Fixed reqwest - using rustls-tls for static linking compatibility +reqwest = { version = "0.11", features = ["json", "rustls-tls"], default-features = false } + +# Fixed OpenSSL - using vendored feature for static linking +openssl = { version = "0.10", features = ["vendored"] } + +url = "2.5.4" + +headless_chrome = "1.0.9" +bcrypt = "0.15" + +# Web dependencies - these should work fine with static linking +axum = { version = "0.7", features = ["ws", "multipart"] } +askama = "0.12" +rand = "0.8" +roxmltree = "0.20" + +# Build profile optimized for release +[profile.release] +opt-level = "z" # Optimize for size +lto = true # Link-time optimization +codegen-units = 1 # Single codegen unit for better optimization +panic = "abort" # Reduce binary size +strip = true # Strip debug symbols diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..423f9fb --- /dev/null +++ b/LICENSE @@ -0,0 +1,198 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship covered by this License, + whether in Source or Object form, made available under the License, + as indicated by a copyright notice that is included in or attached + to the work. (See the Appendix below for an example.) + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based upon (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and derivative works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are + managed by, or on behalf of, the Licensor for the purpose of + discussing and improving the Work, but excluding communication that + is conspicuously marked or otherwise designated in writing by the + copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to use, reproduce, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Work, and to + permit persons to whom the Work is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Work. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" file as part of its + distribution, then any Derivative Works that You distribute + must include a readable copy of the attribution notices + contained within such NOTICE file, excluding those notices + that do not pertain to any part of the Derivative Works, + in at least one of the following places: within a NOTICE file + distributed as part of the Derivative Works; within the Source + form or documentation, if provided along with the Derivative + Works; or, within a display generated by the Derivative Works, + if and wherever such third-party notices normally appear. The + contents of the NOTICE file are for informational purposes only + and do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright notice to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 7. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 8. Accepting Warranty or Support. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and + on Your sole responsibility, not on behalf of any other Contributor, + and only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, + such Contributor by reason of your accepting any such warranty or support. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. Don't include + the brackets! The text should be enclosed in comments for the + purpose of the relevant file format. We also recommend that a + file or class name and description of purpose be included on the + same "license" line as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 P.aïmon + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/docs/screenshots/.gitkeep b/docs/screenshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/screenshots/action-builder.png b/docs/screenshots/action-builder.png new file mode 100644 index 0000000..b9073a8 Binary files /dev/null and b/docs/screenshots/action-builder.png differ diff --git a/docs/screenshots/dashboard.png b/docs/screenshots/dashboard.png new file mode 100644 index 0000000..7f056b6 Binary files /dev/null and b/docs/screenshots/dashboard.png differ diff --git a/docs/screenshots/host-detail.png b/docs/screenshots/host-detail.png new file mode 100644 index 0000000..c3c817d Binary files /dev/null and b/docs/screenshots/host-detail.png differ diff --git a/docs/screenshots/hosts.png b/docs/screenshots/hosts.png new file mode 100644 index 0000000..633b1a9 Binary files /dev/null and b/docs/screenshots/hosts.png differ diff --git a/docs/screenshots/nuclei-pop.png b/docs/screenshots/nuclei-pop.png new file mode 100644 index 0000000..baade74 Binary files /dev/null and b/docs/screenshots/nuclei-pop.png differ diff --git a/docs/screenshots/nuclei-results.png b/docs/screenshots/nuclei-results.png new file mode 100644 index 0000000..f022e97 Binary files /dev/null and b/docs/screenshots/nuclei-results.png differ diff --git a/docs/screenshots/ready.png b/docs/screenshots/ready.png new file mode 100644 index 0000000..7637a65 Binary files /dev/null and b/docs/screenshots/ready.png differ diff --git a/docs/screenshots/scanners.png b/docs/screenshots/scanners.png new file mode 100644 index 0000000..194761d Binary files /dev/null and b/docs/screenshots/scanners.png differ diff --git a/docs/screenshots/settings-blacklist.png b/docs/screenshots/settings-blacklist.png new file mode 100644 index 0000000..30c8756 Binary files /dev/null and b/docs/screenshots/settings-blacklist.png differ diff --git a/docs/screenshots/settings-players.png b/docs/screenshots/settings-players.png new file mode 100644 index 0000000..1713cf2 Binary files /dev/null and b/docs/screenshots/settings-players.png differ diff --git a/docs/screenshots/setup.png b/docs/screenshots/setup.png new file mode 100644 index 0000000..0375a32 Binary files /dev/null and b/docs/screenshots/setup.png differ diff --git a/docs/screenshots/tags.png b/docs/screenshots/tags.png new file mode 100644 index 0000000..f87e025 Binary files /dev/null and b/docs/screenshots/tags.png differ diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..491d57f --- /dev/null +++ b/install.sh @@ -0,0 +1,16 @@ +# /bin/bash +echo "\n" +echo "\n Preparing system for ROFMAD \!" +echo "\n" +sudo apt update +sudo apt install nmap masscan arp-scan tcpdump +echo "\n" +echo "\n Compiling ROFMAD \!" +echo "\n" +rustup target add x86_64-unknown-linux-musl +echo "\n" +cargo build --release --target x86_64-unknown-linux-musl +echo "\n" +cp target/x86_64-unknown-linux-musl/release/ROFMAD ./ +echo "\n" +echo "\n ROFMAD is now ready to launch \!" \ No newline at end of file diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100755 index 0000000..a16123e --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1,3 @@ +// src/config/mod.rs - Configuration management for ROFMAD system paths and settings + +pub mod paths; \ No newline at end of file diff --git a/src/config/paths.rs b/src/config/paths.rs new file mode 100755 index 0000000..95aa02f --- /dev/null +++ b/src/config/paths.rs @@ -0,0 +1,62 @@ +// src/config/paths.rs - Centralized system path management + +use std::path::{Path, PathBuf}; +use std::fs; + +/// System-wide path configuration +pub struct SystemPaths; + +impl SystemPaths { + /// Workflows directory + pub const WORKFLOWS_DIRECTORY: &'static str = "/opt/ROFMAD/workflows"; +} + +fn db_screenshots_base(db_path: &str) -> String { + Path::new(db_path) + .parent() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| ".".to_string()) +} + +/// Get the screenshots directory for a specific project run (relative to db location) +pub fn get_screenshots_directory(db_path: &str, run_id: &str) -> String { + format!("{}/screenshots/{}", db_screenshots_base(db_path), run_id) +} + +/// Get the full path for a screenshot file within a project +pub fn get_screenshot_file_path(db_path: &str, run_id: &str, filename: &str) -> String { + format!("{}/{}", get_screenshots_directory(db_path, run_id), filename) +} + +/// Ensure the screenshots directory exists for a specific run +pub fn ensure_screenshots_directory_for_run(db_path: &str, run_id: &str) -> Result> { + let dir_path = get_screenshots_directory(db_path, run_id); + fs::create_dir_all(&dir_path) + .map_err(|e| format!("Failed to create screenshots directory {}: {}", dir_path, e))?; + Ok(dir_path) +} + +/// Validate that a screenshot file exists within the project's run directory +pub fn validate_screenshot_access(db_path: &str, run_id: &str, filename: &str) -> bool { + let file_path = get_screenshot_file_path(db_path, run_id, filename); + PathBuf::from(&file_path).exists() +} + +/// Count screenshots in a specific run directory +pub fn count_screenshots_for_run(db_path: &str, run_id: &str) -> usize { + let screenshots_dir = get_screenshots_directory(db_path, run_id); + + fs::read_dir(&screenshots_dir) + .map(|entries| { + entries + .flatten() + .filter(|entry| { + entry.file_name() + .to_str() + .map(|name| name.ends_with(".png")) + .unwrap_or(false) + }) + .count() + }) + .unwrap_or(0) +} diff --git a/src/enumeration/dns.rs b/src/enumeration/dns.rs new file mode 100755 index 0000000..e6969d6 --- /dev/null +++ b/src/enumeration/dns.rs @@ -0,0 +1,146 @@ +use std::error::Error; +use tokio::{fs, sync::mpsc::UnboundedSender}; +use crate::sql::ProjectContext; +use crate::utilities::print_enum; +use super::utils::guess_subnet_str; + +fn parse_nameservers_from_resolv(content: &str) -> Vec { + content.lines() + .filter(|l| l.starts_with("nameserver ")) + .filter_map(|l| l.split_whitespace().nth(1).map(str::to_string)) + .collect() +} + +// 127.x.x.x covers Docker embedded DNS (127.0.0.11), systemd-resolved stub (127.0.0.53), +// and any other loopback proxy — none of these are actual network DNS servers. +fn is_stub_resolver(ip: &str) -> bool { + ip.starts_with("127.") +} + +// Returns the first batch of real (non-loopback) nameserver IPs found across three sources. +async fn collect_real_nameservers(raw_tx: &Option>) -> Vec { + // Source 1: /etc/resolv.conf + if let Ok(content) = fs::read_to_string("/etc/resolv.conf").await { + let servers: Vec = parse_nameservers_from_resolv(&content) + .into_iter().filter(|ip| !is_stub_resolver(ip)).collect(); + if !servers.is_empty() { + if let Some(tx) = raw_tx { + let _ = tx.send("[INFO] Nameservers from /etc/resolv.conf".to_string()); + } + return servers; + } + if let Some(tx) = raw_tx { + let _ = tx.send("[INFO] /etc/resolv.conf contains only stub resolvers (Docker environment), trying fallbacks...".to_string()); + } + } + + // Source 2: /run/systemd/resolve/resolv.conf — real upstream on systemd-resolved hosts. + // Silently skipped if not present (normal inside Docker containers). + if let Ok(content) = fs::read_to_string("/run/systemd/resolve/resolv.conf").await { + let servers: Vec = parse_nameservers_from_resolv(&content) + .into_iter().filter(|ip| !is_stub_resolver(ip)).collect(); + if !servers.is_empty() { + if let Some(tx) = raw_tx { + let _ = tx.send("[INFO] Nameservers from /run/systemd/resolve/resolv.conf".to_string()); + } + return servers; + } + } + + // Source 3: ip route show default — extract gateway as DNS heuristic. + // `ip` (iproute2) is available on every Linux including Exegol. + // The default gateway is very often the DNS server on pentesting target networks. + if let Ok(output) = tokio::process::Command::new("ip") + .args(["route", "show", "default"]) + .output().await + { + let text = String::from_utf8_lossy(&output.stdout); + // Line format: "default via X.X.X.X dev ethN ..." + let gateways: Vec = text.lines() + .filter(|l| l.starts_with("default via")) + .filter_map(|l| l.split_whitespace().nth(2).map(str::to_string)) + .filter(|ip| !is_stub_resolver(ip)) + .filter(|ip| ip.parse::().is_ok()) + .collect(); + if !gateways.is_empty() { + if let Some(tx) = raw_tx { + let _ = tx.send(format!( + "[INFO] Using default gateway(s) as DNS candidates (heuristic): {}", + gateways.join(", ") + )); + } + return gateways; + } + } + + if let Some(tx) = raw_tx { + let _ = tx.send("[WARNING] No DNS nameservers discovered from any source".to_string()); + } + Vec::new() +} + +// Context: DNS nameserver discovery for network reconnaissance +// Operation: Collects real nameserver IPs via fallback chain and derives associated subnets +pub async fn local_dns_search_enhanced( + ctx: &ProjectContext, + raw_tx: Option>, +) -> Result<(Vec, Vec), Box> { + if let Some(tx) = &raw_tx { + let _ = tx.send("[INFO] Starting local DNS discovery...".to_string()); + } + print_enum("local_dns_search: Starting local DNS search"); + + let mut discovered_ips = Vec::new(); + let mut discovered_subnets = Vec::new(); + + let nameservers = collect_real_nameservers(&raw_tx).await; + let nameserver_count = nameservers.len(); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INFO] Found {} nameserver(s) to process", nameserver_count)); + } + + for (index, ns) in nameservers.iter().enumerate() { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INFO] Processing nameserver {} of {}: {}", index + 1, nameserver_count, ns)); + } + match ns.parse::() { + Ok(ipv4) => { + let ip_str = ipv4.to_string(); + if !ctx.is_blacklisted(&ip_str).await? { + discovered_ips.push(ip_str.clone()); + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[SUCCESS] Discovered DNS server: {}", ip_str)); + } + if let Ok(net) = guess_subnet_str(&ip_str) { + let base = net.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + discovered_subnets.push(net.clone()); + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INFO] Derived subnet: {}", net)); + } + } + } + } else if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[WARNING] DNS server {} is blacklisted, skipping", ip_str)); + } + } + Err(_) => { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INFO] Skipping non-IPv4 nameserver: {}", ns)); + } + } + } + } + + discovered_ips.sort(); + discovered_ips.dedup(); + discovered_subnets.sort(); + discovered_subnets.dedup(); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[COMPLETE] Discovery finished: {} IPs, {} subnets", + discovered_ips.len(), discovered_subnets.len())); + } + Ok((discovered_ips, discovered_subnets)) +} diff --git a/src/enumeration/masscan.rs b/src/enumeration/masscan.rs new file mode 100755 index 0000000..fd25c04 --- /dev/null +++ b/src/enumeration/masscan.rs @@ -0,0 +1,293 @@ +use std::{collections::HashSet, error::Error, net::IpAddr, process::Stdio, sync::atomic::Ordering, time::Duration}; +use ipnetwork::IpNetwork; +use tokio::{ + io::{AsyncReadExt, BufReader}, + process::Command, + sync::mpsc::UnboundedSender, +}; +use regex::Regex; +use crate::workflow::{MASSCAN_BLOCK_HOOKS, MASSCAN_STATUS, dispatch_event, Hook}; +use crate::sql::ProjectContext; +use crate::utilities::{print_enum, tag_debug, tag_info}; +use super::utils::guess_subnet_str; + +fn ip_in_subnets(ip: &str, targets: &[String]) -> bool { + let Ok(addr) = ip.parse::() else { return false; }; + targets.iter().any(|t| { + t.parse::().map(|net| net.contains(addr)).unwrap_or(false) + }) +} + +// Context: High-speed subnet scanning using Masscan tool +// Operation: Executes masscan on provided subnets, parses output for discovered hosts, tracks progress +pub async fn masscan_discovery_non_interactive( + ctx: &ProjectContext, + subnets: Vec, + raw_tx: Option>, +) -> Result<(), Box> { + MASSCAN_BLOCK_HOOKS.store(true, Ordering::SeqCst); + struct Guard; + impl Drop for Guard { + fn drop(&mut self) { + MASSCAN_BLOCK_HOOKS.store(false, Ordering::SeqCst); + } + } + let _guard = Guard; + + print_enum("masscan_discovery: Launching fast host discovery"); + + let iface = ctx.get_selected_interface().await?; + + let scan_targets = subnets; + + if scan_targets.is_empty() { + tag_info("No subnets provided. Skipping masscan."); + return Ok(()); + } + + for subnet in &scan_targets { + ctx.insert_subnet_no_hook(subnet).await.ok(); + } + + let ports = vec![21,22,80,139,443,445]; + + // Get blacklisted ranges to exclude from scan + let excludes = ctx.get_masscan_excludes().await?; + let blacklist_cidrs = excludes.clone(); + + let mut masscan_cmd = Command::new("masscan"); + masscan_cmd + .arg(scan_targets.join(",")) + .arg("-p") + .arg(ports.iter().map(ToString::to_string).collect::>().join(",")) + .arg("-vvv") + .arg("--rate") + .arg("10000") + .arg("--interface") + .arg(&iface); + + // Add exclude arguments if blacklist exists + if !excludes.is_empty() { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[SECURITY] Excluding {} blacklisted ranges from scan", excludes.len())); + for exclude in &excludes { + let _ = tx.send(format!("[SECURITY] Excluding: {}", exclude)); + } + } + tag_info(&format!("Masscan excluding {} blacklisted ranges", excludes.len())); + + // Add --exclude argument with comma-separated list + masscan_cmd + .arg("--exclude") + .arg(excludes.join(",")); + } else { + if let Some(tx) = &raw_tx { + let _ = tx.send("[SECURITY] No blacklisted ranges to exclude".to_string()); + } + } + + let mut child = masscan_cmd + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let mut out_buf = BufReader::new(child.stdout.take().unwrap()); + let mut err_buf = BufReader::new(child.stderr.take().unwrap()); + + let re_discover = Regex::new(r"Discovered open port \d+/tcp on ([0-9.]+)")?; + let re_status = Regex::new(r"([\d.]+)%\s*done")?; + let re_arp = Regex::new(r"^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\s+:")?; + + let mut loop_iteration_count = 0; + + let mut tmp_stdout = Vec::new(); + let mut tmp_stderr = Vec::new(); + let mut last_pct: Option = None; + let mut seen_ips = HashSet::new(); + let mut seen_lines = HashSet::new(); + + loop { + loop_iteration_count += 1; + if loop_iteration_count % 1000 == 0 { + tokio::task::yield_now().await; + } + + tokio::select! { + result = out_buf.read_u8() => { + let byte = match result { + Ok(b) => b, + Err(_) => break, + }; + if byte == b'\r' || byte == b'\n' { + let line = String::from_utf8_lossy(&tmp_stdout).to_string(); + tmp_stdout.clear(); + + if let Some(tx) = &raw_tx { + if seen_lines.insert(line.clone()) { + let _ = tx.send(line.clone()); + } + } + + if let Some(cap) = re_status.captures(&line) { + let pct = cap[1].parse::().unwrap_or(0.0).round() as u8; + *MASSCAN_STATUS.write().unwrap() = Some(format!("{}%", pct)); + if last_pct != Some(pct) { + last_pct = Some(pct); + if pct < 100 { + tag_info(&format!("Masscan progress: {}%", pct)); + } else { + tag_info("Masscan completed 100%"); + } + } + if pct >= 100 { + tag_info("Masscan reached 100%, initiating exit sequence"); + tokio::time::sleep(Duration::from_millis(500)).await; + break; + } + } + + if let Some(cap) = re_discover.captures(&line) { + let ip = cap[1].to_string(); + if seen_ips.insert(ip.clone()) { + if !ip_in_subnets(&ip, &scan_targets) { + tag_debug(&format!("Filtered out-of-scope host: {}", ip)); + } else if ip_in_subnets(&ip, &blacklist_cidrs) { + tag_debug(&format!("Filtered blacklisted host: {}", ip)); + } else { + match ctx.insert_ip(&ip).await { + Ok(true) => { + tag_info(&format!("Discovered new target: {}", ip)); + if let Ok(net) = guess_subnet_str(&ip) { + ctx.insert_subnet_force(&net).await?; + } + } + Ok(false) => {} + Err(e) => tag_debug(&format!("Failed to record {}: {}", ip, e)), + } + } + } + } + + if let Some(cap) = re_arp.captures(&line) { + let ip = cap[1].to_string(); + if seen_ips.insert(ip.clone()) { + if !ip_in_subnets(&ip, &scan_targets) { + tag_debug(&format!("[ARP] Filtered out-of-scope host: {}", ip)); + } else if ip_in_subnets(&ip, &blacklist_cidrs) { + tag_debug(&format!("[ARP] Filtered blacklisted host: {}", ip)); + } else { + match ctx.insert_ip(&ip).await { + Ok(true) => { + tag_info(&format!("[ARP] Discovered alive host: {}", ip)); + if let Ok(net) = guess_subnet_str(&ip) { + ctx.insert_subnet_force(&net).await?; + } + } + Ok(false) => {} + Err(e) => tag_debug(&format!("Failed to record {}: {}", ip, e)), + } + } + } + } + } else { + tmp_stdout.push(byte); + } + } + + result = err_buf.read_u8() => { + let byte = match result { + Ok(b) => b, + Err(_) => break, + }; + if byte == b'\r' || byte == b'\n' { + let line = String::from_utf8_lossy(&tmp_stderr).to_string(); + tmp_stderr.clear(); + + if let Some(tx) = &raw_tx { + if seen_lines.insert(line.clone()) { + let _ = tx.send(line.clone()); + } + } + + if let Some(cap) = re_status.captures(&line) { + let pct = cap[1].parse::().unwrap_or(0.0).round() as u8; + *MASSCAN_STATUS.write().unwrap() = Some(format!("{}%", pct)); + if last_pct != Some(pct) { + last_pct = Some(pct); + if pct >= 100 { + tag_info("Masscan completed 100% (stderr)"); + } + } + if pct >= 100 { + tag_info("Masscan reached 100% (stderr), initiating exit sequence"); + tokio::time::sleep(Duration::from_millis(500)).await; + break; + } + } + + if let Some(cap) = re_discover.captures(&line) { + let ip = cap[1].to_string(); + if seen_ips.insert(ip.clone()) { + if !ip_in_subnets(&ip, &scan_targets) { + tag_debug(&format!("Filtered out-of-scope host: {}", ip)); + } else if ip_in_subnets(&ip, &blacklist_cidrs) { + tag_debug(&format!("Filtered blacklisted host: {}", ip)); + } else { + match ctx.insert_ip(&ip).await { + Ok(true) => { + tag_info(&format!("Discovered new target: {}", ip)); + if let Ok(net) = guess_subnet_str(&ip) { + ctx.insert_subnet_force(&net).await?; + } + } + Ok(false) => {} + Err(e) => tag_debug(&format!("Failed to record {}: {}", ip, e)), + } + } + } + } + + if let Some(cap) = re_arp.captures(&line) { + let ip = cap[1].to_string(); + if seen_ips.insert(ip.clone()) { + if !ip_in_subnets(&ip, &scan_targets) { + tag_debug(&format!("[ARP] Filtered out-of-scope host: {}", ip)); + } else if ip_in_subnets(&ip, &blacklist_cidrs) { + tag_debug(&format!("[ARP] Filtered blacklisted host: {}", ip)); + } else { + match ctx.insert_ip(&ip).await { + Ok(true) => { + tag_info(&format!("[ARP] Discovered alive host: {}", ip)); + if let Ok(net) = guess_subnet_str(&ip) { + ctx.insert_subnet_force(&net).await?; + } + } + Ok(false) => {} + Err(e) => tag_debug(&format!("Failed to record {}: {}", ip, e)), + } + } + } + } + } else { + tmp_stderr.push(byte); + } + } + } + } + + let _ = child.kill().await; + let _ = child.wait().await; + + *MASSCAN_STATUS.write().unwrap() = None; + tag_info("Masscan discovery complete."); + + for subnet in &scan_targets { + ctx.mark_masscan_scanned(subnet).await?; + // Trigger Phase 1 now that masscan-discovered hosts are in the DB. + // Phase 1 excludes all known IPs (including masscan finds) via seen_global, + // then fires SubnetPhase1Complete → Phase 2 when it completes. + dispatch_event(Hook::Subnet, subnet, ctx); + } + + Ok(()) +} \ No newline at end of file diff --git a/src/enumeration/mod.rs b/src/enumeration/mod.rs new file mode 100755 index 0000000..511938c --- /dev/null +++ b/src/enumeration/mod.rs @@ -0,0 +1,24 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod dns; +pub mod masscan; +pub mod network; +pub mod nmap; +pub mod passive; +pub mod subnet; +pub mod utils; +pub use nmap::*; +pub use subnet::*; +pub use utils::*; \ No newline at end of file diff --git a/src/enumeration/network.rs b/src/enumeration/network.rs new file mode 100755 index 0000000..046b5f4 --- /dev/null +++ b/src/enumeration/network.rs @@ -0,0 +1,235 @@ +use std::{error::Error, process::Stdio}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, + sync::mpsc::UnboundedSender, +}; +use regex::Regex; +use crate::sql::ProjectContext; +use crate::utilities::print_enum; +use super::utils::guess_subnet_str; + +// Context: Network interface discovery for reconnaissance operations +// Operation: Analyzes selected interface to discover IPs, gateways, and subnets from routing tables +pub async fn interface_explorer( + ctx: &ProjectContext, + raw_tx: Option>, +) -> Result<(Vec, Vec), Box> { + let selected_interface = ctx.get_selected_interface().await?; + + if let Some(tx) = &raw_tx { + let _ = tx.send("[INFO] Starting interface and route discovery...".to_string()); + let _ = tx.send(format!("[INFO] Analyzing selected interface: {}", selected_interface)); + } + + print_enum(&format!("interface_explorer: Analyzing interface {}", selected_interface)); + + let mut discovered_ips = Vec::new(); + let mut discovered_subnets = Vec::new(); + + let out = Command::new("ip") + .arg("-o") + .arg("-4") + .arg("addr") + .arg("show") + .arg("dev") + .arg(&selected_interface) + .output() + .await?; + + let raw = String::from_utf8_lossy(&out.stdout); + let re = Regex::new(r"\d+:\s+(\S+)\s+inet\s+([0-9\.]+/\d+)")?; + let mut interface_cidr = None; + + for line in raw.lines() { + if let Some(cap) = re.captures(line) { + let cidr = cap[2].to_string(); + interface_cidr = Some(cidr.clone()); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INTERFACE] {} -> {}", selected_interface, cidr)); + } + } + } + + let cidr = match interface_cidr { + Some(c) => c, + None => { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[ERROR] No IPv4 configuration found for interface {}", selected_interface)); + } + return Err(format!("Interface {} has no IPv4 configuration", selected_interface).into()); + } + }; + + let out = Command::new("ip") + .arg("route") + .arg("show") + .arg("default") + .output() + .await?; + + let route = String::from_utf8_lossy(&out.stdout); + let gw_re = Regex::new(&format!(r"default via ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+).*dev {}", selected_interface))?; + + if let Some(cap) = gw_re.captures(&route) { + let gw = cap[1].to_string(); + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[GATEWAY] Default gateway via {}: {}", selected_interface, gw)); + } + + if !ctx.is_blacklisted(&gw).await? { + discovered_ips.push(gw.clone()); + + let prefix = cidr.split('/').nth(1).unwrap_or("24"); + let full = format!("{}/{}", gw, prefix); + if let Ok(net) = guess_subnet_str(&full) { + let base = net.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + discovered_subnets.push(net.clone()); + } + } + } + } + + let host_ip = cidr.split('/').next().unwrap(); + + let out = Command::new("ip") + .arg("addr") + .arg("show") + .arg("dev") + .arg(&selected_interface) + .output() + .await?; + + let dev = String::from_utf8_lossy(&out.stdout); + let ip_re = Regex::new(r"inet ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)")?; + + for line in dev.lines() { + if let Some(cap) = ip_re.captures(line) { + let ip = cap[1].to_string(); + if ip != host_ip && !ctx.is_blacklisted(&ip).await? { + discovered_ips.push(ip.clone()); + + let prefix = cidr.split('/').nth(1).unwrap_or("24"); + let full = format!("{}/{}", ip, prefix); + if let Ok(net) = guess_subnet_str(&full) { + let base = net.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + discovered_subnets.push(net); + } + } + } + } + } + + let out = Command::new("ip") + .arg("route") + .arg("show") + .arg("dev") + .arg(&selected_interface) + .output() + .await?; + + let rt = String::from_utf8_lossy(&out.stdout); + let rt_re = Regex::new(r"([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/\d+)")?; + + for line in rt.lines() { + if let Some(cap) = rt_re.captures(line) { + let subnet = cap[1].to_string(); + let base = subnet.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + discovered_subnets.push(subnet); + } + } + } + + discovered_ips.sort(); + discovered_ips.dedup(); + discovered_subnets.sort(); + discovered_subnets.dedup(); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[COMPLETE] Interface analysis complete: {} IPs, {} subnets discovered", discovered_ips.len(), discovered_subnets.len())); + } + + Ok((discovered_ips, discovered_subnets)) +} + +// Context: Local network enumeration using ARP scanning +// Operation: Executes arp-scan on selected interface to discover live hosts and populate database +pub async fn enumerate_arp_scan( + ctx: &ProjectContext, + raw_tx: Option>, +) -> Result<(), Box> { + print_enum("enumerate_arp_scan: Starting ARP scan"); + + let iface = ctx.get_selected_interface().await?; + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[INFO] Starting ARP scan on interface: {}", iface)); + } + + let mut child = Command::new("sudo") + .arg("arp-scan") + .arg("--interface").arg(&iface) + .arg("--localnet") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let mut out_reader = BufReader::new(child.stdout.take().unwrap()).lines(); + let mut err_reader = BufReader::new(child.stderr.take().unwrap()).lines(); + let re = Regex::new(r"^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\s+").unwrap(); + + loop { + tokio::select! { + maybe = out_reader.next_line() => match maybe? { + Some(line) => { + if let Some(tx) = &raw_tx { let _ = tx.send(line.clone()); } + if let Some(cap) = re.captures(&line) { + let ip = &cap[1]; + if !ctx.is_blacklisted(ip).await? { + ctx.insert_ip(ip).await.ok(); + let net = guess_subnet_str(ip)?; + let base = net.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + ctx.insert_subnet(&net).await.ok(); + } + } + } + } + None => break, + }, + + maybe_e = err_reader.next_line() => match maybe_e? { + Some(err_line) => { + if let Some(tx) = &raw_tx { let _ = tx.send(err_line.clone()); } + } + None => break, + }, + } + } + + let exit_status = child.wait().await?; + + // Give user time to see the results, especially if there was an error + if let Some(tx) = &raw_tx { + if exit_status.success() { + let _ = tx.send("[INFO] ARP scan completed successfully".to_string()); + } else { + let _ = tx.send("[ERROR] ARP scan failed - check output above for details".to_string()); + let _ = tx.send("[INFO] Common issues: interface not compatible, missing sudo permissions, or arp-scan not installed".to_string()); + } + let _ = tx.send("[INFO] Task will remain visible for 10 seconds...".to_string()); + } + + // Keep task visible for 10 seconds so user can read the output/error + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + + if let Some(tx) = &raw_tx { + let _ = tx.send("[INFO] ARP scan task completed".to_string()); + } + + Ok(()) +} \ No newline at end of file diff --git a/src/enumeration/nmap.rs b/src/enumeration/nmap.rs new file mode 100755 index 0000000..64cf649 --- /dev/null +++ b/src/enumeration/nmap.rs @@ -0,0 +1,225 @@ +use std::{error::Error, time::Duration}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, + sync::mpsc::UnboundedSender, +}; +use regex::Regex; +use crate::tag::{automatic_tag, TriggerContext}; +use crate::sql::ProjectContext; +use crate::utilities::{print_enum, tag_debug, tag_info}; +use crate::workflow::{complete_port_scan_workflow, handle_nmap_timeout, dispatch_event, Hook}; + +// Context: Complete TCP port scanning for comprehensive host enumeration +// Operation: Executes nmap full port scan (-p-) with timeout handling and port discovery +pub async fn nmap_full_port_detection_scan( + ctx: &ProjectContext, + ip: &str, + raw_tx: Option>, +) -> Result<(), Box> { + if !ctx.mark_full_scan_started(ip).await? { + tag_debug(&format!("Full port scan already started for {}, skipping", ip)); + return Ok(()); + } + + print_enum(&format!("nmap_full_port_detection_scan: scan on {}", ip)); + + // Full scan: port discovery only. No -oX — we don't need XML here. + // Only the sV scan (nmap_service_scan) writes to nmap_xml / nmap_scan. + let mut child = Command::new("nmap") + .arg("-p").arg("-") + .arg("-Pn") + .arg("--open") + .arg("-v") + .arg("-T4") + .arg("--min-parallelism").arg("50") + .arg("--max-retries").arg("2") + .arg(ip) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child.stdout.take().unwrap(); + let mut lines = BufReader::new(stdout).lines(); + let re = Regex::new(r"(?i)open port (\d+)/tcp")?; + + tokio::select! { + scan_result = async { + let mut found_ports: Vec = Vec::new(); + while let Some(line) = lines.next_line().await? { + if let Some(tx) = &raw_tx { + let _ = tx.send(line.clone()); + } + + if let Some(cap) = re.captures(&line) { + found_ports.push(cap[1].to_string()); + } + } + + // Single batch insert instead of per-port inserts + if !found_ports.is_empty() { + ctx.insert_ports(ip, &found_ports.join(",")).await.ok(); + } + + let status = child.wait().await?; + if !status.success() { + tag_debug(&format!("nmap_full_port_detection_scan on {} exited {:?}", ip, status)); + } + + Ok::<(), Box>(()) + } => { + if let Err(e) = scan_result { + tag_info(&format!("nmap_full_port_detection_scan error on {}: {}", ip, e)); + } + }, + + _ = tokio::time::sleep(Duration::from_secs(900)) => { + tag_info(&format!("nmap_full_port_detection_scan timed out on {} after 900s", ip)); + + let _ = child.kill().await; + + handle_nmap_timeout(ctx, ip).await; + } + } + + complete_port_scan_workflow(ctx, ip).await +} + +// Context: Service and version enumeration on discovered open ports +// Operation: Runs nmap service detection (-sV -sC) against known open ports and triggers tagging +pub async fn nmap_service_scan( + ctx: &ProjectContext, + ip: &str, + raw_tx: Option>, +) -> Result<(), Box> { + print_enum(&format!("nmap_service_scan: service scan on {}", ip)); + + ctx.mark_service_scan_started(ip).await?; + + let port_list = ctx.get_open_ports(ip).await?; + if port_list.is_empty() { + tag_info(&format!("nmap_service_scan: no ports to scan for {}", ip)); + + ctx.mark_service_scan_complete(ip).await?; + + dispatch_event(Hook::ServiceScanComplete, ip, ctx); + + if ctx.is_host_reconnaissance_complete(ip).await? { + dispatch_event(Hook::ReconnaissanceComplete, ip, ctx); + } + + return Ok(()); + } + let joined_ports = port_list.join(","); + + let xml_path_svc = format!("/tmp/rofmad_{}_{}_{}.xml", + &ctx.run_id[..8.min(ctx.run_id.len())], + ip.replace('.', "_").replace(':', "_"), + "svc" + ); + + let mut child = Command::new("nmap") + .arg("-sC") + .arg("-sV") + .arg("-Pn") + .arg("-T4") + .arg("--script-timeout").arg("60s") + .arg("--max-retries").arg("2") + .arg("-p") + .arg(&joined_ports) + .arg("-v") + .arg("-oX").arg(&xml_path_svc) + .arg(ip) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child.stdout.take().unwrap(); + let mut lines = BufReader::new(stdout).lines(); + let mut collected = String::new(); + + tokio::select! { + scan_result = async { + while let Some(line) = lines.next_line().await? { + if line.contains("PCRE_ERROR_MATCHLIMIT") { + continue; + } + if let Some(tx) = &raw_tx { + let _ = tx.send(line.clone()); + } + collected.push_str(&line); + collected.push('\n'); + } + + let status = child.wait().await?; + if !status.success() { + tag_debug(&format!("nmap_service_scan on {} exited {:?}", ip, status)); + } + + if !port_list.is_empty() { + ctx.insert_nmap_scan(ip, "service_scan", &collected).await?; + } + + // Store XML output alongside raw text for structured parsing + if let Ok(xml) = tokio::fs::read_to_string(&xml_path_svc).await { + ctx.insert_nmap_xml(ip, "service_scan", &xml).await.ok(); + + // Build port→service map from XML and persist for dynamic action targeting + if let Some(parsed) = crate::tag::parse_nmap_xml(&xml) { + let port_services: std::collections::HashMap = parsed.ports.iter() + .filter(|p| !p.service_name.is_empty() + && !crate::tag::nmap_parsing::is_noise_service(&p.service_name)) + .map(|p| { + let svc = crate::tag::nmap_parsing::normalize_service_name(&p.service_name); + // nmap labels WinRM ports (5985/5986) as "http" — normalize to "winrm" + // so has_service: "winrm" conditions match and {{SERVICE_PORT}} resolves correctly + let svc = match (p.port.as_str(), svc.as_str()) { + ("5985" | "5986", "http") => "winrm".to_string(), + _ => svc, + }; + (p.port.clone(), svc) + }) + .collect(); + if !port_services.is_empty() { + ctx.save_port_services(ip, &port_services).await.ok(); + } + } + } + tokio::fs::remove_file(&xml_path_svc).await.ok(); + + ctx.mark_service_scan_complete(ip).await?; + + // Run tagging synchronously before dispatching ServiceScanComplete so that + // automated_tags is fully written before HTTP discovery's auto_tag_http starts + // (both do a read-modify-write on automated_tags — concurrent writes lose tags). + if let Err(e) = automatic_tag(ctx, ip, TriggerContext::ServiceScanComplete).await { + tag_info(&format!("Service scan tagging failed for {}: {}", ip, e)); + } + + dispatch_event(Hook::ServiceScanComplete, ip, ctx); + + if ctx.is_host_reconnaissance_complete(ip).await? { + dispatch_event(Hook::ReconnaissanceComplete, ip, ctx); + } + + Ok::<(), Box>(()) + } => { + if let Err(e) = scan_result { + tag_info(&format!("nmap_service_scan error on {}: {}", ip, e)); + } + }, + + _ = tokio::time::sleep(Duration::from_secs(1800)) => { + tag_info(&format!("nmap_service_scan timed out on {} after 1800s", ip)); + let _ = child.kill().await; + tokio::fs::remove_file(&xml_path_svc).await.ok(); + ctx.mark_service_scan_complete(ip).await.ok(); + dispatch_event(Hook::ServiceScanComplete, ip, ctx); + if ctx.is_host_reconnaissance_complete(ip).await.unwrap_or(false) { + dispatch_event(Hook::ReconnaissanceComplete, ip, ctx); + } + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/enumeration/passive.rs b/src/enumeration/passive.rs new file mode 100755 index 0000000..129e6d9 --- /dev/null +++ b/src/enumeration/passive.rs @@ -0,0 +1,207 @@ +use std::{error::Error, process::Stdio, time::Duration}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, + sync::mpsc::UnboundedSender, +}; +use regex::Regex; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info}; +use super::utils::guess_subnet_str; + +// Context: Passive network traffic monitoring for stealth reconnaissance +// Operation: Uses tcpdump to capture mDNS, LLMNR, NetBIOS, SSDP traffic and discover hosts/hostnames +pub async fn passive_listener_enhanced( + ctx: &ProjectContext, + raw_tx: Option>, +) -> Result<(), Box> { + use std::collections::HashSet; + + let iface = ctx.get_selected_interface().await?; + + if let Some(tx) = &raw_tx { + let _ = tx.send("[INFO] Initializing passive network listener...".to_string()); + let _ = tx.send(format!("[INFO] Selected interface: {}", iface)); + + let interface_check = Command::new("ip") + .arg("link") + .arg("show") + .arg(&iface) + .output() + .await; + + match interface_check { + Ok(output) => { + let output_str = String::from_utf8_lossy(&output.stdout); + if output_str.contains("state UP") || output_str.contains("state UNKNOWN") { + let _ = tx.send(format!("[INFO] Interface {} is UP and available", iface)); + } else { + let _ = tx.send(format!("[WARNING] Interface {} may be DOWN", iface)); + } + } + Err(_) => { + let _ = tx.send(format!("[ERROR] Interface {} not found", iface)); + return Err(format!("Interface '{}' not found or not accessible", iface).into()); + } + } + + let _ = tx.send("[INFO] Monitoring protocols: mDNS, LLMNR, NetBIOS, SSDP, DNS, DHCP".to_string()); + let _ = tx.send("[INFO] Listener is now active - waiting for network traffic...".to_string()); + let _ = tx.send("".to_string()); + } + + tag_info(&format!("Listening on {} (Ctrl-C to stop)", iface)); + + if let Some(tx) = &raw_tx { + let _ = tx.send("[INFO] Testing interface for any network activity...".to_string()); + } + + let test_cmd = Command::new("sudo") + .arg("timeout") + .arg("5s") + .arg("tcpdump") + .arg("-n") + .arg("-i").arg(&iface) + .arg("-c").arg("3") + .output() + .await; + + match test_cmd { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + if let Some(tx) = &raw_tx { + if stdout.contains("packets captured") || stderr.contains("packets captured") { + let _ = tx.send("[INFO] Network interface is capturing traffic".to_string()); + } else if stderr.contains("permission denied") { + let _ = tx.send("[ERROR] Permission denied - sudo access required".to_string()); + return Err("Permission denied for packet capture. Ensure sudo access.".into()); + } else { + let _ = tx.send("[WARNING] No traffic detected in 5s test - network may be quiet".to_string()); + } + } + } + Err(e) => { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[WARNING] Network test failed: {}", e)); + } + } + } + + let mut child = Command::new("sudo") + .arg("tcpdump") + .arg("-n") + .arg("-l") + .arg("-v") + .arg("-i").arg(&iface) + .arg("(udp port 5355 or udp port 137 or udp port 5353 or udp port 1900 or udp port 53 or udp port 67 or udp port 68)") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + let mut out_reader = BufReader::new(child.stdout.take().unwrap()).lines(); + let mut err_reader = BufReader::new(child.stderr.take().unwrap()).lines(); + + let ip_re = Regex::new(r"IP ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.").unwrap(); + let host_re = Regex::new(r"(?i)\b([A-Za-z0-9][A-Za-z0-9_-]{1,31})\.local\b").unwrap(); + let mut seen: HashSet<(String, String)> = HashSet::new(); + let mut discovery_count = 0; + let mut total_packets = 0; + + let start_time = std::time::Instant::now(); + let mut _last_status = start_time; + + loop { + tokio::select! { + maybe = out_reader.next_line() => match maybe? { + Some(line) => { + total_packets += 1; + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[PACKET] {}", line)); + } + + if let Some(cap) = ip_re.captures(&line) { + let ip = cap[1].to_string(); + if !ctx.is_blacklisted(&ip).await? { + if ctx.insert_ip(&ip).await.ok().unwrap_or(false) { + discovery_count += 1; + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[DISCOVERY] New IP discovered: {} (Total: {})", ip, discovery_count)); + } + } + let net = guess_subnet_str(&ip)?; + let base = net.split('/').next().unwrap(); + if !ctx.is_blacklisted(base).await? { + ctx.insert_subnet(&net).await.ok(); + } + } + } + + if let Some(cap2) = host_re.captures(&line) { + let hostname = cap2[1].to_string(); + if hostname != "_tcp" && hostname.len() > 1 { + if ctx.insert_hostname(&hostname, &hostname).await.ok().unwrap_or(false) { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[DISCOVERY] New hostname discovered: {}", hostname)); + } + } + } + } + + let proto = if line.contains("5355") { "LLMNR" } + else if line.contains("5353") { "mDNS" } + else if line.contains("137") { "NetBIOS" } + else if line.contains("1900") { "SSDP" } + else if line.contains("53") { "DNS" } + else if line.contains("67")||line.contains("68") { "DHCP" } + else { "Other" }; + + if let Some(cap) = ip_re.captures(&line) { + let ip = cap[1].to_string(); + if seen.insert((ip.clone(), proto.to_string())) { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[ACTIVITY] {} traffic from {}", proto, ip)); + } + tag_info(&format!("{} activity from {}", proto, ip)); + } + } + } + None => break, + }, + + maybe_e = err_reader.next_line() => match maybe_e? { + Some(err_line) => { + if err_line.contains("tcpdump:") || err_line.contains("error") || err_line.contains("Error") { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[STDERR] {}", err_line)); + } + } + } + None => break, + }, + + _ = tokio::time::sleep(Duration::from_secs(30)) => { + let elapsed = start_time.elapsed(); + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[STATUS] Listening for {:?} - {} packets captured, {} discoveries", + elapsed, total_packets, discovery_count)); + } + _last_status = std::time::Instant::now(); + } + } + } + + let _ = child.kill().await; + + if let Some(tx) = &raw_tx { + let _ = tx.send("".to_string()); + let _ = tx.send("[INFO] Passive listener shutting down...".to_string()); + let _ = tx.send(format!("[SUMMARY] Total: {} packets, {} discoveries", total_packets, discovery_count)); + let _ = tx.send("[INFO] Passive listener stopped".to_string()); + } + + tag_info("Stopping passive listener…"); + Ok(()) +} \ No newline at end of file diff --git a/src/enumeration/subnet.rs b/src/enumeration/subnet.rs new file mode 100755 index 0000000..f1a478d --- /dev/null +++ b/src/enumeration/subnet.rs @@ -0,0 +1,183 @@ +use std::{collections::{HashMap, HashSet}, error::Error, process::Stdio}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, + sync::mpsc::UnboundedSender, +}; +use regex::Regex; +use ipnetwork::IpNetwork; +use crate::sql::ProjectContext; +use crate::utilities::{print_enum, tag_debug, tag_info}; +use crate::workflow::{dispatch_event, Hook}; +use super::utils::guess_subnet_str; + +// Context: Initial subnet reconnaissance with targeted port scanning +// Operation: Executes quick nmap scan on common ports for /24 subnets, excluding known hosts +pub async fn subnet_phase1_scan( + ctx: &ProjectContext, + subnet: String, + raw_tx: Option>, +) -> Result<(), Box> { + if ctx.is_subnet_phase_complete(&subnet, "subnet_phase1").await.unwrap_or(false) { + tag_debug(&format!("Subnet Phase 1 already complete for {}, skipping", subnet)); + return Ok(()); + } + + print_enum(&format!("subnet_phase1_scan: Quick scan on {}", subnet)); + + let net: IpNetwork = subnet.parse()?; + let forced = ctx.is_subnet_marked_rescan(&subnet).await?; + + if !forced && net.prefix() != 24 { + tag_debug(&format!("Skipping subnet phase 1 on non-/24 {}", subnet)); + return Ok(()); + } + + ctx.mark_explorer_scanned(&subnet).await?; + + let seen_global: HashSet = ctx.fetch_all_ips().await?.into_iter().collect(); + let blacklist_cidrs = ctx.get_masscan_excludes().await.unwrap_or_default(); + + let quick_ports = vec![21,22,80,110,135,139,443,445,3389,3306,5432,8000,8080,8443]; + let quick_arg = quick_ports.iter() + .map(ToString::to_string) + .collect::>() + .join(","); + + let mut cmd1 = Command::new("nmap"); + cmd1.arg("--privileged") + .arg("-p").arg(&quick_arg) + .arg("-Pn") + .arg("-v") + .arg("-T4") + .arg("--open") + .arg("--max-retries").arg("2"); + + let mut excl_list: Vec = seen_global.iter().cloned().collect(); + excl_list.extend(blacklist_cidrs.iter().cloned()); + if !excl_list.is_empty() { + cmd1.arg("--exclude").arg(&excl_list.join(",")); + } + + let mut child1 = cmd1.arg(&subnet).stdout(Stdio::piped()).spawn()?; + let stdout1 = child1.stdout.take().unwrap(); + let mut lines1 = BufReader::new(stdout1).lines(); + let re_open = Regex::new(r"(?i)open port \d+/tcp on ([0-9.]+)")?; + + let mut phase1_hosts = Vec::new(); + while let Some(line) = lines1.next_line().await? { + if let Some(tx) = &raw_tx { + let _ = tx.send(line.clone()); + } + if let Some(cap) = re_open.captures(&line) { + let ip = cap[1].to_string(); + if !seen_global.contains(&ip) { + ctx.insert_ip(&ip).await?; + if let Ok(cidr) = guess_subnet_str(&ip) { + ctx.insert_subnet(&cidr).await?; + } + phase1_hosts.push(ip); + } + } + } + + let status1 = child1.wait().await?; + if !status1.success() { + tag_debug(&format!("Phase 1 nmap on {} exited {:?}", subnet, status1)); + } + + ctx.update_phase1_hosts(&subnet, &phase1_hosts).await?; + ctx.mark_subnet_phase_complete(&subnet, "subnet_phase1").await?; + tag_info(&format!("Subnet Phase 1 complete for {} - {} new hosts found", subnet, phase1_hosts.len())); + + dispatch_event(Hook::SubnetPhase1Complete, &subnet, ctx); + + Ok(()) +} + +// Context: Comprehensive subnet enumeration for complete port coverage +// Operation: Performs full port scan (-p-) on subnet, excluding phase1 and known hosts +pub async fn subnet_phase2_scan( + ctx: &ProjectContext, + subnet: String, + raw_tx: Option>, +) -> Result<(), Box> { + if ctx.is_subnet_phase_complete(&subnet, "subnet_phase2").await.unwrap_or(false) { + tag_debug(&format!("Subnet Phase 2 already complete for {}, skipping", subnet)); + return Ok(()); + } + + print_enum(&format!("subnet_phase2_scan: Full scan on {}", subnet)); + + let seen_global: HashSet = ctx.fetch_all_ips().await?.into_iter().collect(); + let blacklist_cidrs = ctx.get_masscan_excludes().await.unwrap_or_default(); + let existing_phase1 = ctx.fetch_subnets_phase1().await? + .into_iter() + .find(|(s, _)| s == &subnet) + .map(|(_, h)| h) + .unwrap_or_default(); + + let mut all_excl: Vec = seen_global.into_iter().collect(); + all_excl.extend(existing_phase1.iter().cloned()); + all_excl.extend(blacklist_cidrs.iter().cloned()); + + let mut cmd2 = Command::new("nmap"); + cmd2.arg("--privileged") + .arg("-p-") + .arg("-Pn") + .arg("-v") + .arg("-T4") + .arg("--open") + .arg("--max-retries").arg("2"); + + if !all_excl.is_empty() { + cmd2.arg("--exclude").arg(&all_excl.join(",")); + } + + let mut child2 = cmd2.arg(&subnet).stdout(Stdio::piped()).spawn()?; + let stdout2 = child2.stdout.take().unwrap(); + let mut lines2 = BufReader::new(stdout2).lines(); + let re_full = Regex::new(r"(?i)open port (\d+)/tcp on ([0-9.]+)")?; + + let mut new_ports_by_host: HashMap> = HashMap::new(); + + while let Some(line) = lines2.next_line().await? { + if let Some(tx) = &raw_tx { + let _ = tx.send(line.clone()); + } + if let Some(cap) = re_full.captures(&line) { + let port = cap[1].to_string(); + let ip = cap[2].to_string(); + + if ctx.insert_ip_no_hook(&ip).await? { + if let Ok(cidr) = guess_subnet_str(&ip) { + ctx.insert_subnet(&cidr).await?; + } + } + if ctx.insert_ports(&ip, &port).await? { + new_ports_by_host.entry(ip).or_default().push(port); + } + } + } + + let status2 = child2.wait().await?; + if !status2.success() { + tag_debug(&format!("Phase 2 nmap on {} exited {:?}", subnet, status2)); + } + + ctx.mark_subnet_phase_complete(&subnet, "subnet_phase2").await?; + tag_info(&format!("Subnet Phase 2 complete for {} - {} new hosts with ports", + subnet, new_ports_by_host.len())); + + // Mark phase 2 hosts as having completed full port scan (phase 2's nmap -p- IS their full scan) + for ip in new_ports_by_host.keys() { + ctx.mark_full_scan_complete(ip).await?; + } + + // Now dispatch service discovery (matching complete_port_scan_workflow timing) + for ip in new_ports_by_host.keys() { + dispatch_event(Hook::ServiceDiscoveryReady, ip, ctx); + } + + Ok(()) +} \ No newline at end of file diff --git a/src/enumeration/utils.rs b/src/enumeration/utils.rs new file mode 100755 index 0000000..8a14bd7 --- /dev/null +++ b/src/enumeration/utils.rs @@ -0,0 +1,52 @@ +use std::{error::Error, net::IpAddr}; +use ipnetwork::{IpNetwork, Ipv4Network}; +use crate::sql::ProjectContext; +use crate::utilities::tag_info; + +// Context: Network address parsing and subnet calculation for enumeration +// Operation: Converts IP address or CIDR notation to standardized subnet format +pub fn guess_subnet_str(ip: &str) -> Result> { + if ip.contains('/') { + let net = ip.parse::()?; + match net { + IpNetwork::V4(v4) => { + let base = v4.network(); + let prefix = v4.prefix(); + return Ok(format!("{}/{}", base, prefix)); + } + _ => return Err("Only IPv4 subnets are supported".into()), + } + } + + let addr: IpAddr = ip.parse()?; + if let IpAddr::V4(a) = addr { + let net = Ipv4Network::new(a, 24)?; + let base = net.network(); + return Ok(format!("{}/24", base)); + } + + Err("Only IPv4 addresses are supported".into()) +} + +// Context: Host classification when no open ports are discovered during scanning +// Operation: Tags host as no-ports, marks all scan phases complete, and triggers workflow completion +pub async fn handle_no_ports_host( + ctx: &ProjectContext, + ip: &str, +) -> Result<(), Box> { + + tag_info(&format!(" No open ports on host {}", ip)); + + if let Ok(mut existing_tags) = ctx.get_manual_tags(ip).await { + existing_tags.insert("no-ports".to_string()); + let _ = ctx.set_manual_tags(ip, existing_tags).await; + } + + ctx.mark_service_scan_complete(ip).await?; + ctx.mark_http_discovery_complete(ip).await?; + ctx.mark_screenshots_complete(ip).await?; + + crate::workflow::dispatch_event(crate::workflow::Hook::ReconnaissanceComplete, ip, ctx); + + Ok(()) +} \ No newline at end of file diff --git a/src/http/browser.rs b/src/http/browser.rs new file mode 100755 index 0000000..7363405 --- /dev/null +++ b/src/http/browser.rs @@ -0,0 +1,37 @@ +use std::error::Error; + +// Context: Browser detection for screenshot functionality that requires Chrome/Chromium +// Operation: Attempts to locate Chrome/Chromium executable across common system paths and verify it's working +pub fn check_chrome_availability() -> Result> { + let chrome_paths = vec![ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "/usr/bin/google-chrome", + "/usr/bin/chromium", + "/opt/google/chrome/chrome", + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + ]; + + for path in chrome_paths { + if let Ok(output) = std::process::Command::new(path) + .arg("--version") + .output() { + if output.status.success() { + let version = String::from_utf8_lossy(&output.stdout); + return Ok(format!("Found Chrome at {}: {}", path, version.trim())); + } + } + } + + Err("Chrome/Chromium not found. Please install Google Chrome or Chromium browser.".into()) +} + +// Context: Simplified browser availability check for async contexts +// Operation: Returns boolean indicating whether Chrome/Chromium browser is available on the system +pub async fn check_browser_availability() -> bool { + check_chrome_availability().is_ok() +} diff --git a/src/http/config.rs b/src/http/config.rs new file mode 100755 index 0000000..6f9c9eb --- /dev/null +++ b/src/http/config.rs @@ -0,0 +1,41 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone)] +pub struct ScreenshotConfig { + pub width: u32, + pub height: u32, + pub wait_for_js: bool, + pub format: ScreenshotFormat, +} + +#[derive(Debug, Clone)] +pub enum ScreenshotFormat { + Png, +} + +impl Default for ScreenshotConfig { + // Context: Default configuration values for screenshot capture settings + // Operation: Returns a ScreenshotConfig with standard HD resolution and PNG format + fn default() -> Self { + Self { + width: 1920, + height: 1080, + wait_for_js: true, + format: ScreenshotFormat::Png, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScreenshotResult { + pub url: String, + pub file_path: String, + pub width: u32, + pub height: u32, + pub file_size: u64, + pub timestamp: String, + pub success: bool, + pub error_message: Option, + pub page_title: Option, + pub response_time_ms: u64, +} \ No newline at end of file diff --git a/src/http/discovery.rs b/src/http/discovery.rs new file mode 100755 index 0000000..dc24330 --- /dev/null +++ b/src/http/discovery.rs @@ -0,0 +1,320 @@ +use std::collections::HashMap; +use std::error::Error; +use std::time::Duration; +use futures::stream::{self, StreamExt}; +use tokio::sync::mpsc::UnboundedSender; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::probe::{probe_with_curl, HttpServiceInfo}; + +// Maximum concurrent port probes within a single host's HTTP discovery. +// This bounds in-flight futures structurally — no tokio::spawn, no FD exhaustion. +const HTTP_PROBE_CONCURRENCY: usize = 10; + +// Cooldown before retrying ECONNREFUSED ports — gives router IDS time to unblock. +// 60s is enough for most home/SOHO implementations that detect aggressive scans. +const HTTP_ECONNREFUSED_RETRY_SECS: u64 = 60; + +// Well-known HTTP/HTTPS ports that always get a probe-failed stub when discovery misses them, +// so they appear in the host detail UI with a manual Retry button. +const WELL_KNOWN_HTTP_PORTS: &[&str] = &[ + "80", "443", "8080", "8443", "8000", "8008", "8888", + "3000", "5000", "9000", "4443", "9443", "10080", "10443", +]; + +// Three-way outcome for a single port's probe pass. +// ConnRefused distinguishes "TCP RST on every attempt" (transient IDS block — retry after +// cooldown) from "reached the service but it isn't HTTP" (Failed — no retry). +enum ProbeOutcome { + Found(String, HttpServiceInfo), // (port, info) — store immediately + ConnRefused(String), // port — all protocols returned ECONNREFUSED + Failed, // other errors — no retry warranted +} + +// Context: Main HTTP service discovery orchestrator for a target IP address +// Operation: Probes all open ports concurrently for HTTP/HTTPS services and stores results in database +// +// Takes ctx and ip by value so the returned future is 'static. Async fns that +// borrow from their caller produce futures whose lifetimes are tied to those +// borrows; default_spawn requires Fut: 'static, so passing references would +// make the enclosing async-move block unprovably 'static to the compiler. +// ProjectContext is Arc-backed (cheap clone); String is moved from the caller. +pub async fn discover_http_services( + ctx: ProjectContext, + ip: String, + raw_tx: Option>, +) -> Result<(), Box> { + + let was_already_started = !ctx.mark_http_discovery_started(&ip).await?; + if was_already_started && ctx.is_http_discovery_complete(&ip).await? { + tag_debug(&format!("HTTP discovery already complete for {}", ip)); + crate::workflow::dispatch_event(crate::workflow::Hook::HttpAutoTagComplete, &ip, &ctx); + return Ok(()); + } + + tag_info(&format!("Starting unified HTTP discovery on {}", ip)); + + let all_ports = ctx.get_open_ports(&ip).await?; + + if all_ports.is_empty() { + tag_debug(&format!("No open ports found for HTTP discovery on {}", ip)); + ctx.mark_http_discovery_complete(&ip).await?; + return Ok(()); + } + + // Load nmap service names so we can choose the correct protocol order per port. + let service_map = load_nmap_service_map(&ctx, &ip).await; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Probing {} ports for HTTP services: {}", + all_ports.len(), + all_ports.join(", "))); + } + + // Collect well-known ports before all_ports is consumed — these get probe-failed stubs + // if they don't respond, so they remain visible in the UI with a manual Retry button. + let well_known_candidates: Vec = all_ports.iter() + .filter(|p| WELL_KNOWN_HTTP_PORTS.contains(&p.as_str())) + .cloned() + .collect(); + + // Build probe list: every open port is included — well-known port numbers + // determine protocol order first; nmap service hints break ties for the rest. + // Vec rather than Vec<&'static str>: owned Strings carry no lifetime + // parameter, eliminating the HRTB that &str in a closure argument position would + // introduce (the async-move block's type depends on the specific input lifetime, + // which buffer_unordered requires to be 'static-provable). + let probe_list: Vec<(String, Vec)> = all_ports + .into_iter() + .map(|port| { + let svc = service_map.get(&port).map(String::as_str).unwrap_or(""); + let protocols = effective_probe_order(&port, svc); + (port, protocols) + }) + .collect(); + + if probe_list.is_empty() { + tag_debug(&format!("No HTTP-candidate ports on {}", ip)); + ctx.mark_http_discovery_complete(&ip).await?; + return Ok(()); + } + + // buffer_unordered(N) polls at most N futures at once — no tokio::spawn, no FD storm. + // curl spawns a fresh process per probe: independent TCP fingerprint, no shared state. + let outcomes: Vec = stream::iter(probe_list) + .map(|(port, protocols)| { + let ip = ip.clone(); + let raw_tx = raw_tx.clone(); + async move { + let mut saw_non_refused_error = false; + for protocol in &protocols { + match probe_with_curl(&ip, &port, protocol, raw_tx.as_ref()).await { + Ok(result) => return ProbeOutcome::Found(port, result), + Err((is_conn_refused, msg)) => { + tag_info(&format!( + "probe {}://{}:{} failed: {}", protocol, ip, port, msg + )); + if !is_conn_refused { + saw_non_refused_error = true; + } + } + } + } + // ConnRefused only when every protocol attempt returned ECONNREFUSED. + // Any other error (TLS, protocol mismatch, reset) means the port was + // reachable — IDS retry would not help. + if saw_non_refused_error { + ProbeOutcome::Failed + } else { + ProbeOutcome::ConnRefused(port) + } + } + }) + .buffer_unordered(HTTP_PROBE_CONCURRENCY) + .collect() + .await; + + let mut services_found = 0; + let mut found_ports: std::collections::HashSet = std::collections::HashSet::new(); + let mut refused_ports: Vec<(String, Vec)> = Vec::new(); + + for outcome in outcomes { + match outcome { + ProbeOutcome::Found(port, service_info) => { + tag_info(&format!("{}://{}:{} - {} ({})", + service_info.protocol, ip, port, + service_info.server_header, + service_info.status_code)); + + let service_json = serde_json::to_string(&service_info)?; + + match ctx.insert_http_service(&ip, &port, &service_json).await { + Ok(_) => { + found_ports.insert(port); + services_found += 1; + } + Err(e) => { + tag_debug(&format!("Failed to store HTTP service for {}:{}: {}", ip, port, e)); + } + } + } + ProbeOutcome::ConnRefused(port) => { + tag_info(&format!("Port {} on {} refused all connections — queued for retry after {}s cooldown", + port, ip, HTTP_ECONNREFUSED_RETRY_SECS)); + let svc = service_map.get(&port).map(String::as_str).unwrap_or(""); + refused_ports.push((port.clone(), effective_probe_order(&port, svc))); + } + ProbeOutcome::Failed => {} + } + } + + // Retry ports that got ECONNREFUSED on every attempt. These are most likely blocked by + // the target's IDS reacting to the preceding nmap scan. Wait for the block to expire, + // then make a single retry pass over those ports only. + if !refused_ports.is_empty() { + tag_info(&format!( + "{} port(s) on {} were connection-refused — waiting {}s before retry", + refused_ports.len(), ip, HTTP_ECONNREFUSED_RETRY_SECS + )); + tokio::time::sleep(Duration::from_secs(HTTP_ECONNREFUSED_RETRY_SECS)).await; + + let retry_results: Vec> = + stream::iter(refused_ports) + .map(|(port, protocols)| { + let ip = ip.clone(); + let raw_tx = raw_tx.clone(); + async move { + for protocol in protocols { + match probe_with_curl(&ip, &port, &protocol, raw_tx.as_ref()).await { + Ok(result) => return Some((port, result)), + Err((_, msg)) => tag_info(&format!( + "retry probe {}://{}:{} failed: {}", protocol, ip, port, msg + )), + } + } + None + } + }) + .buffer_unordered(HTTP_PROBE_CONCURRENCY) + .collect() + .await; + + for result in retry_results { + if let Some((port, service_info)) = result { + tag_info(&format!("{}://{}:{} - {} ({}) [retry]", + service_info.protocol, ip, port, + service_info.server_header, + service_info.status_code)); + + let service_json = serde_json::to_string(&service_info)?; + match ctx.insert_http_service(&ip, &port, &service_json).await { + Ok(_) => { + found_ports.insert(port); + services_found += 1; + } + Err(e) => { + tag_debug(&format!("Failed to store retry HTTP service for {}:{}: {}", ip, port, e)); + } + } + } + } + } + + // Insert probe-failed stubs for well-known ports that never responded. + // These stubs appear in the host detail UI as cards with a manual Retry button, + // so operators can re-probe after the target's IDS block expires. + for port in &well_known_candidates { + if found_ports.contains(port) { + continue; + } + let protocol = if port == "443" || port.ends_with("443") { "https" } else { "http" }; + let stub = HttpServiceInfo { + protocol: protocol.to_string(), + status_code: 0, + server_header: String::new(), + title: String::new(), + response_body: String::new(), + technologies: Vec::new(), + version: None, + screenshot_path: None, + response_headers: std::collections::HashMap::new(), + content_length: 0, + response_time_ms: 0, + detected_services: Vec::new(), + evidence: Vec::new(), + probe_failed: true, + }; + if let Ok(json) = serde_json::to_string(&stub) { + ctx.insert_http_service(&ip, port, &json).await.ok(); + tag_info(&format!("Inserted probe-failed stub for {}:{}", ip, port)); + } + } + + ctx.mark_http_discovery_complete(&ip).await?; + + if services_found > 0 { + tag_info(&format!("HTTP discovery complete - found {} services on {}", services_found, ip)); + } else { + tag_debug(&format!("HTTP discovery complete - no HTTP services found on {}", ip)); + } + + Ok(()) +} + +// Context: Protocol probe ordering by port number and nmap service hint +// Operation: Returns the ordered list of protocols to probe for a given port. +// Well-known port numbers take priority over nmap service labels. +// Every port is probed — non-HTTP services fail fast on the wire. +pub fn effective_probe_order(port: &str, svc: &str) -> Vec { + // Well-known plaintext HTTP ports — HTTP first regardless of what nmap reported + match port { + "80" | "8080" | "8000" | "8008" | "8888" | "3000" | "5000" | "9000" | "10080" => { + return vec!["http".into(), "https".into()]; + } + // Well-known TLS ports — HTTPS first + "443" | "8443" | "4443" | "9443" | "10443" => { + return vec!["https".into(), "http".into()]; + } + _ => {} + } + + // Non-standard ports: use nmap service hint to pick the likely protocol first + match svc { + "http" | "http-proxy" | "http-alt" | "http-mgmt" => vec!["http".into(), "https".into()], + "https" | "ssl" => vec!["https".into(), "http".into()], + _ if svc.starts_with("ssl/") => vec!["https".into(), "http".into()], + // Everything else (ssh, dns, ftp, upnp, tcpwrapped, unknown…): probe both. + // Non-HTTP services respond immediately with a protocol error — cheap to skip. + _ => vec!["https".into(), "http".into()], + } +} + +// Context: Nmap service name lookup for HTTP discovery optimisation +// Operation: Parses stored nmap XML (service_scan key) and returns a port→service_name map +async fn load_nmap_service_map(ctx: &ProjectContext, ip: &str) -> HashMap { + let xml_json_str = match ctx.get_nmap_xml(ip).await.ok().flatten() { + Some(s) => s, + None => return HashMap::new(), + }; + + let xml_map: serde_json::Map = + match serde_json::from_str(&xml_json_str) { + Ok(v) => v, + Err(_) => return HashMap::new(), + }; + + let xml_str = match xml_map.get("service_scan").and_then(|v| v.as_str()) { + Some(s) => s, + None => return HashMap::new(), + }; + + let parsed = match crate::tag::nmap_xml_parsing::parse_nmap_xml(xml_str) { + Some(r) => r, + None => return HashMap::new(), + }; + + parsed.ports.into_iter() + .map(|p| (p.port, p.service_name)) + .collect() +} diff --git a/src/http/mod.rs b/src/http/mod.rs new file mode 100755 index 0000000..221b4ef --- /dev/null +++ b/src/http/mod.rs @@ -0,0 +1,27 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod browser; +pub mod config; +pub mod discovery; +pub mod probe; +pub mod screenshot; +pub mod services; + +pub use browser::*; +// pub use config::*; +pub use discovery::*; +pub use probe::*; +// pub use screenshot::*; +pub use services::*; \ No newline at end of file diff --git a/src/http/probe.rs b/src/http/probe.rs new file mode 100755 index 0000000..bc5ad58 --- /dev/null +++ b/src/http/probe.rs @@ -0,0 +1,179 @@ +use std::collections::HashMap; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc::UnboundedSender; +use tokio::process::Command as TokioCommand; + +// Single source of truth for the HTTP probe User-Agent. +// Real browser UA avoids 403/redirect responses from embedded devices that check the UA. +pub const HTTP_PROBE_USER_AGENT: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ + AppleWebKit/537.36 (KHTML, like Gecko) \ + Chrome/124.0.0.0 Safari/537.36"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HttpServiceInfo { + pub protocol: String, + pub status_code: u16, + pub server_header: String, + pub title: String, + pub response_body: String, + pub technologies: Vec, + pub version: Option, + pub screenshot_path: Option, + pub response_headers: HashMap, + pub content_length: usize, + pub response_time_ms: u64, + pub detected_services: Vec, + pub evidence: Vec, + // True for stub entries inserted when a well-known port failed probing during discovery. + // These appear in the UI with a manual Retry button. + #[serde(default)] + pub probe_failed: bool, +} + +// Context: Curl-based HTTP service probe — subprocess model avoids in-process blocking +// Operation: Spawns curl for each probe, parses headers+body from stdout, returns HttpServiceInfo. +// Returns Err((true, msg)) on ECONNREFUSED (curl exit 7) so callers can schedule retry. +// Returns Err((false, msg)) on all other failures (timeout, protocol error, etc.). +pub async fn probe_with_curl( + ip: &str, + port: &str, + protocol: &str, + raw_tx: Option<&UnboundedSender>, +) -> Result { + let url = format!("{}://{}:{}/", protocol, ip, port); + let start = std::time::Instant::now(); + + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Probing (curl): {}", url)); + } + + let output = TokioCommand::new("curl") + .args([ + "-s", + "-i", + "--insecure", + "--max-time", "15", + "--connect-timeout", "5", + "-L", + "--max-redirs", "3", + "-A", HTTP_PROBE_USER_AGENT, + "-w", "\n--ROFMAD:%{http_code}:%{time_total}--", + &url, + ]) + .output() + .await + .map_err(|e| (false, format!("curl spawn failed: {}", e)))?; + + let elapsed_ms = start.elapsed().as_millis() as u64; + + match output.status.code() { + Some(0) => {} // success + Some(7) => return Err((true, "Connection refused (curl exit 7)".to_string())), + Some(28) => return Err((false, format!("curl timeout after {}ms", elapsed_ms))), + Some(code) => return Err((false, format!("curl exit {}", code))), + None => return Err((false, "curl killed by signal".to_string())), + } + + let raw = String::from_utf8_lossy(&output.stdout); + + // Strip the --ROFMAD:CODE:TIME-- marker appended by -w and extract status + timing + let (http_output, status_code, response_time_ms) = + if let Some(marker_pos) = raw.rfind("\n--ROFMAD:") { + let marker = raw[marker_pos + 1..].trim(); // "--ROFMAD:200:0.045--" + let inner = marker + .trim_start_matches("--ROFMAD:") + .trim_end_matches("--"); + let mut parts = inner.splitn(2, ':'); + let code: u16 = parts.next().unwrap_or("0").parse().unwrap_or(0); + let time_s: f64 = parts.next().unwrap_or("0").parse().unwrap_or(0.0); + (&raw[..marker_pos], code, (time_s * 1000.0) as u64) + } else { + return Err((false, "curl output missing status marker".to_string())); + }; + + // With -L, stdout may contain multiple HTTP response blocks (one per redirect). + // Find the last one — that is the final response we care about. + let last_block_start = http_output + .rfind("\nHTTP/") + .map(|i| i + 1) + .unwrap_or_else(|| if http_output.starts_with("HTTP/") { 0 } else { 0 }); + + let final_block = &http_output[last_block_start..]; + + // Split headers from body on the blank line separator (\r\n\r\n or \n\n) + let (headers_str, body_str) = if let Some(pos) = final_block.find("\r\n\r\n") { + (&final_block[..pos], &final_block[pos + 4..]) + } else if let Some(pos) = final_block.find("\n\n") { + (&final_block[..pos], &final_block[pos + 2..]) + } else { + (final_block, "") + }; + + // Parse response headers (skip status line, lowercase keys) + let mut headers = HashMap::new(); + for line in headers_str.lines().skip(1) { + if let Some(colon) = line.find(':') { + let key = line[..colon].trim().to_lowercase(); + let value = line[colon + 1..].trim().to_string(); + headers.insert(key, value); + } + } + + let server_header = headers + .get("server") + .cloned() + .unwrap_or_else(|| "Unspecified".to_string()); + + // Truncate body at 50 KB + let body = if body_str.len() > 50000 { + let mut end = 50000; + while !body_str.is_char_boundary(end) { end -= 1; } + &body_str[..end] + } else { + body_str + }; + + let title = extract_title(body); + + if let Some(tx) = raw_tx { + let _ = tx.send(format!("{}: {} {} ({}ms)", url, status_code, server_header, response_time_ms)); + } + + Ok(HttpServiceInfo { + protocol: protocol.to_string(), + status_code, + server_header, + title, + response_body: body.to_string(), + technologies: Vec::new(), + version: None, + screenshot_path: None, + response_headers: headers, + content_length: body.len(), + response_time_ms, + detected_services: Vec::new(), + evidence: Vec::new(), + probe_failed: false, + }) +} + +// Context: HTML title extraction from HTTP response body +// Operation: Uses regex to parse and clean HTML title tag content, truncating to 100 characters +pub fn extract_title(html: &str) -> String { + if let Ok(re) = regex::Regex::new(r"(?i)]*>(.*?)") { + if let Some(caps) = re.captures(html) { + let title = caps.get(1).unwrap().as_str(); + return title.trim() + .lines() + .map(|line| line.trim()) + .collect::>() + .join(" ") + .chars() + .take(100) + .collect(); + } + } + + "No Title".to_string() +} \ No newline at end of file diff --git a/src/http/screenshot.rs b/src/http/screenshot.rs new file mode 100755 index 0000000..37bfe86 --- /dev/null +++ b/src/http/screenshot.rs @@ -0,0 +1,261 @@ +use std::error::Error; +use std::ffi::OsStr; +use std::time::Duration; +use tokio::sync::mpsc::UnboundedSender; +use headless_chrome::{Browser, LaunchOptions}; +use headless_chrome::protocol::cdp::Page; +use crate::utilities::{tag_debug}; +use crate::VERBOSE; +use super::config::{ScreenshotConfig, ScreenshotFormat, ScreenshotResult}; + +pub struct BrowserManager { + browser: Option, + config: ScreenshotConfig, + temp_profile_dir: Option, +} + +impl BrowserManager { + // Context: Constructor for browser manager with screenshot configuration + // Operation: Initializes empty browser manager with provided screenshot settings + pub fn new(config: ScreenshotConfig) -> Self { + Self { + browser: None, + config, + temp_profile_dir: None, + } + } + + // Context: Lazy initialization and retrieval of Chrome browser instance + // Operation: Creates headless Chrome browser with security flags if not already initialized + fn get_browser(&mut self) -> Result<&Browser, Box> { + if self.browser.is_none() { + let temp_dir = format!("/tmp/rust-headless-chrome-profile-{}", uuid::Uuid::new_v4()); + let launch_options = LaunchOptions::default_builder() + .headless(true) + .window_size(Some((self.config.width, self.config.height))) + .args(vec![ + OsStr::new("--no-sandbox"), + OsStr::new("--disable-setuid-sandbox"), + OsStr::new("--disable-dev-shm-usage"), + OsStr::new("--disable-gpu"), + OsStr::new("--no-first-run"), + OsStr::new("--no-default-browser-check"), + OsStr::new("--disable-extensions"), + OsStr::new("--disable-plugins"), + OsStr::new("--disable-background-timer-throttling"), + OsStr::new("--disable-background-networking"), + OsStr::new("--disable-client-side-phishing-detection"), + OsStr::new("--disable-sync"), + OsStr::new("--disable-translate"), + OsStr::new("--hide-scrollbars"), + OsStr::new("--metrics-recording-only"), + OsStr::new("--mute-audio"), + OsStr::new("--safebrowsing-disable-auto-update"), + OsStr::new("--ignore-certificate-errors"), + OsStr::new("--ignore-ssl-errors"), + OsStr::new("--ignore-certificate-errors-spki-list"), + ]) + .build() + .map_err(|e| format!("Failed to build launch options: {}", e))?; + + let browser = Browser::new(launch_options) + .map_err(|e| format!("Failed to launch browser: {}", e))?; + + self.browser = Some(browser); + self.temp_profile_dir = Some(temp_dir); + } + + Ok(self.browser.as_ref().unwrap()) + } + + // Context: Main screenshot capture functionality for HTTP services + // Operation: Navigates to URL, waits for page load, captures screenshot and returns metadata + pub async fn take_screenshot( + &mut self, + url: &str, + output_path: &str, + raw_tx: Option<&UnboundedSender>, + ) -> Result> { + let start_time = std::time::Instant::now(); + + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Taking screenshot of: {}", url)); + } + + let browser = self.get_browser()?; + let tab = browser.new_tab() + .map_err(|e| format!("Failed to create new tab: {}", e))?; + + let mut result = ScreenshotResult { + url: url.to_string(), + file_path: output_path.to_string(), + width: self.config.width, + height: self.config.height, + file_size: 0, + timestamp: chrono::Utc::now().to_rfc3339(), + success: false, + error_message: None, + page_title: None, + response_time_ms: 0, + }; + + tab.set_bounds(headless_chrome::types::Bounds::Normal { + left: Some(0), + top: Some(0), + width: Some(self.config.width as f64), + height: Some(self.config.height as f64), + }).map_err(|e| format!("Failed to set bounds: {}", e))?; + + let navigate_result = tokio::time::timeout( + Duration::from_secs(30), + async { + tab.navigate_to(url) + .map_err(|e| format!("Failed to navigate to {}: {}", url, e)) + } + ).await; + + match navigate_result { + Ok(Ok(_)) => { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Successfully navigated to: {}", url)); + } + } + Ok(Err(e)) => { + result.error_message = Some(e); + return Ok(result); + } + Err(_) => { + result.error_message = Some("Navigation timeout".to_string()); + return Ok(result); + } + } + + if self.config.wait_for_js { + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Waiting for page to fully load: {}", url)); + } + + if let Err(e) = tab.wait_for_element("body") { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Warning: Failed to wait for body element: {}", e)); + } + } + + let mut page_ready = false; + let max_attempts = 20; + + for attempt in 1..=max_attempts { + tokio::time::sleep(Duration::from_secs(1)).await; + + if let Ok(title) = tab.get_title() { + if !title.is_empty() && title != "Loading..." { + let script = r#" + (function() { + if (document.readyState !== 'complete') return false; + + const visibleElements = Array.from(document.querySelectorAll('*')) + .filter(el => { + const style = window.getComputedStyle(el); + return style.display !== 'none' && + style.visibility !== 'hidden' && + style.opacity !== '0' && + (el.textContent.trim().length > 0 || + el.tagName.toLowerCase() === 'input' || + el.tagName.toLowerCase() === 'button' || + el.tagName.toLowerCase() === 'img'); + }); + + return visibleElements.length >= 3; + })() + "#; + + if let Ok(result) = tab.evaluate(script, false) { + if let Some(value) = result.value { + if let Some(ready) = value.as_bool() { + if ready { + page_ready = true; + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Page content loaded after {}s: {}", attempt, url)); + } + break; + } + } + } + } + } + } + + if attempt % 10 == 0 { + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Still waiting for content... {}s elapsed: {}", attempt, url)); + } + } + } + + if !page_ready { + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Timeout waiting for content, taking screenshot anyway: {}", url)); + } + } + + if let Err(e) = tab.wait_until_navigated() { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Warning: Navigation wait failed: {}", e)); + } + } + } + + if let Ok(title) = tab.get_title() { + result.page_title = Some(title); + } + + let screenshot_data = match self.config.format { + ScreenshotFormat::Png => { + tab.capture_screenshot( + Page::CaptureScreenshotFormatOption::Png, + None, + None, + true + ).map_err(|e| format!("Failed to capture PNG screenshot: {}", e))? + } + }; + + std::fs::write(output_path, &screenshot_data) + .map_err(|e| format!("Failed to write screenshot to {}: {}", output_path, e))?; + + if let Ok(metadata) = std::fs::metadata(output_path) { + result.file_size = metadata.len(); + } + + result.response_time_ms = start_time.elapsed().as_millis() as u64; + result.success = true; + + if let Err(e) = tab.close(false) { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Warning: Failed to close tab: {}", e)); + } + } + + if let Some(tx) = raw_tx { + let _ = tx.send(format!("Screenshot saved: {} ({}ms)", output_path, result.response_time_ms)); + } + + Ok(result) + } +} + +impl Drop for BrowserManager { + // Context: Resource cleanup when browser manager is dropped + // Operation: Removes temporary Chrome profile directory created during browser initialization + fn drop(&mut self) { + if let Some(temp_dir) = &self.temp_profile_dir { + if temp_dir.starts_with("/tmp/rust-headless-chrome-profile") { + if let Err(e) = std::fs::remove_dir_all(temp_dir) { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Failed to cleanup temp profile {}: {}", temp_dir, e)); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/http/services.rs b/src/http/services.rs new file mode 100755 index 0000000..7e73268 --- /dev/null +++ b/src/http/services.rs @@ -0,0 +1,267 @@ +use std::error::Error; +use std::path::Path; +use std::time::Duration; +use tokio::sync::mpsc::UnboundedSender; +use crate::sql::ProjectContext; +use crate::utilities::tag_info; +use crate::config::paths::ensure_screenshots_directory_for_run; +use super::config::{ScreenshotConfig, ScreenshotResult}; +use super::screenshot::BrowserManager; + +const PREFLIGHT_TIMEOUT_SECS: u64 = 5; +const SCREENSHOT_TIMEOUT_SECS: u64 = 60; +const SCREENSHOT_MAX_RETRIES: u32 = 3; + +enum PreflightResult { + Ready(String), // URL to screenshot (original or resolved redirect) + Failed(String), // reason — port goes into retry queue +} + +// Context: Pre-flight HTTP check before browser screenshot +// Operation: Runs curl with a short timeout; returns Ready(url) on any HTTP response, +// or Redirect(final_url) if 3xx, or Failed if no response within 5s. +// Logs the command and result to raw_tx for full traceability in task output. +async fn preflight_curl( + ip: &str, + port: &str, + protocol: &str, + raw_tx: Option<&UnboundedSender>, +) -> PreflightResult { + let url = format!("{}://{}:{}/", protocol, ip, port); + if let Some(tx) = raw_tx { + let _ = tx.send(format!( + "$ curl -sk --max-time {} -o /dev/null -w '%{{http_code}} %{{redirect_url}}' '{}'", + PREFLIGHT_TIMEOUT_SECS, url + )); + } + + let result = tokio::process::Command::new("curl") + .args([ + "-sk", + "--max-time", &PREFLIGHT_TIMEOUT_SECS.to_string(), + "-o", "/dev/null", + "-w", "%{http_code} %{redirect_url}", + &url, + ]) + .output() + .await; + + match result { + Ok(out) => { + let stdout = String::from_utf8_lossy(&out.stdout); + let trimmed = stdout.trim(); + if let Some(tx) = raw_tx { + let _ = tx.send(format!("curl: {}", trimmed)); + } + let mut parts = trimmed.splitn(2, ' '); + let code: u16 = parts.next().and_then(|s| s.parse().ok()).unwrap_or(0); + let location = parts.next().map(|s| s.trim().to_string()).unwrap_or_default(); + + if code == 0 { + PreflightResult::Failed(format!("no response (code=0, raw='{}')", trimmed)) + } else if (300..400).contains(&code) { + // Follow the redirect target for the screenshot + let target = if location.is_empty() { url } else { location }; + PreflightResult::Ready(target) + } else { + PreflightResult::Ready(url) + } + } + Err(e) => { + if let Some(tx) = raw_tx { + let _ = tx.send(format!("curl error: {}", e)); + } + PreflightResult::Failed(e.to_string()) + } + } +} + +// Context: Single-port browser screenshot with timeout +// Operation: Launches Chrome via BrowserManager for `screenshot_url`; stores result under +// the original port's filename. `original_url` is preserved in ScreenshotResult.url +// so the caller can extract the port for DB update without caring about redirects. +async fn do_screenshot( + screenshot_url: &str, + original_url: &str, + ip: &str, + port: &str, + protocol: &str, + output_dir: &str, + raw_tx: Option<&UnboundedSender>, +) -> ScreenshotResult { + let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string(); + let filename = format!("{}_{}_{}_{}.png", ip.replace('.', "_"), port, protocol, timestamp); + let output_path = Path::new(output_dir).join(&filename).to_string_lossy().to_string(); + + let screenshot_url_owned = screenshot_url.to_string(); + let output_path_c = output_path.clone(); + let original_url_owned = original_url.to_string(); + let raw_tx_c: Option> = raw_tx.cloned(); + + let timed = tokio::time::timeout( + Duration::from_secs(SCREENSHOT_TIMEOUT_SECS), + async move { + tokio::time::sleep(Duration::from_millis(500)).await; + let mut bm = BrowserManager::new(ScreenshotConfig::default()); + bm.take_screenshot(&screenshot_url_owned, &output_path_c, raw_tx_c.as_ref()).await + }, + ) + .await; + + match timed { + Ok(Ok(mut result)) => { + // Always store the original URL so extract_port_from_url works correctly upstream + result.url = original_url_owned; + if result.success { + tag_info(&format!("Screenshot: {}:{} -> {}", ip, port, output_path)); + } else { + tag_info(&format!("Screenshot failed: {}:{} - {}", + ip, port, result.error_message.as_deref().unwrap_or("unknown"))); + } + result + } + Ok(Err(e)) => { + tag_info(&format!("Screenshot error: {}:{} - {}", ip, port, e)); + ScreenshotResult { + url: original_url_owned, + file_path: output_path, + width: 0, height: 0, file_size: 0, + timestamp: chrono::Utc::now().to_rfc3339(), + success: false, + error_message: Some(e.to_string()), + page_title: None, + response_time_ms: 0, + } + } + Err(_) => { + tag_info(&format!("Screenshot timeout ({}s): {}:{}", SCREENSHOT_TIMEOUT_SECS, ip, port)); + ScreenshotResult { + url: original_url_owned, + file_path: String::new(), + width: 0, height: 0, file_size: 0, + timestamp: chrono::Utc::now().to_rfc3339(), + success: false, + error_message: Some(format!("timeout ({}s)", SCREENSHOT_TIMEOUT_SECS)), + page_title: None, + response_time_ms: SCREENSHOT_TIMEOUT_SECS * 1000, + } + } + } +} + +// Context: Wrapper function for multi-screenshot capture with runID-specific directory +// Operation: Calls screenshot_http_services with project-specific screenshot directory path +pub async fn capture_multiple_screenshots( + ctx: &ProjectContext, + ip: &str, + raw_tx: Option>, +) -> Result, Box> { + let output_dir = ensure_screenshots_directory_for_run(&ctx.db_path, &ctx.run_id)?; + screenshot_http_services(ctx, ip, &output_dir, raw_tx).await +} + +// Context: Sequential screenshot capture for all HTTP services on a target IP +// Operation: For each service, runs a 5s curl pre-flight check first (logged to task output). +// Services that pass → screenshot immediately. Services that fail → queued for retry. +// Retries up to SCREENSHOT_MAX_RETRIES times total (including first pass). +// All curl commands and responses are sent to raw_tx for full log traceability. +pub async fn screenshot_http_services( + ctx: &ProjectContext, + ip: &str, + output_dir: &str, + raw_tx: Option>, +) -> Result, Box> { + + let http_services = ctx.get_http_services(ip).await?; + + if http_services.is_empty() { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("No HTTP services found for screenshots on {}", ip)); + } + ctx.mark_screenshots_complete(ip).await?; + return Ok(Vec::new()); + } + + std::fs::create_dir_all(output_dir) + .map_err(|e| format!("Failed to create output directory {}: {}", output_dir, e))?; + + tag_info(&format!("Taking screenshots for {} HTTP services on {}", http_services.len(), ip)); + + let services_vec: Vec<(String, String)> = http_services.into_iter().collect(); + let mut results: Vec = Vec::new(); + + // Pass 1 + up to (SCREENSHOT_MAX_RETRIES - 1) retry passes + let mut current_pass: Vec<(String, String)> = services_vec; + + for attempt in 1..=(SCREENSHOT_MAX_RETRIES) { + let mut next_retry: Vec<(String, String)> = Vec::new(); + + for (port, service_json) in current_pass { + let service_info: Result = serde_json::from_str(&service_json); + let protocol = match &service_info { + Ok(info) => info.protocol.clone(), + Err(_) => { + if port == "443" || port == "8443" || port.ends_with("443") { + "https".to_string() + } else { + "http".to_string() + } + } + }; + + let original_url = format!("{}://{}:{}/", protocol, ip, port); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[pass {}/{}] preflight {}:{}", attempt, SCREENSHOT_MAX_RETRIES, ip, port)); + } + + match preflight_curl(ip, &port, &protocol, raw_tx.as_ref()).await { + PreflightResult::Ready(screenshot_url) => { + let result = do_screenshot( + &screenshot_url, + &original_url, + ip, &port, &protocol, + output_dir, + raw_tx.as_ref(), + ).await; + results.push(result); + } + PreflightResult::Failed(reason) => { + tag_info(&format!("Preflight pass {}/{} failed {}:{} — {}", + attempt, SCREENSHOT_MAX_RETRIES, ip, port, reason)); + if attempt < SCREENSHOT_MAX_RETRIES { + next_retry.push((port, service_json)); + } else { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!( + "Screenshot skipped {}:{}: preflight failed after {} attempts", + ip, port, SCREENSHOT_MAX_RETRIES + )); + } + tag_info(&format!("Screenshot skipped {}:{}: preflight failed after {} attempts", + ip, port, SCREENSHOT_MAX_RETRIES)); + } + } + } + } + + current_pass = next_retry; + if current_pass.is_empty() { + break; + } + + // Small delay between retry passes + if attempt < SCREENSHOT_MAX_RETRIES { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("{} service(s) queued for retry pass {}", current_pass.len(), attempt + 1)); + } + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + + let success_count = results.iter().filter(|r| r.success).count(); + tag_info(&format!("Screenshot session complete for {} - {}/{} successful", + ip, success_count, results.len())); + + Ok(results) +} diff --git a/src/main.rs b/src/main.rs new file mode 100755 index 0000000..b699058 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,815 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// src/main.rs - PATCHED: Fixed startup flow and output handling +mod web; +mod http; +mod tag; +mod sql; +use sql::ProjectContext; +mod utilities; +use crate::utilities::{tag_info, tag_debug, print_startup}; +mod enumeration; +mod workflow; +mod config; +use crate::workflow::{run_workflow, RESUME_MODE}; +use once_cell::sync::Lazy; +use clap::{Parser, Subcommand}; +use std::{ + env, + process::Command, + sync::atomic::{AtomicBool, Ordering}, + path::Path, +}; +use std::os::unix::process::CommandExt; +use tokio::{ + runtime::Builder, + sync::watch, + time::{sleep, Duration}, + task::yield_now, +}; +use if_addrs::{get_if_addrs, IfAddr}; + +// Toggle debug logging with `-v` +static VERBOSE: Lazy = Lazy::new(|| AtomicBool::new(false)); + +#[derive(Parser)] +#[command( + name = "ROFMAD", + author = "P-Aïmon", + version = "0.2.0", + about = "Rust Orchestrator For Modular Automated Discovery - Advanced Network Reconnaissance Framework", + long_about = "ROFMAD is a comprehensive network reconnaissance and enumeration framework featuring:\n\ + • Multi-phase HTTP service discovery and enumeration\n\ + • Advanced web application screening with screenshots\n\ + • YAML-based workflow and tagging system\n\ + • Intelligent concurrency management\n\ + • Real-time web interface with task monitoring\n\ + • Resume-capable project state management" +)] +struct Cli { + #[arg(short = 'H', long = "extended-help", help = "Show workflow YAML format, examples, and available modules")] + extended_help: bool, + + #[arg(short = 'v', long = "verbose", help = "Enable verbose debug logging")] + verbose: bool, + + #[arg(short = 'c', long = "check-deps", help = "Check system dependencies")] + check_deps: bool, + + #[arg(short = 'i', long = "interface", help = "Network interface for scanning (required)", global = true)] + interface: Option, + + #[arg(short = 'w', long = "workflow", help = "Path to workflow YAML file (required)", global = true)] + workflow: Option, + + #[arg(short = 'p', long = "web-port", help = "Web interface port", default_value = "8443", global = true)] + web_port: u16, + + /// Ignored — admin password is now set via the web setup page on first run + #[arg(long = "admin-password", hide = true, global = true)] + _admin_password: Option, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Commands { + #[command(about = "Create a new reconnaissance project")] + CreateProject { + #[arg(help = "Project name (alphanumeric, no spaces)")] + project_name: String, + #[arg(help = "Project directory path (will be created if doesn't exist)")] + project_directory: String, + }, + #[command(about = "Resume an existing project from database")] + ResumeProject { + #[arg(help = "Path to existing project database (.db file)")] + path: String, + }, + #[command(about = "Start web interface only for existing database")] + WebOnly { + #[arg(help = "Path to existing project database (.db file)")] + path: String, + #[arg(short, long, help = "Interface IP to bind to", default_value = "127.0.0.1")] + interface: String, + #[arg(short, long, help = "Port to bind to", default_value = "8443")] + port: u16, + }, +} + +// Context: CLI application branding and version display +// Operation: Returns ASCII art logo with application name and version +fn logo_line() -> String { + r#" +______ ______________ ___ ___ ______ +| ___ \ _ | ___| \/ | / _ \| _ \ +| |_/ / | | | |_ | . . |/ /_\ \ | | | +| /| | | | _| | |\/| || _ | | | | +| |\ \\ \_/ / | | | | || | | | |/ / +\_| \_|\___/\_| \_| |_/\_| |_/___/ + +Rust Orchestrator For Modular Automated Discovery v0.2.0 +Advanced Network Reconnaissance & Web Application Discovery Framework +"#.to_string() +} + +// Context: Extended help combining workflow YAML reference and module list +// Operation: Prints workflow format, condition operators, placeholders, and available modules +fn print_extended_help() { + println!("ROFMAD Extended Help"); + println!(""); + println!("USAGE:"); + println!(" rofmad create-project -i -w "); + println!(" rofmad resume-project -i -w "); + println!(" rofmad web-only # Web interface only (no scanning)"); + println!(" rofmad --check-deps # Verify system dependencies"); + println!(" rofmad -v # Enable verbose logging"); + println!(""); + println!("WORKFLOW YAML FORMAT:"); + println!("```yaml"); + println!("specific_actions:"); + println!(" - name: \"smb_enumeration\""); + println!(" condition:"); + println!(" contains: \"445 AND 139\" # Must have BOTH ports"); + println!(" has_tag: \"smb AND Windows\" # Must have BOTH tags"); + println!(" has_http_service: true # Must have HTTP services"); + println!(" # reconnaissance_complete: true # ALWAYS ENFORCED"); + println!(" command:"); + println!(" tool: \"nxc\""); + println!(" args: \"smb {{IP}} -u '' -p '' --shares\""); + println!(" timeout_s: 120"); + println!(" tags:"); + println!(" - pattern: \"STATUS_SUCCESS\""); + println!(" tag: \"anonymous-smb\""); + println!(" - pattern: \"SMB.*signing.*False\""); + println!(" tag: \"smb-signing-disabled\""); + println!("```"); + println!(""); + println!("CONDITION FIELDS (all optional, combined with AND logic):"); + println!(" contains: \"445 AND 139\" # Port AND — both must be open"); + println!(" contains: \"80 OR 443\" # Port OR — either must be open"); + println!(" equals: \"22\" # Exact single-port match"); + println!(" has_tag: \"smb AND windows\" # Tag AND — both tags required"); + println!(" has_tag: \"web OR apache\" # Tag OR — either tag required"); + println!(" has_service: \"mssql OR rdp\" # Service check (port-agnostic, uses nmap detection)"); + println!(" has_http_service: true/false # HTTP service presence check"); + println!(" # reconnaissance_complete: true # Always enforced — do not set manually"); + println!(""); + println!("PLACEHOLDERS:"); + println!(" {{IP}} / {{ip}} / / # Target IP address"); + println!(" {{PORT}} / {{port}} # First available port"); + println!(""); + println!("SYSTEM DIRECTORIES:"); + println!(" {} # Workflow YAML files", crate::config::paths::SystemPaths::WORKFLOWS_DIRECTORY); + println!(" /screenshots// # Screenshot storage (next to DB file)"); + println!(""); + println!("AVAILABLE MODULES:"); + println!(" Initialization: local_dns_search, passive_listener, enumerate_arp_scan, masscan_discovery"); + println!(" Enumeration: nmap_full_port_detection_scan, nmap_service_scan, subnet_explorator"); + println!(" HTTP: http_probe, web_deep_enumeration, host_screenshots"); + println!(""); + println!("WORKFLOW PHASES:"); + println!(" 1. Discovery — Host and service discovery"); + println!(" 2. Enumeration — Port scanning and service detection"); + println!(" 3. HTTP — Web service identification"); + println!(" 4. Deep — Application-specific probing"); + println!(" 5. Actions — YAML-defined targeted enumeration"); +} + +// Context: System dependency validation for reconnaissance tools +// Operation: Checks for required external tools and directories, reports status +async fn check_system_dependencies() { + println!("Checking ROFMAD System Dependencies:"); + println!(""); + + // Check required tools + let tools = vec![ + ("nmap", "Network port scanner"), + ("masscan", "High-speed port scanner"), + ("arp-scan", "ARP-based network discovery"), + ("tcpdump", "Network packet capture"), + ("nuclei", "Vulnerability scanner"), + ]; + + println!("REQUIRED TOOLS:"); + for (tool, description) in &tools { + if let Ok(output) = tokio::process::Command::new("which") + .arg(tool) + .output() + .await + { + if output.status.success() { + let path_raw = String::from_utf8_lossy(&output.stdout); + let path = path_raw.trim(); + println!("{} — {} ({})", tool, description, path); + } else { + println!("{} — {} (NOT FOUND)", tool, description); + } + } else { + println!("{} — {} (NOT FOUND)", tool, description); + } + } + + if crate::workflow::nuclei::nuclei_templates_present() { + println!("nuclei templates — found"); + } else { + println!(" nuclei templates — NOT FOUND (run 'nuclei -update-templates' or create a project to trigger it automatically)"); + } + + println!(""); + + // Check browser for screenshots + if crate::http::check_browser_availability().await { + println!("Browser — Chrome/Chromium found for screenshots"); + } else { + println!(" Browser — Chrome/Chromium not found (screenshots disabled)"); + } + + println!(""); + println!("DIRECTORIES:"); + let workflows_dir = crate::config::paths::SystemPaths::WORKFLOWS_DIRECTORY; + if std::path::Path::new(workflows_dir).exists() { + println!("Directory — {}", workflows_dir); + } else { + println!(" Directory — {} (missing)", workflows_dir); + } + println!("Screenshots — /screenshots// (created at runtime)"); + + println!(""); + println!("INSTALLATION HINTS:"); + println!(" sudo apt install nmap masscan arp-scan tcpdump nuclei chromium-browser # Debian/Ubuntu"); + println!(" sudo dnf install nmap masscan arp-scan tcpdump nuclei chromium # Fedora/RHEL"); +} + +// Context: Tool availability validation before operations +// Operation: Validates critical reconnaissance tools are available, returns errors for missing tools +async fn validate_required_tools() -> Result<(), Box> { + let critical_tools = vec![ + ("nmap", "Network port scanner - required for reconnaissance"), + ("masscan", "High-speed port scanner - required for discovery"), + ("arp-scan", "ARP-based network discovery - required for local enumeration"), + ("tcpdump", "Network packet capture - required for passive monitoring"), + ("nuclei", "Vulnerability scanner - required for nuclei scanning"), + ]; + + let mut missing_tools = Vec::new(); + + for (tool, description) in &critical_tools { + let available = if let Ok(output) = tokio::process::Command::new("which") + .arg(tool) + .output() + .await + { + output.status.success() + } else { + false + }; + + if available { + tag_debug(&format!("Tool validated: {}", tool)); + } else { + missing_tools.push(format!("{} ({})", tool, description)); + } + } + + if !missing_tools.is_empty() { + let error_msg = format!( + "Missing required tools:\n{}\n\nInstall with:\n sudo apt install nmap masscan arp-scan tcpdump nuclei", + missing_tools.join("\n") + ); + return Err(error_msg.into()); + } + + // nuclei binary is present, but it's useless without templates. Check and update them + // synchronously here so a broken/uninitialized nuclei is caught now, loudly, instead of + // producing silent scan failures later (see src/workflow/nuclei.rs for the readiness logic). + crate::workflow::nuclei::ensure_nuclei_templates_ready().await?; + + print_startup("All required reconnaissance tools validated"); + Ok(()) +} + +// Context: Project initialization and system setup +// Operation: Configures blacklists, interfaces, screenshot system, and stores workflow settings +async fn auto_init( + ctx: &ProjectContext, + interface: &str, + workflow_path: &str, +) -> Result<(), Box> { + print_startup("Starting auto-initialization..."); + + // 1) Always blacklist loopback + print_startup("Blacklisting loopback addresses..."); + if let Err(e) = ctx.insert_blacklist("127.0.0.0/8").await { + tag_debug(&format!("Blacklist failed for 127.0.0.0/8: {}", e)); + } + if let Err(e) = ctx.insert_blacklist("0.0.0.0/8").await { + tag_debug(&format!("Blacklist failed for 0.0.0.0/8: {}", e)); + } + + // 2) Enumerate host IPv4 addresses only (no subnets) + print_startup("Blacklisting host interface addresses..."); + match get_if_addrs() { + Ok(ifaces) => { + for iface in ifaces { + // skip loopback interfaces + if iface.is_loopback() { + continue; + } + // only IPv4, single‐host entries + if let IfAddr::V4(v4) = iface.addr { + let addr = v4.ip.to_string(); + if let Err(e) = ctx.insert_blacklist(&addr).await { + tag_debug(&format!("Blacklist failed for {}: {}", addr, e)); + } + } + } + } + Err(e) => { + tag_debug(&format!("Interface enumeration failed: {}", e)); + } + } + + // yield to let the system process these inserts + yield_now().await; + + // 3) Store interface with default /24 subnet + let interface_ip = match get_if_addrs() { + Ok(ifaces) => { + ifaces.into_iter() + .find(|iface| iface.name == interface) + .and_then(|iface| match iface.addr { + IfAddr::V4(v4) => Some(v4.ip.to_string()), + _ => None, + }) + .unwrap_or_else(|| { + print_startup(&format!(" Interface {} not found, using 127.0.0.1", interface)); + "127.0.0.1".to_string() + }) + } + Err(_) => { + print_startup(" Failed to enumerate interfaces, using 127.0.0.1"); + "127.0.0.1".to_string() + } + }; + + let interface_subnet = format!("{}/24", interface_ip); + ctx.insert_interface(interface, &interface_subnet).await?; + print_startup(&format!("Interface {} configured with IP {}", interface, interface_ip)); + print_startup(&format!("Interface {} configured", interface)); + + // 4) Check screenshot system + print_startup("Checking browser dependencies for screenshots..."); + if crate::http::check_browser_availability().await { + print_startup("Screenshot system ready (Chrome/Chromium detected)"); + } else { + print_startup(" Chrome/Chromium not found - screenshots will be disabled"); + print_startup(" Install Chrome/Chromium to enable web screenshots"); + } + + // 5) Store workflow path in database + sqlx::query("INSERT INTO settings (run_id, workflow_path) VALUES (?1, ?2)") + .bind(&ctx.run_id) + .bind(workflow_path) + .execute(&ctx.pool) + .await?; + print_startup(&format!("Workflow configured: {}", workflow_path)); + + print_startup("Auto-initialization complete"); + Ok(()) +} + +// Context: Privilege validation for network scanning operations +// Operation: Tests sudo access non-interactively, returns error if not available +async fn ensure_sudo() -> Result<(), Box> { + // 1) If already running as root, skip sudo altogether + if unsafe { nix::libc::geteuid() } == 0 { + return Ok(()); + } + + // 2) Otherwise test sudo access non-interactively + let status = tokio::process::Command::new("sudo") + .arg("-n") // non-interactive + .arg("-v") // validate + .status() + .await?; + + if !status.success() { + return Err("Sudo access required but not available. Please run 'sudo -v' first or run as root.".into()); + } + + Ok(()) +} + +// Context: New project creation and initialization workflow +// Operation: Creates database, starts web server, launches workflow execution in background +async fn handle_create_project( + project_name: &str, + project_directory: &str, + interface: &str, + workflow_path: &str, + web_port: u16, + shutdown_rx: watch::Receiver, +) -> Result> { + print_startup(&format!("Creating project: {}", project_name)); + print_startup(&format!("Project directory: {}", project_directory)); + print_startup(&format!("Network interface: {}", interface)); + print_startup(&format!("Workflow: {}", workflow_path)); + print_startup(&format!("Web port: {}", web_port)); + + // 0) Ensure sudo access + ensure_sudo().await?; + + // 0.5) Validate required tools before proceeding + validate_required_tools().await?; + + // 1) Create database context + let ctx = ProjectContext::new(project_name, project_directory).await?; + + // 2) Initialize database logging - CRITICAL: This enables dual output mode + crate::utilities::init_db_logger(ctx.clone()); + + // H7: Background log flush task — drains buffer every 3s in a single transaction + let log_flush_pool = ctx.pool.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(3)).await; + crate::sql::logging::flush_log_buffer(&log_flush_pool).await; + } + }); + + print_startup(&format!( + "Database initialized at {}", + ctx.db_path + )); + + // 3) Auto initialization (replaces init.rs) + auto_init(&ctx, interface, workflow_path).await?; + + // 4) Initialize concurrency — aggressive baseline, limits adjustable via web settings + crate::workflow::initialize_concurrency(crate::workflow::ConcurrencyConfig::aggressive()); + // Restore any previously saved limits over the preset defaults + if let Ok(Some(saved)) = ctx.load_concurrency_config().await { + use crate::workflow::concurrency::{set_concurrency_limit, OperationType}; + set_concurrency_limit(OperationType::Global, saved.global); + set_concurrency_limit(OperationType::NmapFullTcp, saved.nmap_full_tcp); + set_concurrency_limit(OperationType::NmapService, saved.nmap_service); + set_concurrency_limit(OperationType::SubnetPhase1, saved.subnet_phase1); + set_concurrency_limit(OperationType::SubnetPhase2, saved.subnet_phase2); + set_concurrency_limit(OperationType::HttpDiscovery, saved.http_discovery); + print_startup("Restored concurrency limits from database"); + } + print_startup("Concurrency initialized (aggressive baseline; adjust via Settings)"); + + // 4.5) Wire up task history persistence + *crate::workflow::constants::CURRENT_RUN_ID.write().unwrap() = Some(ctx.run_id.clone()); + let (hist_tx, mut hist_rx) = tokio::sync::mpsc::unbounded_channel::(); + *crate::workflow::constants::TASK_HISTORY_TX.write().unwrap() = Some(hist_tx); + let ctx_hist = ctx.clone(); + tokio::spawn(async move { + while let Some(record) = hist_rx.recv().await { + if let Err(e) = ctx_hist.insert_task_history(&record).await { + tag_debug(&format!("task_history write failed: {}", e)); + } + } + }); + + // 5) Get interface IP for web server binding + let interface_ip = ctx.get_selected_interface_ip().await.unwrap_or_else(|e| { + print_startup(&format!(" Could not get selected interface IP: {}, falling back to 127.0.0.1", e)); + "127.0.0.1".to_string() + }); + + // 6) Start web server with proper startup coordination + print_startup("Starting web interface..."); + let web_ctx = ctx.clone(); + let web_shutdown_rx = shutdown_rx.clone(); + let web_interface_ip = interface_ip.clone(); + + let web_server_handle = tokio::spawn(async move { + if let Err(e) = crate::web::run_server_on_interface(web_ctx, &web_interface_ip, web_port, web_shutdown_rx).await { + tag_info(&format!("Web interface error: {}", e)); + } + }); + + // 7) Give web server time to start and bind to port + sleep(Duration::from_millis(2000)).await; + + // 8) Verify web server is running by checking if port is bound + let web_url = format!("http://{}:{}", interface_ip, web_port); + print_startup(&format!("Web interface available at {} (admin username: admin)", web_url)); + print_startup("Access the dashboard to monitor reconnaissance progress"); + print_startup("Starting workflow execution in background..."); + + // 9) Start workflow execution in background (non-blocking) + let workflow_ctx = ctx.clone(); + let workflow_path_clone = workflow_path.to_string(); + let workflow_shutdown_rx = shutdown_rx.clone(); + + tokio::spawn(async move { + if let Err(e) = run_workflow(&workflow_path_clone, &workflow_ctx, workflow_shutdown_rx).await { + tag_info(&format!("Workflow execution error: {}", e)); + } + }); + + // 10) Wait for shutdown signal without blocking startup completion + let mut shutdown_clone = shutdown_rx.clone(); + tokio::select! { + _ = shutdown_clone.changed() => { + print_startup("Shutdown signal received"); + } + _ = tokio::signal::ctrl_c() => { + print_startup("Ctrl-C received, initiating shutdown"); + + // Give workflow time to process the Ctrl+C and shut down gracefully + tokio::time::sleep(Duration::from_secs(5)).await; + } + _ = web_server_handle => { + print_startup("Web server exited"); + } + } + + Ok(ctx) +} + +// Context: Main application entry point and command dispatch +// Operation: Parses CLI arguments, validates inputs, dispatches to appropriate handlers +async fn async_main() -> Result<(), Box> { + // 1) Parse CLI + let cli = Cli::parse(); + + // 2) Handle help and info flags first + if cli.extended_help { + print_extended_help(); + return Ok(()); + } + + if cli.check_deps { + check_system_dependencies().await; + return Ok(()); + } + + // 3) Re-exec logic (keep existing but improve error message) + if env::var("ROFMAD_LAUNCHED").is_err() && env::args().len() > 1 { + unsafe { env::set_var("ROFMAD_LAUNCHED", "1"); } + let exe = env::current_exe() + .map_err(|e| format!("Failed to get executable path: {}", e))?; + let result = Command::new(&exe) + .args(env::args_os().skip(1)) + .exec(); + return Err(format!("Failed to re-execute ROFMAD: {:?}", result).into()); + } + + // 4) Create shutdown channel and set verbose mode + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + // 5) Set verbose mode before any logging + if cli.verbose { + VERBOSE.store(true, Ordering::SeqCst); + println!("Verbose logging enabled"); + } + + + let logo = logo_line(); + println!("{}", logo); + + // 6) Dispatch subcommands with enhanced error handling + match &cli.command { + Some(Commands::CreateProject { project_name, project_directory }) => { + let interface = cli.interface.as_ref() + .ok_or("--interface is required for create-project. Example: --interface eth0")?; + let workflow = cli.workflow.as_ref() + .ok_or_else(|| format!("--workflow is required for create-project. Example: --workflow {}/default.yaml", crate::config::paths::SystemPaths::WORKFLOWS_DIRECTORY))?; + + if !Path::new(workflow).exists() { + return Err(format!("Workflow file not found: {}", workflow).into()); + } + if !project_name.chars().all(|c| c.is_alphanumeric() || c == '_' || c == '-') { + return Err("Project name must contain only alphanumeric characters, underscores, and hyphens".into()); + } + if let Err(e) = std::fs::create_dir_all(project_directory) { + return Err(format!("Failed to create project directory '{}': {}", project_directory, e).into()); + } + + let _ctx = handle_create_project( + project_name, + project_directory, + interface, + workflow, + cli.web_port, + shutdown_rx.clone(), + ).await?; + } + + Some(Commands::ResumeProject { path }) => { + let interface = cli.interface.as_ref() + .ok_or("--interface is required for resume-project. Example: --interface eth0")?; + let workflow = cli.workflow.as_ref() + .ok_or_else(|| format!("--workflow is required for resume-project. Example: --workflow {}/default.yaml", crate::config::paths::SystemPaths::WORKFLOWS_DIRECTORY))?; + + if !std::path::Path::new(path).exists() { + return Err(format!("Database file not found: {}", path).into()); + } + if !path.ends_with(".db") { + return Err("Resume path must be a .db file".into()); + } + if !Path::new(workflow).exists() { + return Err(format!("Workflow file not found: {}", workflow).into()); + } + + // Resolve interface IP before doing anything else — fail fast with a clear message + let interface_ip = match get_if_addrs() { + Ok(ifaces) => { + let available: Vec = ifaces.iter() + .filter(|i| !i.is_loopback()) + .map(|i| i.name.clone()) + .collect::>() + .into_iter() + .collect::>(); + ifaces.into_iter() + .find(|i| i.name == *interface) + .and_then(|i| match i.addr { IfAddr::V4(v4) => Some(v4.ip.to_string()), _ => None }) + .ok_or_else(|| format!( + "Interface '{}' not found. Available interfaces: {}", + interface, + if available.is_empty() { "none detected".to_string() } else { available.join(", ") } + ))? + } + Err(e) => return Err(format!("Failed to enumerate network interfaces: {}", e).into()), + }; + + print_startup(&format!("Resuming project from: {}", path)); + print_startup(&format!("Using interface: {} ({})", interface, interface_ip)); + print_startup(&format!("Using workflow: {}", workflow)); + + // Ensure sudo access + ensure_sudo().await?; + + // Validate required tools before proceeding + validate_required_tools().await?; + + let ctx = ProjectContext::resume(path).await + .map_err(|e| format!("Failed to resume project: {}", e))?; + + // Initialize database logging for resumed project + crate::utilities::init_db_logger(ctx.clone()); + + // H7: Background log flush task for resume mode + let log_flush_pool = ctx.pool.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(3)).await; + crate::sql::logging::flush_log_buffer(&log_flush_pool).await; + } + }); + + RESUME_MODE.store(true, Ordering::SeqCst); + + // Store resolved interface IP/subnet in database + ctx.insert_interface(interface, &format!("{}/24", interface_ip)).await?; + sqlx::query("INSERT OR REPLACE INTO settings (run_id, workflow_path) VALUES (?1, ?2)") + .bind(&ctx.run_id) + .bind(workflow) + .execute(&ctx.pool) + .await?; + + // Initialize concurrency — aggressive baseline, limits adjustable via web settings + crate::workflow::initialize_concurrency(crate::workflow::ConcurrencyConfig::aggressive()); + // Restore any previously saved limits + if let Ok(Some(saved)) = ctx.load_concurrency_config().await { + use crate::workflow::concurrency::{set_concurrency_limit, OperationType}; + set_concurrency_limit(OperationType::Global, saved.global); + set_concurrency_limit(OperationType::NmapFullTcp, saved.nmap_full_tcp); + set_concurrency_limit(OperationType::NmapService, saved.nmap_service); + set_concurrency_limit(OperationType::SubnetPhase1, saved.subnet_phase1); + set_concurrency_limit(OperationType::SubnetPhase2, saved.subnet_phase2); + set_concurrency_limit(OperationType::HttpDiscovery, saved.http_discovery); + print_startup("Restored concurrency limits from database"); + } + + // Wire up task history persistence + *crate::workflow::constants::CURRENT_RUN_ID.write().unwrap() = Some(ctx.run_id.clone()); + let (hist_tx, mut hist_rx) = tokio::sync::mpsc::unbounded_channel::(); + *crate::workflow::constants::TASK_HISTORY_TX.write().unwrap() = Some(hist_tx); + let ctx_hist = ctx.clone(); + tokio::spawn(async move { + while let Some(record) = hist_rx.recv().await { + if let Err(e) = ctx_hist.insert_task_history(&record).await { + tag_debug(&format!("task_history write failed: {}", e)); + } + } + }); + + // Start web server on the already-resolved interface IP + print_startup("Starting web interface..."); + let web_url = format!("http://{}:{}", interface_ip, cli.web_port); + let web_ctx = ctx.clone(); + let web_shutdown_rx = shutdown_rx.clone(); + let web_interface_ip = interface_ip.clone(); + + let web_server_handle = tokio::spawn(async move { + if let Err(e) = crate::web::run_server_on_interface(web_ctx, &web_interface_ip, cli.web_port, web_shutdown_rx).await { + tag_info(&format!("Web interface error: {}", e)); + } + }); + + // Give web server time to start + sleep(Duration::from_millis(2000)).await; + print_startup(&format!("Web interface available at {} (admin username: admin)", web_url)); + + // Start workflow in background + let workflow_ctx = ctx.clone(); + let workflow_clone = workflow.to_string(); + let workflow_shutdown_rx = shutdown_rx.clone(); + + tokio::spawn(async move { + if let Err(e) = run_workflow(&workflow_clone, &workflow_ctx, workflow_shutdown_rx).await { + tag_info(&format!("Workflow execution error: {}", e)); + } + }); + + // Wait for shutdown + let mut shutdown_clone = shutdown_rx.clone(); + tokio::select! { + _ = shutdown_clone.changed() => { + print_startup("Shutdown signal received"); + } + _ = tokio::signal::ctrl_c() => { + print_startup("Ctrl-C received, initiating shutdown"); + + // Give workflow time to process the Ctrl+C and shut down gracefully + tokio::time::sleep(Duration::from_secs(5)).await; + } + _ = web_server_handle => { + print_startup("Web server exited"); + } + } + } + + Some(Commands::WebOnly { path, interface, port }) => { + // Validate database file exists + if !std::path::Path::new(path).exists() { + return Err(format!("Database file not found: {}", path).into()); + } + + // Validate it's a SQLite database + if !path.ends_with(".db") { + return Err("Web-only path must be a .db file".into()); + } + + print_startup(&format!("Starting web-only mode for: {}", path)); + print_startup(&format!("Web interface will be available at http://{}:{}", interface, port)); + + let ctx = ProjectContext::resume(path).await + .map_err(|e| format!("Failed to open database: {}", e))?; + + // Initialize database logging for web-only mode + crate::utilities::init_db_logger(ctx.clone()); + + // Initialize concurrency for stats display (no actual tasks will run) + crate::workflow::initialize_concurrency(crate::workflow::ConcurrencyConfig::aggressive()); + crate::workflow::WEB_ONLY_MODE.store(true, Ordering::SeqCst); + + // Start web server and wait indefinitely + let web_shutdown_rx = shutdown_rx.clone(); + let web_result = crate::web::run_server_on_interface(ctx, interface, *port, web_shutdown_rx).await; + + if let Err(e) = web_result { + return Err(format!("Web server error: {}", e).into()); + } + } + + None => { + println!("No command specified. Run with --help for usage or -H for workflow/module reference."); + return Ok(()); + } + } + + print_startup("Application shutdown complete"); + Ok(()) +} + +// Context: Application bootstrap and runtime configuration +// Operation: Configures multi-threaded Tokio runtime and executes async main +fn main() -> Result<(), Box> { + let threads = num_cpus::get() * 2; + let rt = Builder::new_multi_thread() + .worker_threads(threads) + .enable_all() + .build()?; + rt.block_on(async_main())?; + Ok(()) +} \ No newline at end of file diff --git a/src/sql/actions.rs b/src/sql/actions.rs new file mode 100755 index 0000000..bb299be --- /dev/null +++ b/src/sql/actions.rs @@ -0,0 +1,147 @@ +// src/sql/actions.rs +use std::error::Error; +use serde_json; +use crate::tag::{automatic_tag, TriggerContext}; +use crate::workflow::{dispatch_event, Hook}; +use crate::utilities::tag_debug; +use crate::VERBOSE; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: Action tracking system to prevent duplicate execution and manage action state + // Operation: Records action start in database and returns whether this is a new start or already started + pub async fn mark_action_started(&self, ip: &str, action_name: &str) -> Result> { + // Check if already started + let already_started: bool = sqlx::query_scalar("SELECT started FROM action_status WHERE run_id = ? AND ip = ? AND action_name = ?") + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + if already_started { + return Ok(false); + } + + // Insert or update + sqlx::query("INSERT OR REPLACE INTO action_status (run_id, ip, action_name, started, complete) VALUES (?, ?, ?, TRUE, FALSE)") + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .execute(&self.pool) + .await?; + + // A new action starting means workflow is not yet complete + sqlx::query("UPDATE targets SET user_workflow_complete = FALSE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await + .ok(); + + Ok(true) + } + + // Context: Action tracking system to record completion status for workflow management + // Operation: Updates database to mark specific action as complete for given IP + pub async fn mark_action_complete(&self, ip: &str, action_name: &str) -> Result<(), Box> { + sqlx::query("UPDATE action_status SET complete = TRUE WHERE run_id = ? AND ip = ? AND action_name = ?") + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Action status verification for workflow decision making and dependency checking + // Operation: Queries database to check if specific action has been marked as complete + pub async fn is_action_complete(&self, ip: &str, action_name: &str) -> Result> { + let complete: bool = sqlx::query_scalar("SELECT complete FROM action_status WHERE run_id = ? AND ip = ? AND action_name = ?") + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(complete) + } + + // Context: Action output storage for analysis, reporting, and automated processing + // Operation: Stores structured action output as a row in the action_outputs table — one row per action per host + pub async fn insert_action_output( + &self, + ip: &str, + action_name: &str, + output: &str, + ) -> Result<(), Box> { + let output_value: serde_json::Value = serde_json::from_str(output) + .unwrap_or_else(|_| serde_json::json!({"legacy_output": output})); + let output_json = serde_json::to_string(&output_value)?; + + sqlx::query( + "INSERT OR REPLACE INTO action_outputs (run_id, ip, action_name, output_json) VALUES (?, ?, ?, ?)" + ) + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .bind(&output_json) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Stored action output for '{}' on {}", action_name, ip)); + } + self.log_event("DEBUG", &format!("Stored structured output for '{}' on {}", action_name, ip)) + .await + .ok(); + + // Spawn tagging + let ctx_clone = self.clone(); + let ip_clone = ip.to_string(); + let action_name_clone = action_name.to_string(); + tokio::spawn(async move { + if let Err(e) = automatic_tag(&ctx_clone, &ip_clone, TriggerContext::ActionOutput(action_name_clone.clone())).await { + tag_debug(&format!("Action output tagging failed for {}: {}", ip_clone, e)); + } + }); + + dispatch_event(Hook::ActionOutput, output, self); + Ok(()) + } + + // Context: Action-to-tag correlation tracking + // Operation: Marks action output row as having produced new tags for the host + pub async fn mark_action_tagged(&self, ip: &str, action_name: &str) -> Result<(), Box> { + sqlx::query("UPDATE action_outputs SET tagged = TRUE WHERE run_id = ? AND ip = ? AND action_name = ?") + .bind(&self.run_id) + .bind(ip) + .bind(action_name) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Host detail page and action output display + // Operation: Returns all action outputs for a host ordered by action_name for deterministic display + pub async fn get_action_outputs_for_host( + &self, + ip: &str, + ) -> Result, Box> { + let rows = sqlx::query( + "SELECT action_name, output_json, COALESCE(tagged, 0) as tagged FROM action_outputs WHERE run_id = ? AND ip = ? ORDER BY action_name" + ) + .bind(&self.run_id) + .bind(ip) + .fetch_all(&self.pool) + .await?; + + Ok(rows.into_iter().map(|r| { + let name: String = sqlx::Row::get(&r, "action_name"); + let json: String = sqlx::Row::get(&r, "output_json"); + let tagged: bool = sqlx::Row::get::(&r, "tagged") != 0; + (name, json, tagged) + }).collect()) + } +} \ No newline at end of file diff --git a/src/sql/context.rs b/src/sql/context.rs new file mode 100755 index 0000000..ffe7d11 --- /dev/null +++ b/src/sql/context.rs @@ -0,0 +1,98 @@ +// src/sql/context.rs +use sqlx::query; +use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; +use std::error::Error; +use uuid::Uuid; +use crate::utilities::tag_info; +use super::database::initialize_db; + +#[derive(Clone, Debug)] +pub struct ProjectContext { + pub workflow_path: String, + pub db_path: String, + pub run_id: String, + pub pool: Pool, +} + +impl ProjectContext { + // Context: Project initialization system for creating new database-backed contexts + // Operation: Creates new database, initializes schema, generates run ID, and sets up connection pool + pub async fn new(name: &str, directory: &str) -> Result> { + let db_path = format!("{}/{}.db", directory.trim(), name.trim()); + let conn_str = format!("sqlite://{}?mode=rwc", db_path); + + let pool = SqlitePoolOptions::new() + .max_connections(20) + .connect(&conn_str) + .await?; + initialize_db(&pool).await?; + let _ = sqlx::query("PRAGMA journal_mode = WAL").execute(&pool).await; + + let run_id = Uuid::new_v4().to_string(); + let ctx = ProjectContext { + workflow_path: String::new(), + db_path: db_path.clone(), + run_id: run_id.clone(), + pool: pool.clone(), + }; + + // Reset all started flags for new project + ctx.reset_all_started_flags().await?; + + // store run_id in metadata for resume + let _ = query( + "INSERT OR REPLACE INTO metadata (key, value) VALUES ('run_id', ?1)", + ) + .bind(&run_id) + .execute(&pool) + .await; + Ok(ctx) + } + + // Context: Project resumption system for continuing work with existing database + // Operation: Connects to existing database, retrieves run ID and workflow path, validates state + pub async fn resume(path: &str) -> Result> { + let conn_str = format!("sqlite://{}?mode=rwc", path.trim()); + let pool = SqlitePoolOptions::new() + .max_connections(20) + .connect(&conn_str) + .await?; + initialize_db(&pool).await?; + let _ = query("PRAGMA journal_mode = WAL").execute(&pool).await; + + // fetch the stored run_id + let (run_id,): (String,) = sqlx::query_as( + "SELECT value FROM metadata WHERE key = 'run_id'" + ) + .fetch_one(&pool) + .await?; + + // fetch the stored workflow_path with fallback + let workflow_path: String = sqlx::query_scalar( + "SELECT workflow_path FROM settings WHERE run_id = ?" + ) + .bind(&run_id) + .fetch_one(&pool) + .await + .unwrap_or_else(|_| String::new()); + + let ctx = ProjectContext { + workflow_path, + db_path: path.to_string(), + run_id: run_id.clone(), + pool: pool.clone(), + }; + + + tag_info(&format!("Resumed DB at {}", path)); + + // If workflow_path is empty, warn the user + if ctx.workflow_path.is_empty() { + tag_info(" No workflow path found in database - you may need to specify it manually"); + } else { + tag_info(&format!("Using workflow: {}", ctx.workflow_path)); + } + + Ok(ctx) + } +} \ No newline at end of file diff --git a/src/sql/database.rs b/src/sql/database.rs new file mode 100755 index 0000000..e634247 --- /dev/null +++ b/src/sql/database.rs @@ -0,0 +1,500 @@ +// src/sql/database.rs +use sqlx::{Pool, Sqlite, Row}; +use std::error::Error; + +// Context: Database schema initialization for project data storage and management +// Operation: Creates all required tables and ensures schema compatibility with migrations +pub async fn initialize_db(pool: &Pool) -> Result<(), Box> { + // create targets table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS targets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + ip TEXT NOT NULL, + ports TEXT NOT NULL + ) + "#, + ) + .execute(pool) + .await?; + + // create subnets table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS subnets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + target TEXT NOT NULL, + phase1_hosts TEXT DEFAULT '', + masscan_scanned INTEGER DEFAULT 0, + explorer_scanned INTEGER DEFAULT 0, + phase1_complete BOOLEAN DEFAULT FALSE, + phase2_complete BOOLEAN DEFAULT FALSE + ) + "#, + ) + .execute(pool) + .await?; + + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS http_services ( + run_id TEXT NOT NULL, + ip TEXT NOT NULL, + port TEXT NOT NULL, + service_info TEXT NOT NULL, + PRIMARY KEY (run_id, ip, port) + ) + "#, + ) + .execute(pool) + .await?; + + // create interfaces table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS interfaces ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + interface TEXT NOT NULL, + subnet TEXT NOT NULL + ) + "#, + ) + .execute(pool) + .await?; + + // create blacklist table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS blacklist ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + cidr TEXT NOT NULL, + UNIQUE(run_id, cidr) + ) + "#, + ) + .execute(pool) + .await?; + + // metadata table to store the original run_id + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + "#, + ) + .execute(pool) + .await?; + + // metadata table to store the original run_id + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS settings ( + run_id TEXT PRIMARY KEY, + workflow_path TEXT NOT NULL + ) + "#, + ) + .execute(pool) + .await?; + + // pending_tasks table to record in‐flight work + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS pending_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + target TEXT NOT NULL, + status TEXT NOT NULL -- "active" or "queued" + ) + "#, + ) + .execute(pool) + .await?; + + // create logs table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + level TEXT NOT NULL, + message TEXT NOT NULL + ) + "#, + ) + .execute(pool) + .await?; + + // Add analyzed column to targets table + let info_analyzed = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_analyzed.iter().any(|r| r.get::("name") == "analyzed") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN analyzed BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Add interesting column to targets table + let info_interesting = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_interesting.iter().any(|r| r.get::("name") == "interesting") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN interesting BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Add nuclei_reviewed column to targets table + let info_nuclei_reviewed = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_nuclei_reviewed.iter().any(|r| r.get::("name") == "nuclei_reviewed") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN nuclei_reviewed BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // ensure nmap_scan column on targets + let info = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info.iter().any(|r| r.get::("name") == "nmap_scan") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN nmap_scan TEXT DEFAULT '{}'"#) + .execute(pool) + .await?; + } + + // ensure nmap_xml column on targets (XML output for structured parsing; raw text kept in nmap_scan) + let info_nmap_xml = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_nmap_xml.iter().any(|r| r.get::("name") == "nmap_xml") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN nmap_xml TEXT DEFAULT '{}'"#) + .execute(pool) + .await?; + } + + let info_service_started = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + + if !info_service_started.iter().any(|r| r.get::("name") == "service_scan_started") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN service_scan_started BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Add slow scan timeout column to targets + let info_slow = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_slow.iter().any(|r| r.get::("name") == "slow_scan_timeout") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN slow_scan_timeout BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Ensure user_workflow_complete column + let info_wf_complete = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_wf_complete.iter().any(|r| r.get::("name") == "user_workflow_complete") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN user_workflow_complete BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Ensure full_scan_started column + let info_full_started = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_full_started.iter().any(|r| r.get::("name") == "full_scan_started") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN full_scan_started BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Ensure full_scan_complete column + let info_full_complete = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_full_complete.iter().any(|r| r.get::("name") == "full_scan_complete") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN full_scan_complete BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Ensure screenshots_started column + let info_screenshot_started = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_screenshot_started.iter().any(|r| r.get::("name") == "screenshots_started") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN screenshots_started BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Ensure screenshots_complete column + let info_screenshot_complete = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_screenshot_complete.iter().any(|r| r.get::("name") == "screenshots_complete") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN screenshots_complete BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // Create action_status table + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS action_status ( + run_id TEXT NOT NULL, + ip TEXT NOT NULL, + action_name TEXT NOT NULL, + started BOOLEAN DEFAULT FALSE, + complete BOOLEAN DEFAULT FALSE, + PRIMARY KEY (run_id, ip, action_name) + ) + "#, + ) + .execute(pool) + .await?; + + // Normalized action outputs table — replaces targets.action_outputs JSON blob + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS action_outputs ( + run_id TEXT NOT NULL, + ip TEXT NOT NULL, + action_name TEXT NOT NULL, + output_json TEXT NOT NULL DEFAULT '{}', + tagged BOOLEAN DEFAULT FALSE, + created_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (run_id, ip, action_name) + ) + "#, + ) + .execute(pool) + .await?; + + // Create users table for role-based authentication + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT UNIQUE NOT NULL, + password_hash TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin', 'player')), + enabled BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + "#, + ) + .execute(pool) + .await?; + + // ensure hostname column on targets + let info_hostname = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_hostname.iter().any(|r| r.get::("name") == "hostname") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN hostname TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure automated_tags column on targets + let info_automated_tags = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_automated_tags.iter().any(|r| r.get::("name") == "automated_tags") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN automated_tags TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure manual_tags column on targets + let info_manual_tags = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_manual_tags.iter().any(|r| r.get::("name") == "manual_tags") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN manual_tags TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure refined_tags column on targets + let info_refined_tags = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_refined_tags.iter().any(|r| r.get::("name") == "refined_tags") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN refined_tags TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure action_tags column on targets (tags generated by specific_action pattern matching) + let info_action_tags = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_action_tags.iter().any(|r| r.get::("name") == "action_tags") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN action_tags TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure phase1_hosts column on subnets + let info_sub = sqlx::query("PRAGMA table_info(subnets);") + .fetch_all(pool) + .await?; + if !info_sub.iter().any(|r| r.get::("name") == "phase1_hosts") { + sqlx::query(r#"ALTER TABLE subnets ADD COLUMN phase1_hosts TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // Add AD domain column to targets table + let info_domain = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_domain.iter().any(|r| r.get::("name") == "ad_domain") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN ad_domain TEXT DEFAULT ''"#) + .execute(pool) + .await?; + } + + // ensure masscan_scanned column on subnets + let info_sub2 = sqlx::query("PRAGMA table_info(subnets);") + .fetch_all(pool) + .await?; + if !info_sub2.iter().any(|r| r.get::("name") == "masscan_scanned") { + sqlx::query(r#"ALTER TABLE subnets ADD COLUMN masscan_scanned INTEGER DEFAULT 0"#) + .execute(pool) + .await?; + } + + // ensure 'rescan' column on subnets + let info = sqlx::query("PRAGMA table_info(subnets);") + .fetch_all(pool) + .await?; + if !info.iter().any(|r| r.get::("name") == "rescan") { + sqlx::query(r#"ALTER TABLE subnets ADD COLUMN rescan INTEGER DEFAULT 0"#) + .execute(pool) + .await?; + } + + // NEW: HTTP enumeration columns on targets + let info_http = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + + if !info_http.iter().any(|r| r.get::("name") == "service_scan_complete") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN service_scan_complete BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + if !info_http.iter().any(|r| r.get::("name") == "http_discovery_started") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN http_discovery_started BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + if !info_http.iter().any(|r| r.get::("name") == "http_discovery_complete") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN http_discovery_complete BOOLEAN DEFAULT FALSE"#) + .execute(pool) + .await?; + } + + // ensure port_services column on targets (port→service JSON map populated after nmap -sV) + let info_port_services = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(pool) + .await?; + if !info_port_services.iter().any(|r| r.get::("name") == "port_services") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN port_services TEXT DEFAULT '{}'"#) + .execute(pool) + .await?; + } + + // Add concurrency_config column to settings if missing (migration for existing DBs) + let info_settings = sqlx::query("PRAGMA table_info(settings)") + .fetch_all(pool) + .await?; + if !info_settings.iter().any(|r| r.get::("name") == "concurrency_config") { + sqlx::query(r#"ALTER TABLE settings ADD COLUMN concurrency_config TEXT DEFAULT NULL"#) + .execute(pool) + .await?; + } + + // Unique constraint on (run_id, ip) — enables INSERT OR IGNORE dedup in insert_ip() + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_targets_unique_run_ip ON targets(run_id, ip)" + ).execute(pool).await?; + + // Composite index for per-host lookups (run_id + ip) — used by almost every query + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_targets_run_ip ON targets(run_id, ip)" + ).execute(pool).await?; + + // Index for status flag queries used in polling loop and dashboard + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_targets_run_status ON targets(run_id, full_scan_complete, service_scan_complete, http_discovery_complete, screenshots_complete)" + ).execute(pool).await?; + + // Index for action_status dedup lookups (run_id + ip + action_name) + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_action_status_lookup ON action_status(run_id, ip, action_name)" + ).execute(pool).await?; + + // Index for action_outputs per-host lookups + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_action_outputs_lookup ON action_outputs(run_id, ip)" + ).execute(pool).await?; + + // Index for http_services per-host lookups + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_http_services_run_ip ON http_services(run_id, ip)" + ).execute(pool).await?; + + // Index for log queries filtered by level + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_logs_run_level ON logs(run_id, level)" + ).execute(pool).await?; + + // Index for subnet queries scoped by run_id + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_subnets_run ON subnets(run_id)" + ).execute(pool).await?; + + // Task history table for persistent output storage + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS task_history ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + task_name TEXT NOT NULL, + target TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER NOT NULL, + status TEXT NOT NULL, + output TEXT NOT NULL DEFAULT '' + ) + "#, + ) + .execute(pool) + .await?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_task_history_run ON task_history(run_id, started_at DESC)" + ).execute(pool).await?; + + Ok(()) +} \ No newline at end of file diff --git a/src/sql/hosts.rs b/src/sql/hosts.rs new file mode 100755 index 0000000..cb0e1a7 --- /dev/null +++ b/src/sql/hosts.rs @@ -0,0 +1,851 @@ +// src/sql/hosts.rs +use sqlx::Row; +use std::error::Error; +use std::collections::{HashSet, HashMap}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use crate::workflow::{dispatch_event, Hook}; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::context::ProjectContext; + +// Snapshot of a host row fetched once per polling tick — avoids per-host/per-action DB queries +#[derive(Debug, Clone)] +pub struct HostSnapshot { + pub ip: String, + pub ports: String, + pub automated_tags: String, + pub manual_tags: String, + pub refined_tags: String, + pub action_tags: String, + pub full_scan_complete: bool, + pub service_scan_complete: bool, + pub http_discovery_complete: bool, + pub has_http_services: bool, + // JSON map of port → normalized service name; populated after nmap -sV completes + // e.g. {"22":"ssh","445":"smb","1433":"mssql"} + pub port_services: String, + // (port, protocol) pairs for foreach=http_service fan-out — derived from http_services table + pub http_service_list: Vec<(String, String)>, +} + +#[derive(Debug, Clone)] +pub struct DashboardStatistics { + pub total_hosts: usize, + pub total_subnets: usize, + pub hosts_with_open_ports: usize, + pub total_open_ports: usize, + pub hosts_with_http_services: usize, + pub total_screenshots: usize, +} + +impl ProjectContext { + // Context: Host analysis tracking to prevent duplicate work and provide workflow management + // Operation: Updates host analyzed status with transaction safety and timeout protection + pub async fn mark_host_analyzed(&self, ip: &str, analyzed: bool) -> Result<(), Box> { + // Use a transaction to prevent race conditions + let mut tx = self.pool.begin().await?; + + // Add timeout to prevent hanging + let timeout_duration = std::time::Duration::from_secs(5); + + let result = tokio::time::timeout(timeout_duration, async { + sqlx::query("UPDATE targets SET analyzed = ? WHERE run_id = ? AND ip = ?") + .bind(analyzed) + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await + }).await; + + match result { + Ok(Ok(query_result)) => { + if query_result.rows_affected() > 0 { + tx.commit().await?; + + let status = if analyzed { "analyzed" } else { "unanalyzed" }; + let msg = format!("Host {} marked as {}", ip, status); + self.log_event("INFO", &msg).await.ok(); + + Ok(()) + } else { + tx.rollback().await?; + Err(format!("Host {} not found", ip).into()) + } + } + Ok(Err(e)) => { + tx.rollback().await?; + Err(e.into()) + } + Err(_) => { + tx.rollback().await?; + Err("Database operation timed out".into()) + } + } + } + + // Context: Analysis status verification for workflow decision making and filtering + // Operation: Queries database to check if specific host has been marked as analyzed + pub async fn is_host_analyzed(&self, ip: &str) -> Result> { + let analyzed: bool = sqlx::query_scalar("SELECT analyzed FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(analyzed) + } + + // Context: Analysis queue management for web interface and automated processing + // Operation: Retrieves hosts that completed all reconnaissance phases but await analysis + pub async fn get_hosts_ready_for_analysis(&self) -> Result, Box> { + let rows = sqlx::query( + "SELECT ip, + COALESCE(hostname, '') as hostname, + COALESCE(ports, '') as ports, + COALESCE(ad_domain, '') as ad_domain, + COALESCE(full_scan_complete, 0) as full_scan_complete, + COALESCE(service_scan_complete, 0) as service_scan_complete, + COALESCE(http_discovery_complete, 0) as http_discovery_complete, + COALESCE(screenshots_complete, 0) as screenshots_complete, + (SELECT COUNT(*) FROM action_outputs WHERE run_id = targets.run_id AND ip = targets.ip) as action_count, + COALESCE(analyzed, 0) as analyzed + FROM targets + WHERE run_id = ? + AND full_scan_complete = 1 + AND service_scan_complete = 1 + AND http_discovery_complete = 1 + AND screenshots_complete = 1 + AND user_workflow_complete = 1 + AND analyzed = 0 + ORDER BY ip" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut hosts = Vec::new(); + for row in rows { + hosts.push(crate::web::handlers::parse_host_summary(row, self).await); + } + Ok(hosts) + } + + // Context: Scan performance tracking for optimization and troubleshooting purposes + // Operation: Records that host experienced scan timeouts and may need special handling + pub async fn mark_host_slow(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET slow_scan_timeout = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Performance status verification for scan optimization and reporting + // Operation: Queries database to check if host has been flagged for slow scan performance + pub async fn is_host_slow(&self, ip: &str) -> Result> { + let slow: bool = sqlx::query_scalar("SELECT slow_scan_timeout FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(slow) + } + + // Context: Target prioritization system for focusing attention on valuable hosts + // Operation: Updates host interest status and logs event for analysis tracking + pub async fn mark_host_interesting(&self, ip: &str, interesting: bool) -> Result<(), Box> { + sqlx::query("UPDATE targets SET interesting = ? WHERE run_id = ? AND ip = ?") + .bind(interesting) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + let status = if interesting { "interesting" } else { "not interesting" }; + let msg = format!("Host {} marked as {}", ip, status); + self.log_event("INFO", &msg).await.ok(); + + Ok(()) + } + + // Context: Priority filtering system for focusing on high-value targets + // Operation: Queries database to check if host has been marked as interesting + pub async fn is_host_interesting(&self, ip: &str) -> Result> { + let interesting: bool = sqlx::query_scalar("SELECT interesting FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(interesting) + } + + pub async fn mark_host_nuclei_reviewed(&self, ip: &str, reviewed: bool) -> Result<(), Box> { + sqlx::query("UPDATE targets SET nuclei_reviewed = ? WHERE run_id = ? AND ip = ?") + .bind(reviewed) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + let status = if reviewed { "nuclei-reviewed" } else { "nuclei-unreviewed" }; + self.log_event("INFO", &format!("Host {} marked as {}", ip, status)).await.ok(); + Ok(()) + } + + pub async fn is_host_nuclei_reviewed(&self, ip: &str) -> Result> { + let reviewed: bool = sqlx::query_scalar( + "SELECT COALESCE(nuclei_reviewed, 0) FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(reviewed) + } + + // Context: Nmap scan result storage for detailed service analysis and reporting + // Operation: Stores nmap output as JSON, prevents duplicates, and merges with existing data + pub async fn insert_nmap_scan( + &self, + ip: &str, + port: &str, + output: &str, + ) -> Result<(), Box> { + let (existing_opt,): (Option,) = sqlx::query_as( + "SELECT nmap_scan FROM targets WHERE run_id = ? AND ip = ?", + ) + .bind(&self.run_id) + .bind(ip) + .fetch_one(&self.pool) + .await?; + + let mut map: JsonMap = existing_opt + .as_deref() + .unwrap_or("{}") + .parse::()? + .as_object() + .cloned() + .unwrap_or_default(); + + // Only insert if port doesn't exist or content is different + if let Some(existing_value) = map.get(port) { + if let Some(existing_str) = existing_value.as_str() { + if existing_str == output { + return Ok(()); // Skip duplicate + } + } + } + + map.insert(port.to_string(), JsonValue::String(output.to_string())); + let merged = JsonValue::Object(map).to_string(); + + sqlx::query("UPDATE targets SET nmap_scan = ? WHERE run_id = ? AND ip = ?") + .bind(merged) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Nmap XML output storage for structured parsing alongside raw text + // Operation: Merges XML into JSON map keyed by scan type, skips if content unchanged + pub async fn insert_nmap_xml( + &self, + ip: &str, + key: &str, + xml: &str, + ) -> Result<(), Box> { + let (existing_opt,): (Option,) = sqlx::query_as( + "SELECT nmap_xml FROM targets WHERE run_id = ? AND ip = ?", + ) + .bind(&self.run_id) + .bind(ip) + .fetch_one(&self.pool) + .await?; + + let mut map: JsonMap = existing_opt + .as_deref() + .unwrap_or("{}") + .parse::()? + .as_object() + .cloned() + .unwrap_or_default(); + + if let Some(existing_value) = map.get(key) { + if existing_value.as_str() == Some(xml) { + return Ok(()); + } + } + + map.insert(key.to_string(), JsonValue::String(xml.to_string())); + let merged = JsonValue::Object(map).to_string(); + + sqlx::query("UPDATE targets SET nmap_xml = ? WHERE run_id = ? AND ip = ?") + .bind(merged) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Nmap XML retrieval for structured tag parsing and HTTP discovery optimisation + // Operation: Returns JSON map string containing XML keyed by scan type + pub async fn get_nmap_xml( + &self, + ip: &str, + ) -> Result, Box> { + let result: Option = sqlx::query_scalar( + "SELECT nmap_xml FROM targets WHERE run_id = ? AND ip = ?", + ) + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + Ok(result) + } + + // Context: Host discovery during phase 2 scanning without triggering workflow automation + // Operation: Inserts new IP into targets table, checks blacklist, but skips hook firing + pub async fn insert_ip_no_hook(&self, ip: &str) -> Result> { + if self.is_blacklisted(ip).await? { + return Ok(false); + } + + let result = sqlx::query( + "INSERT OR IGNORE INTO targets (run_id, ip, ports) VALUES (?, ?, '')" + ) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Ok(false); + } + + tag_info(&format!("New host (phase2): {}", ip)); + let msg = format!("Inserted IP '{}' for Run ID {}", ip, self.run_id); + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&msg); + } + // **no** dispatch_event(Hook::Ip, …) here + + self.log_event("INFO", &msg).await.ok(); + Ok(true) + } + + // Context: Primary host discovery system with full workflow integration and automation + // Operation: Inserts new IP, checks blacklist, logs event, and triggers workflow hooks + pub async fn insert_ip(&self, ip: &str) -> Result> { + if self.is_blacklisted(ip).await? { + return Ok(false); + } + + let result = sqlx::query( + "INSERT OR IGNORE INTO targets (run_id, ip, ports) VALUES (?, ?, '')" + ) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Ok(false); + } + + tag_info(&format!("New host: {}", ip)); + let msg = format!("Inserted IP '{}' for Run ID {}", ip, self.run_id); + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&msg); + } + self.log_event("INFO", &msg).await.ok(); + dispatch_event(Hook::Ip, ip, self); + + Ok(true) + } + + // Context: Polling loop optimization — single bulk fetch replacing N×M per-host queries + // Operation: Returns all hosts as a HashMap for O(1) lookup during workflow tick + pub async fn get_all_host_snapshots(&self) -> Result, Box> { + let rows = sqlx::query( + "SELECT ip, ports, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(action_tags, '') as action_tags, + COALESCE(full_scan_complete, 0) as full_scan_complete, + COALESCE(service_scan_complete, 0) as service_scan_complete, + COALESCE(http_discovery_complete, 0) as http_discovery_complete, + COALESCE(port_services, '{}') as port_services + FROM targets WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + // Bulk fetch HTTP services — ip, port, protocol for has_http_services and foreach fan-out + let http_rows = sqlx::query( + "SELECT ip, port, COALESCE(json_extract(service_info, '$.protocol'), 'http') as protocol + FROM http_services WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut http_ips: HashSet = HashSet::new(); + let mut http_services_by_ip: HashMap> = HashMap::new(); + for row in http_rows { + let ip: String = row.get("ip"); + let port: String = row.get("port"); + let protocol: String = row.get("protocol"); + http_ips.insert(ip.clone()); + http_services_by_ip.entry(ip).or_default().push((port, protocol)); + } + + let mut map = HashMap::with_capacity(rows.len()); + for row in rows { + let ip: String = row.get("ip"); + let snapshot = HostSnapshot { + has_http_services: http_ips.contains(&ip), + http_service_list: http_services_by_ip.get(&ip).cloned().unwrap_or_default(), + ip: ip.clone(), + ports: row.get("ports"), + automated_tags: row.get("automated_tags"), + manual_tags: row.get("manual_tags"), + refined_tags: row.get("refined_tags"), + action_tags: row.get("action_tags"), + full_scan_complete: row.get::("full_scan_complete"), + service_scan_complete: row.get::("service_scan_complete"), + http_discovery_complete: row.get::("http_discovery_complete"), + port_services: row.get("port_services"), + }; + map.insert(ip, snapshot); + } + Ok(map) + } + + // Context: Port discovery data retrieval for service enumeration and analysis + // Operation: Fetches and parses comma-separated port list from database into vector + pub async fn get_open_ports(&self, ip: &str) -> Result, Box> { + let row = sqlx::query("SELECT ports FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + match row { + Some(row) => { + let ports_str: String = row.get("ports"); + let ports = ports_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + Ok(ports) + } + None => Ok(Vec::new()), + } +} + + // Context: Port discovery system with deduplication and workflow notification + // Operation: Adds new ports to existing list, prevents duplicates, triggers hooks for new ports + pub async fn insert_ports( + &self, + ip: &str, + new_ports: &str, +) -> Result> { + let new_ports_vec: Vec<&str> = new_ports + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + + // Get current ports + let current_ports = self.get_open_ports(ip).await?; + let mut existing_ports: HashSet = current_ports.into_iter().collect(); + + let mut any_new = false; + + // Add new ports to the set + for port in new_ports_vec { + if existing_ports.insert(port.to_string()) { + any_new = true; + tag_info(&format!("Discovered open port {} on {}", port, ip)); + let msg = format!("New port for {}: {}", ip, port); + self.log_event("INFO", &msg).await.ok(); + dispatch_event(Hook::Ports, &format!("{}:{}", ip, port), self); + } + } + + if any_new { + // Update the ports column with comma-separated list + let all_ports: Vec = existing_ports.into_iter().collect(); + let ports_str = all_ports.join(","); + + sqlx::query("UPDATE targets SET ports = ? WHERE run_id = ? AND ip = ?") + .bind(&ports_str) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + } + + Ok(any_new) +} + + // Context: Hostname discovery and storage for target identification and reporting + // Operation: Updates hostname field with schema migration, prevents duplicates, triggers hooks + pub async fn insert_hostname( + &self, + ip: &str, + hostname: &str, + ) -> Result> { + let info = sqlx::query("PRAGMA table_info(targets);") + .fetch_all(&self.pool) + .await?; + if !info.iter().any(|r| r.get::("name") == "hostname") { + sqlx::query(r#"ALTER TABLE targets ADD COLUMN hostname TEXT;"#) + .execute(&self.pool) + .await?; + } + + let (existing,): (Option,) = sqlx::query_as( + "SELECT hostname FROM targets WHERE run_id = ? AND ip = ?", + ) + .bind(&self.run_id) + .bind(ip) + .fetch_one(&self.pool) + .await?; + if existing.as_deref() == Some(hostname) { + return Ok(false); + } + + sqlx::query("UPDATE targets SET hostname = ? WHERE run_id = ? AND ip = ?") + .bind(hostname) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + tag_info(&format!("New hostname for {}: {}", ip, hostname)); + let msg = format!("Set hostname '{}' for {}", hostname, ip); + tag_info(&msg); + self.log_event("INFO", &msg).await.ok(); + dispatch_event(Hook::Hostname, hostname, self); + + Ok(true) + } + + // Context: Complete host inventory retrieval for bulk operations and reporting + // Operation: Queries database to return all IP addresses associated with current run + pub async fn fetch_all_ips(&self) -> Result, Box> { + let rows = sqlx::query("SELECT ip FROM targets WHERE run_id = ?") + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(|r| r.get("ip")).collect()) + } + + // Context: Reconnaissance phase completion verification for workflow progression + // Operation: Checks if host completed all scanning phases (full, service, HTTP discovery) + pub async fn is_host_reconnaissance_complete(&self, ip: &str) -> Result> { + let row = sqlx::query("SELECT full_scan_complete, service_scan_complete, http_discovery_complete FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + if let Some(row) = row { + let full_complete: bool = row.get("full_scan_complete"); + let service_complete: bool = row.get("service_scan_complete"); + let http_complete: bool = row.get("http_discovery_complete"); + + Ok(full_complete && service_complete && http_complete) + } else { + Ok(false) // Host not found = not complete + } +} + + // Context: Active Directory domain discovery for environment mapping and targeting + // Operation: Updates AD domain with transaction safety, prevents duplicates, logs discoveries + pub async fn insert_ad_domain(&self, ip: &str, domain: &str) -> Result> { + // Use a transaction to prevent race conditions + let mut tx = self.pool.begin().await?; + + let (existing,): (Option,) = sqlx::query_as( + "SELECT ad_domain FROM targets WHERE run_id = ? AND ip = ?", + ) + .bind(&self.run_id) + .bind(ip) + .fetch_one(&mut *tx) + .await?; + + // Only update if different or empty + if existing.as_deref() != Some(domain) && !domain.is_empty() { + sqlx::query("UPDATE targets SET ad_domain = ? WHERE run_id = ? AND ip = ?") + .bind(domain) + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Only log ONCE when we actually make a change + tag_info(&format!("AD Domain for {}: {}", ip, domain)); + let msg = format!("Set AD domain '{}' for {}", domain, ip); + self.log_event("INFO", &msg).await.ok(); + return Ok(true); + } + + tx.rollback().await?; + Ok(false) + } + + // Context: Dashboard statistics aggregation with optimized bulk operations + // Operation: Calculates comprehensive system statistics using 3 queries instead of 2N+1 + pub async fn get_dashboard_statistics(&self) -> Result> { + // Query 1: total host count + let total_hosts: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM targets WHERE run_id = ?") + .bind(&self.run_id) + .fetch_one(&self.pool) + .await?; + + // Query 2: subnet count + let total_subnets: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM subnets WHERE run_id = ?") + .bind(&self.run_id) + .fetch_one(&self.pool) + .await?; + + // Query 3: all port strings in one pass (replaces N per-host port queries) + let port_rows = sqlx::query( + "SELECT ports FROM targets WHERE run_id = ? AND ports != ''" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut hosts_with_open_ports = 0usize; + let mut total_open_ports = 0usize; + for row in &port_rows { + let ports_str: String = row.get("ports"); + let count = ports_str.split(',').filter(|s| !s.trim().is_empty()).count(); + if count > 0 { + hosts_with_open_ports += 1; + total_open_ports += count; + } + } + + // Query 4: http service counts per IP, grouped (replaces N per-host COUNT queries) + let http_rows = sqlx::query( + "SELECT COUNT(DISTINCT ip) FROM http_services WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_one(&self.pool) + .await?; + let hosts_with_http_services: i64 = http_rows.get(0); + + // Count total screenshots for current project only + let total_screenshots = crate::config::paths::count_screenshots_for_run(&self.db_path, &self.run_id); + + Ok(DashboardStatistics { + total_hosts: total_hosts as usize, + total_subnets: total_subnets as usize, + hosts_with_open_ports, + total_open_ports, + hosts_with_http_services: hosts_with_http_services as usize, + total_screenshots, + }) + } + + // Context: Port-to-service mapping persistence after nmap -sV completes + // Operation: Serializes port→service HashMap to JSON and stores in targets.port_services + pub async fn save_port_services( + &self, + ip: &str, + services: &HashMap, + ) -> Result<(), sqlx::Error> { + let json = serde_json::to_string(services).unwrap_or_else(|_| "{}".to_string()); + sqlx::query("UPDATE targets SET port_services = ? WHERE run_id = ? AND ip = ?") + .bind(json) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Tag-based host search functionality using core parsing logic + // Operation: Searches hosts matching tag criteria and returns processed host summaries + pub async fn search_hosts_by_tag(&self, tag: &str) -> Result, Box> { + let rows = sqlx::query( + "SELECT ip, + COALESCE(hostname, '') as hostname, + COALESCE(ports, '') as ports, + COALESCE(ad_domain, '') as ad_domain, + COALESCE(full_scan_complete, 0) as full_scan_complete, + COALESCE(service_scan_complete, 0) as service_scan_complete, + COALESCE(http_discovery_complete, 0) as http_discovery_complete, + COALESCE(screenshots_complete, 0) as screenshots_complete, + (SELECT COUNT(*) FROM action_outputs WHERE run_id = targets.run_id AND ip = targets.ip) as action_count, + COALESCE(analyzed, 0) as analyzed, + COALESCE(interesting, 0) as interesting + FROM targets + WHERE run_id = ? AND ( + manual_tags LIKE ? OR + refined_tags LIKE ? OR + automated_tags LIKE ? OR + action_tags LIKE ? + ) + ORDER BY ip" + ) + .bind(&self.run_id) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .fetch_all(&self.pool) + .await?; + + let mut hosts = Vec::new(); + for row in rows { + hosts.push(crate::web::handlers::utils::parse_host_summary(row, self).await); + } + + Ok(hosts) + } + + // Context: Bulk nuclei scanning — resolves the set of hosts with HTTP services to target, + // optionally filtered by tag include/exclude lists (OR-match on combined tag columns) and + // by IP/CIDR include/exclude lists (OR-match on containment) + // Operation: Returns IPs of hosts that have at least one http_services row, matching the filters + pub async fn get_hosts_with_http_services_filtered( + &self, + include_tags: &[String], + exclude_tags: &[String], + include_ips: &[ipnetwork::IpNetwork], + exclude_ips: &[ipnetwork::IpNetwork], + ) -> Result, Box> { + let rows = sqlx::query( + "SELECT ip, + COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(action_tags, '') as action_tags + FROM targets + WHERE run_id = ? AND EXISTS ( + SELECT 1 FROM http_services WHERE run_id = targets.run_id AND ip = targets.ip + ) + ORDER BY ip" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut hosts = Vec::new(); + for row in rows { + let ip: String = row.get("ip"); + + let mut combined_tags = HashSet::new(); + for col in ["manual_tags", "refined_tags", "automated_tags", "action_tags"] { + let tags_str: String = row.get(col); + for t in tags_str.split(',').map(str::trim).filter(|s| !s.is_empty()) { + combined_tags.insert(t.to_string()); + } + } + + if !include_tags.is_empty() && !include_tags.iter().any(|t| combined_tags.contains(t)) { + continue; + } + if !exclude_tags.is_empty() && exclude_tags.iter().any(|t| combined_tags.contains(t)) { + continue; + } + + if !include_ips.is_empty() || !exclude_ips.is_empty() { + match ip.parse::() { + Ok(addr) => { + if !include_ips.is_empty() && !include_ips.iter().any(|net| net.contains(addr)) { + continue; + } + if !exclude_ips.is_empty() && exclude_ips.iter().any(|net| net.contains(addr)) { + continue; + } + } + Err(_) => continue, + } + } + + hosts.push(ip); + } + + Ok(hosts) + } + + // Context: Per-host queue re-evaluation — fresh snapshot for a single IP after action completes + // Operation: Same SELECT as get_all_host_snapshots but filtered to one IP + http_services join + pub async fn get_host_snapshot(&self, ip: &str) -> Result, Box> { + let row = sqlx::query( + "SELECT ip, ports, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(action_tags, '') as action_tags, + COALESCE(full_scan_complete, 0) as full_scan_complete, + COALESCE(service_scan_complete, 0) as service_scan_complete, + COALESCE(http_discovery_complete, 0) as http_discovery_complete, + COALESCE(port_services, '{}') as port_services + FROM targets WHERE run_id = ? AND ip = ?" + ) + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + let Some(row) = row else { return Ok(None); }; + + let http_rows = sqlx::query( + "SELECT port, COALESCE(json_extract(service_info, '$.protocol'), 'http') as protocol + FROM http_services WHERE run_id = ? AND ip = ?" + ) + .bind(&self.run_id) + .bind(ip) + .fetch_all(&self.pool) + .await?; + + let mut http_service_list: Vec<(String, String)> = Vec::new(); + for r in http_rows { + let port: String = r.get("port"); + let protocol: String = r.get("protocol"); + http_service_list.push((port, protocol)); + } + let has_http_services = !http_service_list.is_empty(); + + Ok(Some(HostSnapshot { + ip: row.get("ip"), + ports: row.get("ports"), + automated_tags: row.get("automated_tags"), + manual_tags: row.get("manual_tags"), + refined_tags: row.get("refined_tags"), + action_tags: row.get("action_tags"), + full_scan_complete: row.get::("full_scan_complete"), + service_scan_complete: row.get::("service_scan_complete"), + http_discovery_complete: row.get::("http_discovery_complete"), + port_services: row.get("port_services"), + has_http_services, + http_service_list, + })) + } +} + +// Context: Port-to-service map deserialization for use in condition evaluation and placeholder resolution +// Operation: Parses JSON string into HashMap; returns empty map on failure +pub fn parse_port_services(json: &str) -> HashMap { + serde_json::from_str(json).unwrap_or_default() +} \ No newline at end of file diff --git a/src/sql/http.rs b/src/sql/http.rs new file mode 100755 index 0000000..da788d1 --- /dev/null +++ b/src/sql/http.rs @@ -0,0 +1,125 @@ +// src/sql/http.rs +use sqlx::Row; +use std::error::Error; +use crate::utilities::{tag_debug}; +use crate::VERBOSE; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: HTTP service discovery data storage for web application enumeration + // Operation: Performs atomic upsert of HTTP service information with logging and event tracking + pub async fn insert_http_service(&self, ip: &str, port: &str, service_info: &str) -> Result> { + // Use INSERT OR REPLACE for atomic upsert + let result = sqlx::query( + "INSERT OR REPLACE INTO http_services (run_id, ip, port, service_info) VALUES (?, ?, ?, ?)" + ) + .bind(&self.run_id) + .bind(ip) + .bind(port) + .bind(service_info) + .execute(&self.pool) + .await?; + + if result.rows_affected() > 0 { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Stored HTTP service info for {}:{}", ip, port)); + } + + let msg = format!("HTTP service stored for {}:{}", ip, port); + self.log_event("INFO", &msg).await.ok(); + + return Ok(true); + } + + Ok(false) + } + + // Context: HTTP service data retrieval for web application analysis and reporting + // Operation: Fetches all discovered HTTP services for host as port-to-service mapping + pub async fn get_http_services(&self, ip: &str) -> Result, Box> { + let rows = sqlx::query("SELECT port, service_info FROM http_services WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_all(&self.pool) + .await?; + + let mut services = std::collections::HashMap::new(); + for row in rows { + let port: String = row.get("port"); + let service_info: String = row.get("service_info"); + services.insert(port, service_info); + } + + Ok(services) + } + + // Context: HTTP enumeration phase tracking to prevent duplicate scanning efforts + // Operation: Records HTTP discovery start, returns false if already started + pub async fn mark_http_discovery_started(&self, ip: &str) -> Result> { + let already_started: bool = sqlx::query_scalar("SELECT http_discovery_started FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + if already_started { + return Ok(false); + } + + sqlx::query("UPDATE targets SET http_discovery_started = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("HTTP discovery marked started for {}", ip)); + } + + Ok(true) + } + + // Context: HTTP phase completion verification for workflow progression control + // Operation: Queries database to check if HTTP discovery phase has been completed + pub async fn is_http_discovery_complete(&self, ip: &str) -> Result> { + let complete: bool = sqlx::query_scalar("SELECT http_discovery_complete FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + Ok(complete) + } + + // Context: HTTP service existence verification for targeting and analysis decisions + // Operation: Determines if host has discovered HTTP services by checking service count + pub async fn has_http_services(&self, ip: &str) -> Result> { + let services = self.get_http_services(ip).await?; + Ok(!services.is_empty()) + } + + // Context: HTTP enumeration completion with automated tagging and workflow integration + // Operation: Marks discovery complete, triggers HTTP auto-tagging, and fires workflow hooks + pub async fn mark_http_discovery_complete(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET http_discovery_complete = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("HTTP discovery marked complete for {}", ip)); + } + + // Trigger comprehensive HTTP tagging once discovery complete + if let Err(e) = crate::tag::auto_tag_http(self, ip).await { + tag_debug(&format!("HTTP auto-tagging failed for {}: {}", ip, e)); + } + + crate::workflow::dispatch_event(crate::workflow::Hook::HttpAutoTagComplete, ip, self); + + Ok(()) + } +} \ No newline at end of file diff --git a/src/sql/logging.rs b/src/sql/logging.rs new file mode 100755 index 0000000..240e1f7 --- /dev/null +++ b/src/sql/logging.rs @@ -0,0 +1,132 @@ +// src/sql/logging.rs +use chrono::Utc; +use std::error::Error; +use std::sync::Mutex; +use sqlx::{Pool, Row, Sqlite}; +use serde::Serialize; +use once_cell::sync::Lazy; +use super::context::ProjectContext; + +#[derive(Serialize)] +pub struct LogEntry { + pub timestamp: String, + pub level: String, + pub message: String, +} + +// Internal buffer entry — includes run_id so flush works across any pool +struct BufferedEntry { + run_id: String, + timestamp: String, + level: String, + message: String, +} + +// Global log buffer — log_event() pushes here, flush_log_buffer() drains every 3s +static LOG_BUFFER: Lazy>> = Lazy::new(|| Mutex::new(Vec::new())); + +/// Drain the log buffer and write all entries in a single transaction. +/// Called by the background flush task started in main.rs. +pub async fn flush_log_buffer(pool: &Pool) { + let entries = { + let mut buf = LOG_BUFFER.lock().unwrap(); + buf.drain(..).collect::>() + }; + if entries.is_empty() { + return; + } + let mut tx = match pool.begin().await { + Ok(tx) => tx, + Err(_) => return, + }; + for entry in entries { + sqlx::query( + "INSERT INTO logs (run_id, timestamp, level, message) VALUES (?, ?, ?, ?)" + ) + .bind(&entry.run_id) + .bind(&entry.timestamp) + .bind(&entry.level) + .bind(&entry.message) + .execute(&mut *tx) + .await + .ok(); + } + tx.commit().await.ok(); +} + +impl ProjectContext { + // Context: Structured logging system for audit trails and debugging information + // Operation: Pushes timestamped log entry to in-memory buffer (flushed every 3s) + pub async fn insert_log(&self, level: &str, message: &str) -> Result<(), Box> { + let timestamp = Utc::now().to_rfc3339(); + LOG_BUFFER.lock().unwrap().push(BufferedEntry { + run_id: self.run_id.clone(), + timestamp, + level: level.to_string(), + message: message.to_string(), + }); + Ok(()) + } + + // Context: Event logging system for tracking significant system operations and changes + // Operation: Pushes timestamped event to in-memory buffer (flushed every 3s in batch) + pub async fn log_event( + &self, + level: &str, + message: &str, + ) -> Result<(), Box> { + let ts = Utc::now().to_rfc3339(); + LOG_BUFFER.lock().unwrap().push(BufferedEntry { + run_id: self.run_id.clone(), + timestamp: ts, + level: level.to_string(), + message: message.to_string(), + }); + Ok(()) + } + + // Context: Log retrieval system with flexible filtering for web interfaces and APIs + // Operation: Queries logs with optional level and timestamp filters, returns structured entries + pub async fn get_logs_filtered( + &self, + level: Option<&str>, + since: Option<&str>, + ) -> Result, Box> { + // Flush buffer before reading so the page shows recent entries + flush_log_buffer(&self.pool).await; + + let mut query = "SELECT timestamp, level, message FROM logs WHERE run_id = ?".to_string(); + let mut bind_params: Vec = vec![self.run_id.clone()]; + + if let Some(level) = level { + if !level.is_empty() { + query.push_str(" AND level = ?"); + bind_params.push(level.to_string()); + } + } + + if let Some(since) = since { + if !since.is_empty() { + query.push_str(" AND timestamp > ?"); + bind_params.push(since.to_string()); + } + } + + query.push_str(" ORDER BY timestamp DESC LIMIT 1000"); + + let mut sqlx_query = sqlx::query(&query); + for param in &bind_params { + sqlx_query = sqlx_query.bind(param); + } + + let rows = sqlx_query.fetch_all(&self.pool).await?; + + let log_entries: Vec = rows.into_iter().map(|row| LogEntry { + timestamp: row.get("timestamp"), + level: row.get("level"), + message: row.get("message"), + }).collect(); + + Ok(log_entries) + } +} diff --git a/src/sql/mod.rs b/src/sql/mod.rs new file mode 100755 index 0000000..d7d0fd6 --- /dev/null +++ b/src/sql/mod.rs @@ -0,0 +1,33 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// src/sql/mod.rs +pub mod context; +pub mod database; +pub mod hosts; +pub mod subnets; +pub mod scanning; +pub mod http; +pub mod actions; +pub mod nuclei; +pub mod tagging; +pub mod users; +pub mod workflow; +pub mod networks; +pub mod logging; +pub mod task_history; + +// Re-export the main types and functions for convenience +pub use context::ProjectContext; +pub use logging::LogEntry; \ No newline at end of file diff --git a/src/sql/networks.rs b/src/sql/networks.rs new file mode 100755 index 0000000..b159823 --- /dev/null +++ b/src/sql/networks.rs @@ -0,0 +1,133 @@ +// src/sql/networks.rs +use sqlx::Row; +use ipnetwork::IpNetwork; +use std::error::Error; +use std::net::IpAddr; +use crate::utilities::tag_info; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: Network interface IP extraction for scanning source address determination + // Operation: Retrieves IP address from first stored interface by parsing CIDR notation + pub async fn get_selected_interface_ip(&self) -> Result> { + let (subnet,): (String,) = sqlx::query_as( + "SELECT subnet FROM interfaces WHERE run_id = ? LIMIT 1", + ) + .bind(&self.run_id) + .fetch_one(&self.pool) + .await?; + + // Extract IP from CIDR notation (e.g., "192.168.1.10/24" -> "192.168.1.10") + let ip = subnet.split('/').next().unwrap_or("127.0.0.1"); + Ok(ip.to_string()) + } + + // Context: Network interface discovery and storage for scanning configuration + // Operation: Records interface-subnet mapping with duplicate prevention and logging + pub async fn insert_interface( + &self, + interface: &str, + subnet: &str, + ) -> Result<(), Box> { + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM interfaces WHERE run_id = ? AND interface = ?", + ) + .bind(&self.run_id) + .bind(interface) + .fetch_one(&self.pool) + .await?; + if count == 0 { + sqlx::query( + "INSERT INTO interfaces (run_id, interface, subnet) VALUES (?1, ?2, ?3)", + ) + .bind(&self.run_id) + .bind(interface) + .bind(subnet) + .execute(&self.pool) + .await?; + + tag_info(&format!("Stored interface {} -> {}", interface, subnet)); + } + Ok(()) + } + + // Context: Network interface retrieval for scanning tool configuration + // Operation: Returns first stored interface name for use in network operations + pub async fn get_selected_interface(&self) -> Result> { + let (iface,): (String,) = sqlx::query_as( + "SELECT interface FROM interfaces WHERE run_id = ? LIMIT 1", + ) + .bind(&self.run_id) + .fetch_one(&self.pool) + .await?; + Ok(iface) + } + + // Context: Target exclusion system for preventing scanning of restricted addresses + // Operation: Normalizes and stores IPv4 addresses/CIDRs in blacklist with deduplication + pub async fn insert_blacklist(&self, entry: &str) -> Result<(), Box> { + // Normalize: if it lacks '/', assume single‐host CIDR /32 + let normalized = if entry.contains('/') { + let net: IpNetwork = entry.parse()?; + match net { + IpNetwork::V4(v4) => format!("{}/{}", v4.network(), v4.prefix()), + _ => return Err("Only IPv4 is supported".into()), + } + } else { + // treat "192.168.1.5" as "192.168.1.5/32" + format!("{}/32", entry) + }; + + sqlx::query("INSERT OR IGNORE INTO blacklist (run_id, cidr) VALUES (?1, ?2)") + .bind(&self.run_id) + .bind(&normalized) + .execute(&self.pool) + .await?; + + tag_info(&format!("Blacklisted {}", normalized)); + Ok(()) + } + + // Context: Target validation system for enforcing scanning restrictions and compliance + // Operation: Checks if IP address falls within any blacklisted CIDR ranges + pub async fn is_blacklisted(&self, ip: &str) -> Result> { + // Parse into IpAddr and early-return for non-IPv4 + let addr = match ip.parse::()? { + IpAddr::V4(v4) => v4, + _ => return Ok(false), + }; + + // Fetch all stored CIDRs + let rows = sqlx::query("SELECT cidr FROM blacklist WHERE run_id = ?") + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + for row in rows { + let cidr_str: String = row.get("cidr"); + if let Ok(IpNetwork::V4(net)) = cidr_str.parse::() { + if net.contains(addr) { + return Ok(true); + } + } + } + Ok(false) + } + + // Context: Masscan security integration for blacklist compliance + // Operation: Returns all blacklisted CIDRs formatted for masscan --exclude argument + pub async fn get_masscan_excludes(&self) -> Result, Box> { + let blacklisted_cidrs = sqlx::query("SELECT cidr FROM blacklist WHERE run_id = ?") + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut excludes = Vec::new(); + for row in blacklisted_cidrs { + let cidr: String = row.get("cidr"); + excludes.push(cidr); + } + + Ok(excludes) + } +} \ No newline at end of file diff --git a/src/sql/nuclei.rs b/src/sql/nuclei.rs new file mode 100644 index 0000000..7b417b4 --- /dev/null +++ b/src/sql/nuclei.rs @@ -0,0 +1,232 @@ +// src/sql/nuclei.rs +use std::error::Error; +use std::collections::BTreeMap; +use serde::{Deserialize, Serialize}; +use sqlx::Row; +use super::context::ProjectContext; + +#[derive(Serialize, Deserialize, Clone)] +pub struct NucleiFinding { + pub template_id: String, + pub name: String, + pub severity: String, + pub matched_at: String, + // which named matcher fired (e.g., "ms-httpapi", "nginx") — empty when absent + #[serde(default)] + pub matcher_name: String, + // values pulled out by extractor-based templates (version strings, etc.) — empty when absent + #[serde(default)] + pub extracted_results: Vec, + pub host: String, + pub description: String, + pub references: Vec, + // template classification tags from nuclei's info.tags — empty when absent + #[serde(default)] + pub nuclei_tags: Vec, +} + +/// Per-host counts of nuclei findings grouped by severity, for dashboard summary cards. +#[derive(Serialize, Clone, Default)] +pub struct NucleiSeverityCounts { + pub critical: usize, + pub high: usize, + pub medium: usize, + pub low: usize, + pub info: usize, + pub unknown: usize, +} + +/// One entry per unique template name — aggregates all matched_at URLs for that template. +#[derive(Serialize)] +pub struct NucleiGroupedFinding { + pub template_id: String, + pub name: String, + pub severity: String, + pub description: String, + pub references: Vec, + pub matched_urls: Vec, +} + +#[derive(Serialize)] +pub struct NucleiPortResult { + pub port: String, + pub url: String, + pub findings: Vec, +} + +#[derive(Serialize)] +struct NucleiOutputJson<'a> { + scan_target: &'a str, + timestamp: String, + findings: &'a [NucleiFinding], +} + +fn build_match_display(matched_at: &str, matcher_name: &str, extracted: &[String]) -> String { + let prefix = if !matcher_name.is_empty() { format!("[{}] ", matcher_name) } else { String::new() }; + let suffix = if !extracted.is_empty() { format!(" → {}", extracted.join(", ")) } else { String::new() }; + format!("{}{}{}", prefix, matched_at, suffix) +} + +impl ProjectContext { + pub async fn insert_nuclei_output( + &self, + ip: &str, + port: &str, + findings: Vec, + scan_target: &str, + ) -> Result<(), Box> { + let output = NucleiOutputJson { + scan_target, + timestamp: chrono::Utc::now().to_rfc3339(), + findings: &findings, + }; + let json = serde_json::to_string(&output)?; + let action_name = format!("nuclei_{}", port); + + sqlx::query( + "INSERT OR REPLACE INTO action_outputs (run_id, ip, action_name, output_json, tagged) + VALUES (?, ?, ?, ?, FALSE)" + ) + .bind(&self.run_id) + .bind(ip) + .bind(&action_name) + .bind(&json) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn get_nuclei_results_for_host( + &self, + ip: &str, + ) -> Result, Box> { + let rows = sqlx::query( + "SELECT action_name, output_json FROM action_outputs + WHERE run_id = ? AND ip = ? AND action_name LIKE 'nuclei_%' + ORDER BY action_name" + ) + .bind(&self.run_id) + .bind(ip) + .fetch_all(&self.pool) + .await?; + + let mut results = Vec::new(); + for row in rows { + let action_name: String = row.get("action_name"); + let output_json: String = row.get("output_json"); + let port = action_name + .strip_prefix("nuclei_") + .unwrap_or(&action_name) + .to_string(); + + let raw_findings: Vec = if let Ok(v) = + serde_json::from_str::(&output_json) + { + v.get("findings") + .and_then(|f| serde_json::from_value::>(f.clone()).ok()) + .unwrap_or_default() + } else { + Vec::new() + }; + + // Group by name (BTreeMap keeps alphabetical order) + let mut groups: BTreeMap = BTreeMap::new(); + for f in raw_findings { + let entry = groups.entry(f.name.clone()).or_insert_with(|| NucleiGroupedFinding { + template_id: f.template_id.clone(), + name: f.name.clone(), + severity: f.severity.clone(), + description: f.description.clone(), + references: f.references.clone(), + matched_urls: Vec::new(), + }); + let display = build_match_display(&f.matched_at, &f.matcher_name, &f.extracted_results); + if !display.is_empty() && !entry.matched_urls.contains(&display) { + entry.matched_urls.push(display); + } + } + + results.push(NucleiPortResult { + port, + url: String::new(), + findings: groups.into_values().collect(), + }); + } + Ok(results) + } + + /// Counts nuclei findings for a host by severity, across all its nuclei_% action outputs. + pub async fn get_nuclei_severity_counts( + &self, + ip: &str, + ) -> Result> { + let rows = sqlx::query( + "SELECT output_json FROM action_outputs + WHERE run_id = ? AND ip = ? AND action_name LIKE 'nuclei_%'" + ) + .bind(&self.run_id) + .bind(ip) + .fetch_all(&self.pool) + .await?; + + let mut counts = NucleiSeverityCounts::default(); + for row in rows { + let output_json: String = row.get("output_json"); + let findings: Vec = serde_json::from_str::(&output_json) + .ok() + .and_then(|v| v.get("findings") + .and_then(|f| serde_json::from_value::>(f.clone()).ok())) + .unwrap_or_default(); + + for f in findings { + match f.severity.to_lowercase().as_str() { + "critical" => counts.critical += 1, + "high" => counts.high += 1, + "medium" => counts.medium += 1, + "low" => counts.low += 1, + "info" => counts.info += 1, + _ => counts.unknown += 1, + } + } + } + Ok(counts) + } + + pub async fn get_hosts_with_nuclei_findings( + &self, + ) -> Result, Box> { + let rows = sqlx::query( + "SELECT t.ip, + COALESCE(t.hostname, '') as hostname, + COALESCE(t.ports, '') as ports, + COALESCE(t.ad_domain, '') as ad_domain, + COALESCE(t.full_scan_complete, 0) as full_scan_complete, + COALESCE(t.service_scan_complete, 0) as service_scan_complete, + COALESCE(t.http_discovery_complete, 0) as http_discovery_complete, + COALESCE(t.screenshots_complete, 0) as screenshots_complete, + (SELECT COUNT(*) FROM action_outputs + WHERE run_id = t.run_id AND ip = t.ip) as action_count, + COALESCE(t.analyzed, 0) as analyzed + FROM targets t + WHERE t.run_id = ? + AND EXISTS ( + SELECT 1 FROM action_outputs + WHERE run_id = t.run_id AND ip = t.ip AND action_name LIKE 'nuclei_%' + ) + AND NOT COALESCE(t.nuclei_reviewed, 0) + ORDER BY t.ip" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut hosts = Vec::new(); + for row in rows { + let mut host = crate::web::handlers::parse_host_summary(row, self).await; + host.nuclei_severity_counts = self.get_nuclei_severity_counts(&host.ip).await.unwrap_or_default(); + hosts.push(host); + } + Ok(hosts) + } +} diff --git a/src/sql/scanning.rs b/src/sql/scanning.rs new file mode 100755 index 0000000..b12c7da --- /dev/null +++ b/src/sql/scanning.rs @@ -0,0 +1,215 @@ +// src/sql/scanning.rs +use std::error::Error; +use crate::utilities::tag_debug; +use crate::VERBOSE; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: Service enumeration phase tracking to manage scan workflow progression + // Operation: Marks service scan as started for specific host with debug logging + pub async fn mark_service_scan_started(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET service_scan_started = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Service scan marked started for {}", ip)); + } + + Ok(()) + } + + // Context: Full port scan phase tracking to prevent duplicate scanning efforts + // Operation: Records full scan start, returns false if already started + pub async fn mark_full_scan_started(&self, ip: &str) -> Result> { + // Check if already started + let already_started: bool = sqlx::query_scalar("SELECT full_scan_started FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + if already_started { + return Ok(false); // Already started + } + + sqlx::query("UPDATE targets SET full_scan_started = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(true) + } + + // Context: Full port scan completion tracking for workflow phase transitions + // Operation: Updates database to mark full port scan as complete for host + pub async fn mark_full_scan_complete(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET full_scan_complete = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Scan phase verification for workflow decision making and dependencies + // Operation: Queries database to check if full port scan has been completed + pub async fn is_full_scan_complete(&self, ip: &str) -> Result> { + let complete: bool = sqlx::query_scalar("SELECT full_scan_complete FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + Ok(complete) + } + + // Context: Screenshot capture phase tracking to prevent duplicate visual enumeration + // Operation: Records screenshot start, returns false if already started + pub async fn mark_screenshots_started(&self, ip: &str) -> Result> { + let already_started: bool = sqlx::query_scalar("SELECT screenshots_started FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + if already_started { + return Ok(false); + } + + sqlx::query("UPDATE targets SET screenshots_started = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(true) + } + + // Context: Screenshot phase completion tracking for reconnaissance workflow management + // Operation: Updates database to mark screenshot capture as complete for host + pub async fn mark_screenshots_complete(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET screenshots_complete = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: YAML workflow completion gate — host ready for analysis only after all actions exhausted + // Operation: Sets user_workflow_complete flag; called from re_evaluate_single_host and global loop + pub async fn mark_workflow_complete(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET user_workflow_complete = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Service scan completion verification for workflow progression control + // Operation: Queries database to check if service enumeration has been completed + pub async fn is_service_scan_complete(&self, ip: &str) -> Result> { + let complete: bool = sqlx::query_scalar("SELECT service_scan_complete FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + Ok(complete) + } + + // Context: Project resume and cleanup system for ensuring clean state transitions + // Operation: Resets all scanning phase flags while preserving analysis status + pub async fn reset_all_started_flags(&self) -> Result<(), Box> { + // First ensure all columns exist before trying to reset them + let columns_to_ensure = vec![ + "full_scan_started BOOLEAN DEFAULT FALSE", + "full_scan_complete BOOLEAN DEFAULT FALSE", + "service_scan_started BOOLEAN DEFAULT FALSE", + "service_scan_complete BOOLEAN DEFAULT FALSE", + "http_discovery_started BOOLEAN DEFAULT FALSE", + "http_discovery_complete BOOLEAN DEFAULT FALSE", + "web_enumeration_started BOOLEAN DEFAULT FALSE", + "web_enumeration_complete BOOLEAN DEFAULT FALSE", + "screenshots_started BOOLEAN DEFAULT FALSE", + "screenshots_complete BOOLEAN DEFAULT FALSE" + ]; + + for column_def in columns_to_ensure { + let column_name = column_def.split_whitespace().next().unwrap(); + + // Check if column exists + let column_exists = sqlx::query("SELECT name FROM pragma_table_info('targets') WHERE name = ?") + .bind(column_name) + .fetch_optional(&self.pool) + .await? + .is_some(); + + if !column_exists { + let alter_sql = format!("ALTER TABLE targets ADD COLUMN {}", column_def); + if let Err(e) = sqlx::query(&alter_sql).execute(&self.pool).await { + tag_debug(&format!("Warning: Failed to add column {}: {}", column_name, e)); + } + } + } + + // Now safely reset all flags + let reset_queries = vec![ + "UPDATE targets SET full_scan_started = FALSE WHERE run_id = ?", + "UPDATE targets SET service_scan_started = FALSE WHERE run_id = ?", + "UPDATE targets SET http_discovery_started = FALSE WHERE run_id = ?", + "UPDATE targets SET screenshots_started = FALSE WHERE run_id = ?", + ]; + + for query in reset_queries { + if let Err(e) = sqlx::query(query).bind(&self.run_id).execute(&self.pool).await { + tag_debug(&format!("Warning: Failed to reset flag: {}", e)); + // Continue with other resets even if one fails + } + } + + // Reset action flags (should always exist) + if let Err(e) = sqlx::query("UPDATE action_status SET started = FALSE WHERE run_id = ?") + .bind(&self.run_id) + .execute(&self.pool) + .await { + tag_debug(&format!("Warning: Failed to reset action flags: {}", e)); + } + + // DON'T reset analyzed status - it should persist across resumes + + // Reset subnet flags (these should always exist) + if let Err(e) = sqlx::query("UPDATE subnets SET explorer_scanned = 0 WHERE run_id = ?") + .bind(&self.run_id) + .execute(&self.pool) + .await { + tag_debug(&format!("Warning: Failed to reset subnet flags: {}", e)); + } + + // Reset completed for clean resume + Ok(()) + } + + // Context: Service enumeration completion tracking for workflow phase transitions + // Operation: Updates database to mark service scan as complete with debug logging + pub async fn mark_service_scan_complete(&self, ip: &str) -> Result<(), Box> { + sqlx::query("UPDATE targets SET service_scan_complete = TRUE WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Service scan marked complete for {}", ip)); + } + + Ok(()) + } +} \ No newline at end of file diff --git a/src/sql/subnets.rs b/src/sql/subnets.rs new file mode 100755 index 0000000..e76f663 --- /dev/null +++ b/src/sql/subnets.rs @@ -0,0 +1,328 @@ +// src/sql/subnets.rs +use sqlx::Row; +use ipnetwork::IpNetwork; +use std::error::Error; +use crate::workflow::{dispatch_event, Hook}; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: Subnet scanning phase completion tracking for workflow progression management + // Operation: Updates phase completion status with dynamic schema migration and logging + pub async fn mark_subnet_phase_complete(&self, subnet: &str, phase: &str) -> Result<(), Box> { + let column = match phase { + "subnet_phase1" => "phase1_complete", + "subnet_phase2" => "phase2_complete", + _ => return Err(format!("Unknown subnet phase: {}", phase).into()), + }; + + // Add columns if they don't exist + let info = sqlx::query("PRAGMA table_info(subnets);") + .fetch_all(&self.pool) + .await?; + if !info.iter().any(|r| r.get::("name") == column) { + let alter_sql = format!("ALTER TABLE subnets ADD COLUMN {} BOOLEAN DEFAULT FALSE", column); + sqlx::query(&alter_sql).execute(&self.pool).await?; + } + + let update_sql = format!("UPDATE subnets SET {} = TRUE WHERE run_id = ? AND target = ?", column); + sqlx::query(&update_sql) + .bind(&self.run_id) + .bind(subnet) + .execute(&self.pool) + .await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Marked {} complete for subnet {}", phase, subnet)); + } + + Ok(()) + } + + // Context: Subnet phase verification for workflow decision making and dependency checking + // Operation: Queries database to check if specific subnet phase has been completed + pub async fn is_subnet_phase_complete(&self, subnet: &str, phase: &str) -> Result> { + let column = match phase { + "subnet_phase1" => "phase1_complete", + "subnet_phase2" => "phase2_complete", + _ => return Ok(false), + }; + + // Check if column exists + let info = sqlx::query("PRAGMA table_info(subnets);") + .fetch_all(&self.pool) + .await?; + if !info.iter().any(|r| r.get::("name") == column) { + return Ok(false); // Column doesn't exist, so not complete + } + + let query_sql = format!("SELECT {} FROM subnets WHERE run_id = ? AND target = ?", column); + let complete: bool = sqlx::query_scalar(&query_sql) + .bind(&self.run_id) + .bind(subnet) + .fetch_optional(&self.pool) + .await? + .unwrap_or(false); + + Ok(complete) + } + + // Context: Masscan scanning progress tracking to prevent duplicate subnet enumeration + // Operation: Marks subnet as having completed masscan discovery phase + pub async fn mark_masscan_scanned(&self, subnet: &str) -> Result<(), Box> { + sqlx::query( + "UPDATE subnets SET masscan_scanned = 1 + WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(subnet) + .execute(&self.pool) + .await?; + Ok(()) + } + + + // Context: Host exploration phase tracking for comprehensive subnet enumeration + // Operation: Marks subnet as having completed host exploration scanning phase + pub async fn mark_explorer_scanned(&self, subnet: &str) -> Result<(), Box> { + sqlx::query( + "UPDATE subnets SET explorer_scanned = 1 + WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(subnet) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Subnet insertion for masscan-managed paths that own their own Phase 1 → Phase 2 chain + // Operation: Inserts subnet without firing Hook::Subnet — prevents Phase 1 queuing before masscan runs + pub async fn insert_subnet_no_hook( + &self, + target: &str, + ) -> Result> { + let new_net: IpNetwork = target.parse()?; + if new_net.prefix() == 32 { + return Ok(false); + } + let new_cidr = format!("{}/{}", new_net.network(), new_net.prefix()); + + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM subnets WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(&new_cidr) + .fetch_one(&self.pool) + .await?; + if count > 0 { + return Ok(false); + } + + sqlx::query("INSERT INTO subnets (run_id, target) VALUES (?1, ?2)") + .bind(&self.run_id) + .bind(&new_cidr) + .execute(&self.pool) + .await?; + + tag_info(&format!("New subnet (masscan): {}", new_cidr)); + Ok(true) + } + + // Context: Forced subnet insertion for masscan-discovered networks without consolidation + // Operation: Adds valid IPv4 networks to database, skipping duplicate checks and consolidation rules + pub async fn insert_subnet_force( + &self, + target: &str, + ) -> Result> { + // Normalize & ensure it's a valid IPv4 network + let new_net: IpNetwork = target.parse()?; + // Don't treat a single host as a "subnet" + if new_net.prefix() == 32 { + return Ok(false); + } + let new_cidr = format!("{}/{}", new_net.network(), new_net.prefix()); + + // Check for exact duplicate in the DB + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM subnets WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(&new_cidr) + .fetch_one(&self.pool) + .await?; + if count > 0 { + // Already present + return Ok(false); + } + + // Not found—insert it + sqlx::query("INSERT INTO subnets (run_id, target) VALUES (?1, ?2)") + .bind(&self.run_id) + .bind(&new_cidr) + .execute(&self.pool) + .await?; + + tag_info(&format!("[MASSCAN] New Subnet with alive hosts discovered: {}", new_cidr)); + dispatch_event(Hook::Subnet, &new_cidr, self); + Ok(true) + } + + // Context: Intelligent subnet discovery system with consolidation rules and deduplication + // Operation: Inserts subnet with /24 boundary rules, consolidation logic, and workflow hooks + pub async fn insert_subnet( + &self, + target: &str, +) -> Result> { + let new_net: IpNetwork = target.parse()?; + if new_net.prefix() == 32 { + return Ok(false); + } + let new_cidr = format!("{}/{}", new_net.network(), new_net.prefix()); + + // Check for exact duplicate + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM subnets WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(&new_cidr) + .fetch_one(&self.pool) + .await?; + if count > 0 { + return Ok(false); + } + + // RULE 2: /24 always bypasses consolidation + if new_net.prefix() == 24 { + sqlx::query("INSERT INTO subnets (run_id, target) VALUES (?1, ?2)") + .bind(&self.run_id) + .bind(&new_cidr) + .execute(&self.pool) + .await?; + + tag_info(&format!("New subnet: {}", new_cidr)); + dispatch_event(Hook::Subnet, &new_cidr, self); + return Ok(true); + } + + // For non-/24 subnets, apply consolidation rules + let rows = sqlx::query("SELECT target FROM subnets WHERE run_id = ?") + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut to_delete = Vec::new(); + for row in rows { + let existing_str: String = row.get("target"); + let existing_net: IpNetwork = existing_str.parse()?; + + // RULE 1: Only consolidate smaller subnets UP TO /24 + if new_net.prefix() < 24 && existing_net.prefix() < 24 { + // Both broader than /24 - apply normal consolidation + if existing_net.prefix() <= new_net.prefix() + && existing_net.contains(new_net.network()) + { + return Ok(false); // Already covered + } + if new_net.prefix() <= existing_net.prefix() + && new_net.contains(existing_net.network()) + { + to_delete.push(existing_str); // Replace with broader + } + } else if new_net.prefix() >= 24 && existing_net.prefix() >= 24 { + // Both /24 or smaller - consolidate UP TO /24 only + if existing_net.prefix() <= new_net.prefix() + && existing_net.contains(new_net.network()) + { + return Ok(false); // Already covered + } + if new_net.prefix() <= existing_net.prefix() + && new_net.contains(existing_net.network()) + && new_net.prefix() >= 24 // Don't consolidate beyond /24 + { + to_delete.push(existing_str); // Replace with consolidated /24 + } + } + // RULE 3: Don't consolidate across /24 boundary + // If one is >/24 and other is Result)>, Box> { + let rows = sqlx::query("SELECT target, phase1_hosts FROM subnets WHERE run_id = ?") + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut out = Vec::new(); + for row in rows { + let target: String = row.get("target"); + let raw: Option = row.get("phase1_hosts"); + let list = raw + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + out.push((target, list)); + } + Ok(out) + } + + // Context: Phase-1 host discovery result storage for tracking subnet enumeration progress + // Operation: Updates comma-separated host list for specific subnet in database + pub async fn update_phase1_hosts( + &self, + subnet: &str, + hosts: &[String], + ) -> Result<(), Box> { + let joined = hosts.join(","); + sqlx::query( + "UPDATE subnets SET phase1_hosts = ? WHERE run_id = ? AND target = ?", + ) + .bind(&joined) + .bind(&self.run_id) + .bind(subnet) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Subnet rescan flag verification for handling incomplete or failed scans + // Operation: Queries database to check if subnet has been marked for rescanning + pub async fn is_subnet_marked_rescan(&self, subnet: &str) -> Result> { + let (flag,): (i64,) = sqlx::query_as( + "SELECT rescan FROM subnets WHERE run_id = ? AND target = ?", + ) + .bind(&self.run_id) + .bind(subnet) + .fetch_one(&self.pool) + .await?; + Ok(flag != 0) + } +} \ No newline at end of file diff --git a/src/sql/tagging.rs b/src/sql/tagging.rs new file mode 100755 index 0000000..8dbdafb --- /dev/null +++ b/src/sql/tagging.rs @@ -0,0 +1,295 @@ +// src/sql/tagging.rs +use std::error::Error; +use std::collections::{HashSet, HashMap}; +use serde::Serialize; +use sqlx::Row; +use super::context::ProjectContext; + +#[derive(Serialize)] +pub struct TagInfo { + pub name: String, + pub count: usize, +} + +#[derive(Serialize)] +pub struct AllTagsWithCounts { + pub manual_tags: Vec, + pub refined_tags: Vec, + pub automated_tags: Vec, + pub action_tags: Vec, +} + +impl ProjectContext { + // Context: Automated tag retrieval for AI-generated host classifications and insights + // Operation: Fetches comma-separated automated tags from database and returns as set + pub async fn get_automated_tags(&self, ip: &str) -> Result, Box> { + let result: Option = sqlx::query_scalar("SELECT automated_tags FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + match result { + Some(tags_str) => { + let tags = tags_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + Ok(tags) + } + None => Ok(HashSet::new()), + } + } + + // Context: Automated tag storage for AI-generated host classifications and analysis + // Operation: Converts tag set to sorted comma-separated string and updates database + pub async fn set_automated_tags(&self, ip: &str, tags: HashSet) -> Result<(), Box> { + let tags_str = if tags.is_empty() { + String::new() + } else { + let mut tag_vec: Vec = tags.into_iter().collect(); + tag_vec.sort(); + tag_vec.join(",") + }; + + sqlx::query("UPDATE targets SET automated_tags = ? WHERE run_id = ? AND ip = ?") + .bind(&tags_str) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Manual tag retrieval for user-defined host classifications and notes + // Operation: Fetches comma-separated manual tags from database and returns as set + pub async fn get_manual_tags(&self, ip: &str) -> Result, Box> { + let result: Option = sqlx::query_scalar("SELECT manual_tags FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + match result { + Some(tags_str) => { + let tags = tags_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + Ok(tags) + } + None => Ok(HashSet::new()), + } + } + + // Context: Manual tag storage for user-defined host classifications and annotations + // Operation: Converts tag set to sorted comma-separated string and updates database + pub async fn set_manual_tags(&self, ip: &str, tags: HashSet) -> Result<(), Box> { + let tags_str = if tags.is_empty() { + String::new() + } else { + let mut tag_vec: Vec = tags.into_iter().collect(); + tag_vec.sort(); + tag_vec.join(",") + }; + + sqlx::query("UPDATE targets SET manual_tags = ? WHERE run_id = ? AND ip = ?") + .bind(&tags_str) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn get_action_tags(&self, ip: &str) -> Result, Box> { + let result: Option = sqlx::query_scalar("SELECT action_tags FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + match result { + Some(tags_str) => { + let tags = tags_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + Ok(tags) + } + None => Ok(HashSet::new()), + } + } + + pub async fn set_action_tags(&self, ip: &str, tags: HashSet) -> Result<(), Box> { + let tags_str = if tags.is_empty() { + String::new() + } else { + let mut tag_vec: Vec = tags.into_iter().collect(); + tag_vec.sort(); + tag_vec.join(",") + }; + + sqlx::query("UPDATE targets SET action_tags = ? WHERE run_id = ? AND ip = ?") + .bind(&tags_str) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Refined tag retrieval for processed and validated host classifications + // Operation: Fetches comma-separated refined tags from database and returns as set + pub async fn get_refined_tags(&self, ip: &str) -> Result, Box> { + let result: Option = sqlx::query_scalar("SELECT refined_tags FROM targets WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .fetch_optional(&self.pool) + .await?; + + match result { + Some(tags_str) => { + let tags = tags_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + Ok(tags) + } + None => Ok(HashSet::new()), + } + } + + // Context: Refined tag storage for processed and validated host classifications + // Operation: Converts tag set to sorted comma-separated string and updates database + pub async fn set_refined_tags(&self, ip: &str, tags: HashSet) -> Result<(), Box> { + let tags_str = if tags.is_empty() { + String::new() + } else { + let mut tag_vec: Vec = tags.into_iter().collect(); + tag_vec.sort(); + tag_vec.join(",") + }; + + sqlx::query("UPDATE targets SET refined_tags = ? WHERE run_id = ? AND ip = ?") + .bind(&tags_str) + .bind(&self.run_id) + .bind(ip) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Comprehensive tag statistics aggregation using single bulk query + // Operation: Fetches all three tag columns in one SELECT (replaces 3N+1 queries) + pub async fn get_all_tags_with_counts(&self) -> Result> { + let rows = sqlx::query( + "SELECT COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(action_tags, '') as action_tags + FROM targets WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut manual_counts: HashMap = HashMap::new(); + let mut refined_counts: HashMap = HashMap::new(); + let mut automated_counts: HashMap = HashMap::new(); + let mut action_counts: HashMap = HashMap::new(); + + for row in rows { + let manual: String = row.get("manual_tags"); + let refined: String = row.get("refined_tags"); + let automated: String = row.get("automated_tags"); + let action: String = row.get("action_tags"); + + for tag in manual.split(',').map(str::trim).filter(|s| !s.is_empty()) { + *manual_counts.entry(tag.to_string()).or_insert(0) += 1; + } + for tag in refined.split(',').map(str::trim).filter(|s| !s.is_empty()) { + *refined_counts.entry(tag.to_string()).or_insert(0) += 1; + } + for tag in automated.split(',').map(str::trim).filter(|s| !s.is_empty()) { + *automated_counts.entry(tag.to_string()).or_insert(0) += 1; + } + for tag in action.split(',').map(str::trim).filter(|s| !s.is_empty()) { + *action_counts.entry(tag.to_string()).or_insert(0) += 1; + } + } + + let mut manual_tags: Vec = manual_counts.into_iter() + .map(|(name, count)| TagInfo { name, count }) + .collect(); + manual_tags.sort_by(|a, b| a.name.cmp(&b.name)); + + let mut refined_tags: Vec = refined_counts.into_iter() + .map(|(name, count)| TagInfo { name, count }) + .collect(); + refined_tags.sort_by(|a, b| b.count.cmp(&a.count)); + + let mut automated_tags: Vec = automated_counts.into_iter() + .map(|(name, count)| TagInfo { name, count }) + .collect(); + automated_tags.sort_by(|a, b| b.count.cmp(&a.count)); + + let mut action_tags: Vec = action_counts.into_iter() + .map(|(name, count)| TagInfo { name, count }) + .collect(); + action_tags.sort_by(|a, b| a.name.cmp(&b.name)); + + Ok(AllTagsWithCounts { + manual_tags, + refined_tags, + automated_tags, + action_tags, + }) + } + + // Context: Interactive tag pickers (e.g. bulk nuclei host filter on the Scanners page) + // Operation: Returns every distinct tag across all four tag columns, with the number of + // distinct hosts carrying each tag + pub async fn get_distinct_tags(&self) -> Result, Box> { + let rows = sqlx::query( + "SELECT ip, + COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(action_tags, '') as action_tags + FROM targets WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_all(&self.pool) + .await?; + + let mut tag_hosts: HashMap> = HashMap::new(); + for row in rows { + let ip: String = row.get("ip"); + for col in ["manual_tags", "refined_tags", "automated_tags", "action_tags"] { + let val: String = row.get(col); + for tag in val.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_hosts.entry(tag.to_string()).or_default().insert(ip.clone()); + } + } + } + + let mut tags: Vec = tag_hosts.into_iter() + .map(|(name, ips)| TagInfo { name, count: ips.len() }) + .collect(); + tags.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(tags) + } +} \ No newline at end of file diff --git a/src/sql/task_history.rs b/src/sql/task_history.rs new file mode 100644 index 0000000..ea9db10 --- /dev/null +++ b/src/sql/task_history.rs @@ -0,0 +1,94 @@ +// src/sql/task_history.rs +use std::error::Error; +use sqlx::Row; +use crate::workflow::types::CompletedTaskRecord; +use super::context::ProjectContext; + +#[derive(serde::Serialize)] +pub struct TaskHistorySummary { + pub id: String, + pub task_name: String, + pub target: String, + pub started_at: i64, + pub ended_at: i64, + pub status: String, +} + +impl ProjectContext { + pub async fn insert_task_history(&self, record: &CompletedTaskRecord) -> Result<(), Box> { + sqlx::query( + "INSERT OR IGNORE INTO task_history (id, run_id, task_name, target, started_at, ended_at, status, output) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) + .bind(&record.id) + .bind(&record.run_id) + .bind(&record.task_name) + .bind(&record.target) + .bind(record.started_at) + .bind(record.ended_at) + .bind(&record.status) + .bind(&record.output) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn list_task_history( + &self, + run_id: &str, + search: &str, + limit: i64, + ) -> Result, Box> { + let rows = if search.is_empty() { + sqlx::query( + "SELECT id, task_name, target, started_at, ended_at, status + FROM task_history WHERE run_id = ? + ORDER BY started_at DESC LIMIT ?" + ) + .bind(run_id) + .bind(limit) + .fetch_all(&self.pool) + .await? + } else { + let pattern = format!("%{}%", search); + sqlx::query( + "SELECT id, task_name, target, started_at, ended_at, status + FROM task_history WHERE run_id = ? + AND (task_name || ' ' || target || ' ' || status) LIKE ? + ORDER BY started_at DESC LIMIT ?" + ) + .bind(run_id) + .bind(&pattern) + .bind(limit) + .fetch_all(&self.pool) + .await? + }; + + Ok(rows.into_iter().map(|r| TaskHistorySummary { + id: r.get("id"), + task_name: r.get("task_name"), + target: r.get("target"), + started_at: r.get("started_at"), + ended_at: r.get("ended_at"), + status: r.get("status"), + }).collect()) + } + + pub async fn get_task_output(&self, id: &str) -> Result, Box> { + let row = sqlx::query( + "SELECT task_name, target, status, started_at, ended_at, output FROM task_history WHERE id = ?" + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|r| ( + r.get::("task_name"), + r.get::("target"), + r.get::("status"), + r.get::("output"), + r.get::("started_at"), + r.get::("ended_at"), + ))) + } +} diff --git a/src/sql/users.rs b/src/sql/users.rs new file mode 100755 index 0000000..d8ffe12 --- /dev/null +++ b/src/sql/users.rs @@ -0,0 +1,164 @@ +// src/sql/users.rs - User management for role-based authentication + +use std::error::Error; +use sqlx::Row; +use crate::sql::ProjectContext; +use crate::web::auth::{UserRole, hash_password}; + +#[derive(Clone, Debug)] +pub struct User { + pub id: i64, + pub username: String, + pub role: UserRole, + pub enabled: bool, + pub created_at: String, +} + +impl ProjectContext { + // Context: Creates new player user account + // Operation: Inserts new user with hashed password into database + pub async fn create_player_user( + &self, + username: &str, + password: &str, + ) -> Result<(), Box> { + let password_hash = hash_password(password)?; + + sqlx::query( + "INSERT INTO users (username, password_hash, role, enabled) VALUES (?1, ?2, ?3, TRUE)" + ) + .bind(username) + .bind(password_hash) + .bind(UserRole::Player.as_str()) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Retrieves all player users for management interface + // Operation: Returns list of player accounts with their status + pub async fn get_player_users(&self) -> Result, Box> { + let rows = sqlx::query( + "SELECT id, username, role, enabled, created_at FROM users WHERE role = ?1 ORDER BY created_at DESC" + ) + .bind(UserRole::Player.as_str()) + .fetch_all(&self.pool) + .await?; + + let mut users = Vec::new(); + for row in rows { + let role_str: String = row.get("role"); + let role = UserRole::from_str(&role_str).unwrap_or(UserRole::Player); + + users.push(User { + id: row.get("id"), + username: row.get("username"), + role, + enabled: row.get("enabled"), + created_at: row.get("created_at"), + }); + } + + Ok(users) + } + + // Context: Enables or disables player user account + // Operation: Updates enabled status for specified user + pub async fn set_user_enabled( + &self, + username: &str, + enabled: bool, + ) -> Result<(), Box> { + sqlx::query("UPDATE users SET enabled = ?1 WHERE username = ?2 AND role = ?3") + .bind(enabled) + .bind(username) + .bind(UserRole::Player.as_str()) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Resets player user password + // Operation: Updates password hash for specified user + pub async fn reset_user_password( + &self, + username: &str, + new_password: &str, + ) -> Result<(), Box> { + let password_hash = hash_password(new_password)?; + + sqlx::query("UPDATE users SET password_hash = ?1 WHERE username = ?2 AND role = ?3") + .bind(password_hash) + .bind(username) + .bind(UserRole::Player.as_str()) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Removes player user account + // Operation: Deletes user from database (admin cannot be deleted) + pub async fn delete_user(&self, username: &str) -> Result<(), Box> { + sqlx::query("DELETE FROM users WHERE username = ?1 AND role = ?2") + .bind(username) + .bind(UserRole::Player.as_str()) + .execute(&self.pool) + .await?; + + Ok(()) + } + + // Context: Checks if username already exists + // Operation: Returns true if username is already taken + pub async fn username_exists(&self, username: &str) -> Result> { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM users WHERE username = ?1") + .bind(username) + .fetch_one(&self.pool) + .await?; + + Ok(count > 0) + } + + // Context: First-run check — determines whether admin setup is required + // Operation: Returns true if an admin account already exists in the database + pub async fn admin_exists(&self) -> Result> { + let count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM users WHERE role = 'admin'" + ) + .fetch_one(&self.pool) + .await?; + Ok(count > 0) + } + + // Context: First-run admin account creation + // Operation: Creates the admin user with a bcrypt-hashed password; fails if admin already exists + pub async fn create_admin_user(&self, password: &str) -> Result<(), Box> { + let password_hash = hash_password(password)?; + sqlx::query( + "INSERT INTO users (username, password_hash, role, enabled) VALUES ('admin', ?1, 'admin', TRUE)" + ) + .bind(password_hash) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Admin login authentication + // Operation: Retrieves admin password hash and verifies it with bcrypt + pub async fn verify_admin_password(&self, password: &str) -> Result> { + use crate::web::auth::verify_password; + let hash: Option = sqlx::query_scalar( + "SELECT password_hash FROM users WHERE username = 'admin' AND role = 'admin' AND enabled = TRUE" + ) + .fetch_optional(&self.pool) + .await?; + + match hash { + Some(h) => Ok(verify_password(password, &h)), + None => Ok(false), + } + } +} \ No newline at end of file diff --git a/src/sql/workflow.rs b/src/sql/workflow.rs new file mode 100755 index 0000000..db137f7 --- /dev/null +++ b/src/sql/workflow.rs @@ -0,0 +1,153 @@ +// src/sql/workflow.rs +use std::error::Error; +use crate::utilities::{tag_info, tag_debug}; +use super::context::ProjectContext; + +impl ProjectContext { + // Context: Host workflow restart system for re-analyzing or re-scanning failed/incomplete hosts + // Operation: Resets host completely, clears hook cache, and triggers full workflow restart + pub async fn restart_host_workflow(&self, ip: &str) -> Result<(), Box> { + tag_info(&format!("Starting workflow restart for {}", ip)); + + // Reset the host completely (this function handles EVERYTHING): + // - Clears all flags and data + // - Clears fired hooks cache + // - Dispatches Hook::Ip to start workflow + self.reset_host_completely(ip).await?; + + tag_info(&format!("Host {} workflow restarted - nmap scan should begin shortly", ip)); + Ok(()) + } + + // Context: Complete host state reset system for workflow restart and cleanup operations + // Operation: Clears all host data, flags, and files with transaction safety and hook cache cleanup + pub async fn reset_host_completely(&self, ip: &str) -> Result<(), Box> { + let mut tx = self.pool.begin().await?; + + // Reset ALL flags and clear ALL data including new tag fields + sqlx::query( + "UPDATE targets SET + ports = '', + hostname = '', + automated_tags = '', + manual_tags = '', + refined_tags = '', + action_tags = '', + ad_domain = '', + nmap_scan = '{}', + nmap_xml = '{}', + full_scan_started = FALSE, + full_scan_complete = FALSE, + service_scan_started = FALSE, + service_scan_complete = FALSE, + http_discovery_started = FALSE, + http_discovery_complete = FALSE, + screenshots_started = FALSE, + screenshots_complete = FALSE, + analyzed = FALSE, + interesting = FALSE, + slow_scan_timeout = FALSE, + user_workflow_complete = FALSE + WHERE run_id = ? AND ip = ?" + ) + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await?; + + // Clear HTTP services + sqlx::query("DELETE FROM http_services WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await?; + + // Clear action status + sqlx::query("DELETE FROM action_status WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await?; + + // Clear action outputs (normalized table — replaces action_outputs JSON blob) + sqlx::query("DELETE FROM action_outputs WHERE run_id = ? AND ip = ?") + .bind(&self.run_id) + .bind(ip) + .execute(&mut *tx) + .await?; + + // CRITICAL: Commit the transaction + tx.commit().await?; + + // Clear the workflow hooks cache for this IP + { + let mut fired = crate::workflow::FIRED_HOOKS.lock().unwrap(); + fired.retain(|(_, target)| target != ip); + } + + // Clear per-host action queue state — drops busy flag and any queued actions + // so post-reset dispatches start fresh without inheriting stale queue + { + let mut states = crate::workflow::host_queue::HOST_ACTION_STATES.lock().unwrap(); + states.remove(ip); + } + + let screenshots_dir = crate::config::paths::get_screenshots_directory(&self.db_path, &self.run_id); + if let Ok(entries) = std::fs::read_dir(&screenshots_dir) { + for entry in entries.flatten() { + if let Some(filename) = entry.file_name().to_str() { + if filename.starts_with(&format!("{}_", ip.replace(".", "_"))) { + let file_path = entry.path(); + if let Err(e) = std::fs::remove_file(&file_path) { + tag_debug(&format!("Failed to remove screenshot {}: {}", file_path.display(), e)); + } + } + } + } + } + + // Trigger workflow restart + crate::workflow::dispatch_event(crate::workflow::Hook::Ip, ip, self); + tag_info(&format!("Host {} completely reset", ip)); + Ok(()) + } + + // Context: Persists current concurrency limits so they survive restart/resume + // Operation: Serialises ConcurrencyConfig to JSON and stores in the settings row + pub async fn save_concurrency_config( + &self, + config: &crate::workflow::concurrency::ConcurrencyConfig, + ) -> Result<(), Box> { + let json = serde_json::to_string(config)?; + sqlx::query( + "UPDATE settings SET concurrency_config = ?1 WHERE run_id = ?2" + ) + .bind(&json) + .bind(&self.run_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + // Context: Loads persisted concurrency limits at startup to restore previous settings + // Operation: Reads JSON from settings row and deserialises into ConcurrencyConfig + pub async fn load_concurrency_config( + &self, + ) -> Result, Box> { + let json: Option = sqlx::query_scalar( + "SELECT concurrency_config FROM settings WHERE run_id = ?" + ) + .bind(&self.run_id) + .fetch_optional(&self.pool) + .await? + .flatten(); + + match json { + Some(s) if !s.is_empty() => { + let config = serde_json::from_str(&s)?; + Ok(Some(config)) + } + _ => Ok(None), + } + } +} \ No newline at end of file diff --git a/src/tag/automation.rs b/src/tag/automation.rs new file mode 100755 index 0000000..f09c6eb --- /dev/null +++ b/src/tag/automation.rs @@ -0,0 +1,341 @@ +// src/tag/automation.rs +use std::error::Error; +use sqlx::Row; +use regex::Regex; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::{EnhancedTargetData, TriggerContext}; +use super::nmap_parsing::{extract_direct_nmap_tags, extract_domain_from_action_output, extract_os_family, normalize_service_name, is_noise_service}; +use super::nmap_xml_parsing::parse_nmap_xml; + + +// Context: Action commands produce outputs that may contain patterns indicating specific technologies or vulnerabilities +// Operation: Parses action outputs and applies regex pattern matching to extract and apply new manual tags +pub async fn apply_action_tags( + ctx: &ProjectContext, + ip: &str, + action_name: &str, + command_output: &str, + tag_patterns: &[crate::workflow::TagPattern], +) -> Result> { + let mut new_tags = Vec::new(); + + // Parse the JSON command output to get the actual command output text + let output_text = if let Ok(json_data) = serde_json::from_str::(command_output) { + // Extract output from JSON structure: {"commands": [{"output": "..."}]} + if let Some(commands) = json_data.get("commands").and_then(|c| c.as_array()) { + commands.iter() + .filter_map(|cmd| cmd.get("output").and_then(|o| o.as_str())) + .collect::>() + .join("\n") + } else { + command_output.to_string() + } + } else { + // Fallback: treat as plain text + command_output.to_string() + }; + + // Apply regex patterns to the extracted output text + for tag_pattern in tag_patterns { + let matched = Regex::new(&tag_pattern.pattern) + .map(|re| re.is_match(&output_text)) + .unwrap_or(false); + + if matched { + new_tags.push(tag_pattern.tag.clone()); + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Action '{}' pattern '{}' matched for {} -> tag '{}'", + action_name, tag_pattern.pattern, ip, tag_pattern.tag)); + } + } + } + + // Apply new tags if any were found + if !new_tags.is_empty() { + let mut existing_tags = ctx.get_action_tags(ip).await?; + + // Track which tags are genuinely new before inserting + let mut added: Vec = Vec::new(); + for tag in new_tags { + if existing_tags.insert(tag.clone()) { + added.push(tag); + } + } + + if !added.is_empty() { + ctx.set_action_tags(ip, existing_tags).await?; + + tag_info(&format!(" Applied {} new action tags to {} from '{}'", + added.len(), ip, action_name)); + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("New action tags for {}: {}", ip, added.join(","))); + } + + return Ok(true); + } + } + + Ok(false) +} + +// Context: Different triggers (service scan completion, action outputs) require different data gathering approaches +// Operation: Retrieves and enriches target data with services, OS info, domains, and action outputs based on trigger type +pub async fn get_enhanced_target_data( + ctx: &ProjectContext, + ip: &str, + trigger_context: &TriggerContext, +) -> Result> { + // Get basic target data including AD domain + let row = sqlx::query("SELECT ports, nmap_xml, hostname, ad_domain FROM targets WHERE run_id = ? AND ip = ?") + .bind(&ctx.run_id) + .bind(ip) + .fetch_one(&ctx.pool) + .await?; + + let nmap_xml: Option = row.get("nmap_xml"); + let ad_domain: Option = row.get("ad_domain"); + + let nmap_xml = nmap_xml.unwrap_or_default(); + + let mut target_data = EnhancedTargetData { + ad_domain: ad_domain.filter(|d| !d.is_empty()), + ..Default::default() + }; + + match trigger_context { + TriggerContext::ServiceScanComplete => { + // Parse from XML (sole structured source — no regex fallback) + let xml_result = if !nmap_xml.is_empty() && nmap_xml != "{}" { + serde_json::from_str::>(&nmap_xml) + .ok() + .and_then(|map| map.get("service_scan").cloned()) + .and_then(|xml| parse_nmap_xml(&xml)) + } else { + None + }; + + if let Some(parsed) = xml_result { + // Build ServiceInfo from XML ports + for port in &parsed.ports { + let svc_name = normalize_service_name(&port.service_name); + if !svc_name.is_empty() && !is_noise_service(&svc_name) { + target_data.services.push(super::ServiceInfo { + service_name: svc_name, + product: port.product.clone(), + }); + } + } + + // OS info from XML osmatch (or smb-os-discovery / ntlm-info — set in parse_nmap_xml) + if let Some(os_name) = parsed.os_name { + let family = extract_os_family(&os_name); + if family != "unknown" { + target_data.os_info = Some(super::OSInfo { + family, + name: os_name, + }); + } + } + + // Fallback C: CPE-based OS inference when osmatch + script extraction failed + if target_data.os_info.is_none() { + 'cpe: for port in &parsed.ports { + for cpe in &port.cpe { + let family = if cpe.starts_with("cpe:/o:linux") + || cpe.starts_with("cpe:/o:debian") + || cpe.starts_with("cpe:/o:canonical") + || cpe.starts_with("cpe:/o:redhat") + || cpe.starts_with("cpe:/o:centos") + { + Some("Linux") + } else if cpe.starts_with("cpe:/o:microsoft:windows") { + Some("Windows") + } else { + None + }; + if let Some(f) = family { + target_data.os_info = Some(super::OSInfo { + family: f.to_string(), + name: f.to_string(), + }); + break 'cpe; + } + } + } + } + + // Fallback D: version + extrainfo string scan + if target_data.os_info.is_none() { + let combined: String = parsed.ports.iter() + .flat_map(|p| [p.version.as_str(), p.extrainfo.as_str()]) + .collect::>() + .join(" ") + .to_lowercase(); + let family = if combined.contains("windows") { + Some("Windows") + } else if combined.contains("linux") + || combined.contains("debian") + || combined.contains("ubuntu") + || combined.contains("centos") + { + Some("Linux") + } else { + None + }; + if let Some(f) = family { + target_data.os_info = Some(super::OSInfo { + family: f.to_string(), + name: f.to_string(), + }); + } + } + + // Hostname from XML + if let Some(hostname) = parsed.hostname { + if let Err(e) = ctx.insert_hostname(ip, &hostname).await { + tag_debug(&format!("Failed to insert hostname for {}: {}", ip, e)); + } + } + + // AD domain from XML (priority-ordered extraction already done in parse_nmap_xml) + if let Some(domain) = parsed.ad_domain { + if target_data.ad_domain.as_ref() != Some(&domain) { + if ctx.insert_ad_domain(ip, &domain).await.unwrap_or(false) { + target_data.ad_domain = Some(domain); + } else { + target_data.ad_domain = Some(domain); + } + } + } + } + } + + TriggerContext::ActionOutput(action_name) => { + // Single-row fetch from normalized action_outputs table — correct type (Value not String) + let output_json: Option = sqlx::query_scalar( + "SELECT output_json FROM action_outputs WHERE run_id = ? AND ip = ? AND action_name = ?" + ) + .bind(&ctx.run_id) + .bind(ip) + .bind(action_name.as_str()) + .fetch_optional(&ctx.pool) + .await? + .flatten(); + + if let Some(json) = output_json { + if let Ok(v) = serde_json::from_str::(&json) { + target_data.action_outputs = v.get("commands") + .and_then(|c| c.as_array()) + .map(|cmds| cmds.iter() + .filter_map(|cmd| cmd.get("output").and_then(|o| o.as_str())) + .collect::>() + .join("\n")) + .unwrap_or_default(); + } + } + + if !target_data.action_outputs.is_empty() { + if let Some(domain) = extract_domain_from_action_output(&target_data.action_outputs) { + if target_data.ad_domain.as_ref() != Some(&domain) { + if ctx.insert_ad_domain(ip, &domain).await.unwrap_or(false) { + target_data.ad_domain = Some(domain); + } else { + target_data.ad_domain = Some(domain); + } + } + } + + // Extract hostname from nxc output pattern (name:HOSTNAME) if not already set + let existing_hostname: Option = row.get("hostname"); + if existing_hostname.map(|h| h.is_empty()).unwrap_or(true) { + if let Ok(re) = Regex::new(r"\(name:([^)]+)\)") { + if let Some(caps) = re.captures(&target_data.action_outputs) { + let name = caps[1].trim().to_string(); + if !name.is_empty() { + if let Err(e) = ctx.insert_hostname(ip, &name).await { + tag_debug(&format!("Failed to insert hostname for {}: {}", ip, e)); + } + } + } + } + } + } + } + } + + Ok(target_data) +} + +// Context: Main entry point for automated tagging triggered by service scans or action completions +// Operation: Coordinates the automatic tagging process by gathering enhanced data and applying direct extraction tags +pub async fn automatic_tag( + ctx: &ProjectContext, + ip: &str, + trigger_context: TriggerContext, +) -> Result<(), Box> { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Starting automatic tagging for {} with context: {:?}", ip, trigger_context)); + } + + // Get enhanced target data based on trigger context + let target_data = get_enhanced_target_data(ctx, ip, &trigger_context).await?; + + // Get current automated tags + let mut existing_tags = ctx.get_automated_tags(ip).await?; + let original_count = existing_tags.len(); + + // Apply only direct extraction tags (no YAML rules) + let auto_tags_added = match trigger_context { + TriggerContext::ServiceScanComplete => { + let auto_tags = extract_direct_nmap_tags(&target_data); + let before_auto = existing_tags.len(); + for tag in auto_tags { + existing_tags.insert(tag); + } + existing_tags.len() - before_auto + } + + TriggerContext::ActionOutput(_) => { + // Tag generation for action output is handled by apply_action_tags() elsewhere. + // This branch exists because get_enhanced_target_data() has a useful side-effect: + // it parses action output to extract the AD domain and writes it back to the DB + // via ctx.update_ad_domain(). That domain write happens before we reach this + // match arm — so returning 0 here is intentional, not an oversight. + 0 + } + }; + + // Update automated tags in database if we have new ones + if existing_tags.len() > original_count { + ctx.set_automated_tags(ip, existing_tags.clone()).await?; + + let trigger_desc = match trigger_context { + TriggerContext::ServiceScanComplete => "service scan", + TriggerContext::ActionOutput(ref action_name) => &format!("action '{}'", action_name), + }; + + if auto_tags_added > 0 { + tag_info(&format!(" Applied {} new automated tags to {} ({})", + auto_tags_added, ip, trigger_desc)); + } + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Final automated tags for {}: {}", ip, + existing_tags.iter().cloned().collect::>().join(","))); + } + } else { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + let trigger_desc = match trigger_context { + TriggerContext::ServiceScanComplete => "service scan", + TriggerContext::ActionOutput(ref action_name) => &format!("action '{}'", action_name), + }; + tag_debug(&format!("No new automated tags applied for {} ({})", ip, trigger_desc)); + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/tag/http.rs b/src/tag/http.rs new file mode 100755 index 0000000..ac8590e --- /dev/null +++ b/src/tag/http.rs @@ -0,0 +1,530 @@ +// src/tag/http.rs +use std::collections::{HashSet, HashMap}; +use std::error::Error; +use crate::sql::ProjectContext; +use crate::http::HttpServiceInfo; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; + +// Context: String truncation can cause panic on non-UTF-8 character boundaries in Rust +// Operation: Safely truncates strings by finding valid UTF-8 character boundaries before the max length +fn safe_truncate_string(s: &str, max_chars: usize) -> &str { + if s.len() <= max_chars { + return s; + } + + // Find the last valid UTF-8 character boundary before max_chars + let mut end = max_chars; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + + if end == 0 { + // Fallback: if we can't find a boundary, return empty string + return ""; + } + + &s[..end] +} + +// Context: HTTP services contain rich information in headers, titles, and content that can identify technologies +// Operation: Analyzes all HTTP services for a target and applies multi-layer detection to generate automated tags +pub async fn auto_tag_http( + ctx: &ProjectContext, + ip: &str, +) -> Result<(), Box> { + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Starting enhanced HTTP auto-tagging for {}", ip)); + } + + // Get all HTTP services for this host from database + let http_services = ctx.get_http_services(ip).await?; + + if http_services.is_empty() { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("No HTTP services found for auto-tagging on {}", ip)); + } + return Ok(()); + } + + let mut discovered_tags = HashSet::new(); + let mut detection_details = Vec::new(); + + // Analyze each HTTP service and aggregate tags + for (port, service_json) in http_services { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Analyzing HTTP service on {}:{}", ip, port)); + } + + // Parse service info from JSON + let service_info: HttpServiceInfo = match serde_json::from_str(&service_json) { + Ok(info) => info, + Err(e) => { + tag_debug(&format!("Failed to parse HTTP service JSON for {}:{}: {}", ip, port, e)); + continue; + } + }; + + // Extract tags from this service using enhanced detection + let service_tags = analyze_http_service(&service_info); + + // Log detection details for debugging + if !service_tags.is_empty() { + detection_details.push(format!("{}:{} -> {}", + port, + service_info.title.chars().take(30).collect::(), + service_tags.iter().cloned().collect::>().join(",") + )); + } + + discovered_tags.extend(service_tags); + } + + if discovered_tags.is_empty() { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("No HTTP technologies detected for {}", ip)); + } + return Ok(()); + } + + // Get current automated tags and merge with new HTTP tags + let mut existing_tags = ctx.get_automated_tags(ip).await?; + let original_count = existing_tags.len(); + + // Add new HTTP-derived tags + for tag in discovered_tags { + existing_tags.insert(tag); + } + + // Update automated tags in database if we have new ones + if existing_tags.len() > original_count { + ctx.set_automated_tags(ip, existing_tags.clone()).await?; + + let new_count = existing_tags.len() - original_count; + tag_info(&format!("Applied {} enhanced HTTP tags to {}", new_count, ip)); + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("HTTP detection details for {}:", ip)); + for detail in detection_details { + tag_debug(&format!(" {}", detail)); + } + } + } else { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("No new enhanced HTTP tags applied for {}", ip)); + } + } + + Ok(()) +} + +// Context: Single HTTP service may reveal multiple technologies through different indicators +// Operation: Applies multiple detection layers (headers, title, content, device types) to extract comprehensive tags +pub fn analyze_http_service(service_info: &HttpServiceInfo) -> HashSet { + let mut tags = HashSet::new(); + + // Layer 1: Server header analysis + tags.extend(detect_web_servers(&service_info.response_headers)); + + // Layer 2: Title and header analysis + tags.extend(detect_web_applications(&service_info.title, &service_info.response_headers)); + + // Layer 3: Content-based analysis for missing cases + tags.extend(detect_from_response_content(&service_info.response_body)); + + // Layer 4: Storage/NAS detection + tags.extend(detect_storage_devices(&service_info.title, &service_info.response_headers)); + + // Layer 5: Network infrastructure detection + tags.extend(detect_network_infrastructure(&service_info.title, &service_info.response_headers)); + + // Layer 6: Printer detection + tags.extend(detect_printers(&service_info.title, &service_info.response_headers)); + + tags +} + +// Context: Server headers and related headers contain specific patterns that identify web server software +// Operation: Generic RFC 7231 product-token extraction; each space-delimited token before '/' is a product name. +// A minimal disambiguation table handles the few cases where the raw product name is misleading. +pub fn detect_web_servers(headers: &HashMap) -> HashSet { + let mut tags = HashSet::new(); + + // Raw product substring → canonical tag. Only for genuinely misleading names. + const RENAMES: &[(&str, &str)] = &[ + ("apache-coyote", "tomcat"), + ("microsoft-iis", "iis"), + ("microsoft-httpapi", "windows-http-api"), + ("km-mfp-http", "kyocera-printer"), + ("hp_compact_server", "hp-printer"), + ("hp http server", "hp-printer"), + ]; + const SKIP: &[&str] = &["server", "http", "https", "web", "the", "and"]; + + let mut extract_from = |header_val: &str| { + for raw_tok in header_val.split_whitespace() { + if raw_tok.starts_with('(') { continue; } + let product = raw_tok.split('/').next().unwrap_or(""); + let product = product + .trim_matches(|c: char| !c.is_alphanumeric() && c != '-') + .to_lowercase(); + if product.len() < 3 + || product.chars().all(|c| c.is_numeric() || c == '.') + || SKIP.contains(&product.as_str()) + { + continue; + } + let tag = RENAMES + .iter() + .find(|(from, _)| product.contains(*from)) + .map(|(_, to)| to.to_string()) + .unwrap_or(product); + tags.insert(tag); + } + }; + + if let Some(server) = headers.get("server") { + extract_from(server); + } + if let Some(xpb) = headers.get("x-powered-by") { + extract_from(xpb); + } + if headers.contains_key("x-jenkins") { + tags.insert("jenkins".to_string()); + } + + tags +} + +// Context: Web page titles and specific headers often contain application names and technology indicators +// Operation: Analyzes page titles and headers to identify web applications, frameworks, and development tools +pub fn detect_web_applications(title: &str, headers: &HashMap) -> HashSet { + let mut tags = HashSet::new(); + let title_lower = title.to_lowercase(); + + // FIX: Enhanced Tomcat detection from title (CRITICAL for missing cases like 10.112.10.237:8080) + if title_lower.contains("apache tomcat") || + title_lower.contains("tomcat/") || + title_lower.contains("tomcat ") { + tags.insert("tomcat".to_string()); + } + + // Java application servers from title + if title_lower.contains("jetty") { + tags.insert("jetty".to_string()); + } + if title_lower.contains("jboss") || title_lower.contains("wildfly") || title_lower.contains("eap ") { + tags.insert("jboss".to_string()); + } + if title_lower.contains("weblogic") { + tags.insert("weblogic".to_string()); + } + if title_lower.contains("websphere") { + tags.insert("websphere".to_string()); + } + if title_lower.contains("glassfish") { + tags.insert("glassfish".to_string()); + } + + // Development tools (found in actual data) + if title_lower.contains("jenkins") { + tags.insert("jenkins".to_string()); + } + if title_lower.contains("gitlab") { + tags.insert("gitlab".to_string()); + } + if title_lower.contains("grafana") { + tags.insert("grafana".to_string()); + } + if title_lower.contains("nexus") { + tags.insert("nexus".to_string()); + } + if title_lower.contains("artifactory") { + tags.insert("artifactory".to_string()); + } + + // Database admin tools + if title_lower.contains("phpmyadmin") { + tags.insert("phpmyadmin".to_string()); + } + if title_lower.contains("adminer") { + tags.insert("adminer".to_string()); + } + if title_lower.contains("pgadmin") { + tags.insert("pgadmin".to_string()); + } + + // Network management (found in actual data) + if title_lower.contains("aos-web") || title_lower.contains("omnivista") { + tags.insert("network-management".to_string()); + } + if title_lower.contains("pure storage") { + tags.insert("pure-storage".to_string()); + } + if title_lower.contains("eaton ups") || title_lower.contains("network management card") { + tags.insert("ups-management".to_string()); + } + if title_lower.contains("nport web console") { + tags.insert("moxa-device".to_string()); + } + + // Content filters (found in actual data) + if title_lower.contains("web filter block") || title_lower.contains("access restricted") { + tags.insert("web-filter".to_string()); + } + + // WAMP/Development stacks + if title_lower.contains("wampserver") || title_lower.contains("xampp") { + tags.insert("wamp-stack".to_string()); + } + + // GIS applications + if title_lower.contains("arcgis") { + tags.insert("arcgis".to_string()); + } + + // Web servers from title (fallback when server header missing) + if title_lower.contains("apache http server") || title_lower.contains("apache2") { + tags.insert("apache".to_string()); + } + if title_lower.contains("nginx") { + tags.insert("nginx".to_string()); + } + if title_lower.contains("iis windows") || title_lower.contains("internet information services") { + tags.insert("iis".to_string()); + } + + // CMS Detection + if title_lower.contains("wordpress") { + tags.insert("wordpress".to_string()); + } + if title_lower.contains("drupal") { + tags.insert("drupal".to_string()); + } + if title_lower.contains("joomla") { + tags.insert("joomla".to_string()); + } + if title_lower.contains("sharepoint") { + tags.insert("sharepoint".to_string()); + } + + // Framework detection from headers + if let Some(x_powered_by) = headers.get("x-powered-by") { + let powered_by_lower = x_powered_by.to_lowercase(); + if powered_by_lower.contains("wordpress") { + tags.insert("wordpress".to_string()); + } + if powered_by_lower.contains("drupal") { + tags.insert("drupal".to_string()); + } + if powered_by_lower.contains("spring") { + tags.insert("spring-boot".to_string()); + } + } + + tags +} + +// Context: Some technologies are only identifiable through specific content patterns in response bodies +// Operation: Analyzes response body content for technology-specific strings and patterns using safe UTF-8 handling +pub fn detect_from_response_content(response_body: &str) -> HashSet { + let mut tags = HashSet::new(); + let content_lower = response_body.to_lowercase(); + + // CRITICAL FIX: Use safe UTF-8 truncation instead of slice + let content_sample = safe_truncate_string(&content_lower, 2000); + + // Tomcat-specific content patterns + if content_sample.contains("catalina_home") || + content_sample.contains("manager application") || + content_sample.contains("apache tomcat") { + tags.insert("tomcat".to_string()); + } + + // Spring Boot error pages + if content_sample.contains("whitelabel error page") || + content_sample.contains("spring boot") { + tags.insert("spring-boot".to_string()); + } + + // Jenkins content + if content_sample.contains("dashboard [jenkins]") || + content_sample.contains("jenkins ver.") { + tags.insert("jenkins".to_string()); + } + + // VMware products + if content_sample.contains("vmware vsphere") || + content_sample.contains("vmware vcenter") { + tags.insert("vmware".to_string()); + } + + // Database admin tools + if content_sample.contains("welcome to phpmyadmin") { + tags.insert("phpmyadmin".to_string()); + } + if content_sample.contains("welcome to couchdb") { + tags.insert("couchdb".to_string()); + } + + // Network management systems + if content_sample.contains("pfsense") && content_sample.contains("firewall") { + tags.insert("pfsense".to_string()); + } + if content_sample.contains("mikrotik") && content_sample.contains("routeros") { + tags.insert("mikrotik".to_string()); + } + + // Storage systems + if content_sample.contains("synology diskstation") || + content_sample.contains("dsm login") { + tags.insert("synology".to_string()); + } + if content_sample.contains("qnap") && content_sample.contains("qts") { + tags.insert("qnap".to_string()); + } + + tags +} + +// Context: Network storage devices have distinctive web interfaces with specific title and header patterns +// Operation: Identifies NAS devices, storage systems, and storage management interfaces from titles and headers +pub fn detect_storage_devices(title: &str, headers: &HashMap) -> HashSet { + let mut tags = HashSet::new(); + let title_lower = title.to_lowercase(); + + // Synology NAS (high confidence) + if title_lower.contains("synology diskstation") || + title_lower.contains("synology rackstation") || + title_lower.contains("synology") { + tags.insert("synology".to_string()); + } + + // QNAP NAS (high confidence) + if title_lower.contains("qts") && (title_lower.contains("login") || title_lower.contains("qnap")) { + tags.insert("qnap".to_string()); + } + + // Pure Storage (found in actual data) + if title_lower.contains("pure storage") { + tags.insert("pure-storage".to_string()); + } + + // NetApp (for future detection) + if title_lower.contains("netapp") || title_lower.contains("ontap") { + tags.insert("netapp".to_string()); + } + + // Check server header for additional NAS detection + if let Some(server) = headers.get("server") { + let server_lower = server.to_lowercase(); + if server_lower.contains("synology") { + tags.insert("synology".to_string()); + } + } + + tags +} + +// Context: Network infrastructure devices expose web management interfaces with identifying characteristics +// Operation: Detects routers, switches, firewalls, UPS systems, and other network infrastructure from page titles +pub fn detect_network_infrastructure(title: &str, _headers: &HashMap) -> HashSet { + let mut tags = HashSet::new(); + let title_lower = title.to_lowercase(); + + // Firewall/Router detection + if title_lower.contains("pfsense") { + tags.insert("pfsense".to_string()); + } + if title_lower.contains("routeros") || + (title_lower.contains("mikrotik") && title_lower.contains("router")) { + tags.insert("mikrotik".to_string()); + } + if title_lower.contains("ubiquiti") || title_lower.contains("unifi") { + tags.insert("ubiquiti".to_string()); + } + + // Network management systems (found in actual data) + if title_lower.contains("aos-web") { + tags.insert("alcatel-switch".to_string()); + } + if title_lower.contains("omnivista") { + tags.insert("network-management".to_string()); + } + + // UPS/Power management (found in actual data) + if title_lower.contains("eaton ups") || + title_lower.contains("network management card") { + tags.insert("ups-management".to_string()); + } + + // Industrial networking (found in actual data) + if title_lower.contains("nport") || title_lower.contains("moxa") { + tags.insert("moxa-device".to_string()); + } + + tags +} + +// Context: Network printers expose web interfaces with manufacturer-specific patterns in titles and headers +// Operation: Identifies various printer brands and models by analyzing title patterns and server headers +pub fn detect_printers(title: &str, headers: &HashMap) -> HashSet { + let mut tags = HashSet::new(); + let title_lower = title.to_lowercase(); + + // HP Printers (high confidence) - enhanced patterns from real data + if title_lower.contains("hp") && + (title_lower.contains("device status") || + title_lower.contains("laserjet") || + title_lower.contains("officejet") || + title_lower.contains("deskjet") || + title_lower.contains("printer")) { + tags.insert("hp-printer".to_string()); + } + + // Kyocera Printers (found extensively in actual data) + if title_lower.contains("kyocera") || + title_lower.contains("taskalfa") || + title_lower.contains("ecosys") { + tags.insert("kyocera-printer".to_string()); + } + + // Canon Printers + if title_lower.contains("canon") && + (title_lower.contains("printer") || + title_lower.contains("pixma") || + title_lower.contains("imageclass")) { + tags.insert("canon-printer".to_string()); + } + + // Ricoh Printers + if title_lower.contains("ricoh") || + (title_lower.contains("aficio") && title_lower.contains("mp")) { + tags.insert("ricoh-printer".to_string()); + } + + // Brother Printers + if title_lower.contains("brother") && title_lower.contains("printer") { + tags.insert("brother-printer".to_string()); + } + + // Xerox Printers + if title_lower.contains("xerox") { + tags.insert("xerox-printer".to_string()); + } + + // Check server headers for printer detection + if let Some(server) = headers.get("server") { + let server_lower = server.to_lowercase(); + if server_lower.contains("hp http server") || + server_lower.contains("hp_compact_server") { + tags.insert("hp-printer".to_string()); + } + if server_lower.contains("km-mfp-http") { + tags.insert("kyocera-printer".to_string()); + } + } + + tags +} \ No newline at end of file diff --git a/src/tag/mod.rs b/src/tag/mod.rs new file mode 100755 index 0000000..c3f58cf --- /dev/null +++ b/src/tag/mod.rs @@ -0,0 +1,46 @@ +// src/tag/mod.rs + +pub mod nmap_parsing; +pub mod nmap_xml_parsing; +pub mod http; +pub mod refinement; +pub mod automation; + +// Re-export commonly used items to maintain compatibility + +// Core types +/// Enhanced service information with AD domain support +#[derive(Debug, Default, Clone)] +pub struct ServiceInfo { + pub service_name: String, // http, ssh, mysql, etc. + pub product: String, // Apache, nginx, OpenSSH, etc. +} + +/// Minimal OS information for direct tagging +#[derive(Debug, Default, Clone)] +pub struct OSInfo { + pub family: String, // Windows, Linux, etc. + pub name: String, // Windows Server 2019, Ubuntu 20.04 +} + +/// Enhanced target data with AD domain information +#[derive(Debug, Default)] +pub struct EnhancedTargetData { + pub services: Vec, + pub os_info: Option, + pub action_outputs: String, // Action output (for action triggers) + pub ad_domain: Option, // Active Directory domain +} + +/// Trigger context for detection +#[derive(Debug, Clone)] +pub enum TriggerContext { + ServiceScanComplete, + ActionOutput(String), // action_name +} + +// Re-export main public functions +pub use automation::{apply_action_tags, automatic_tag}; +pub use http::auto_tag_http; +pub use refinement::refine_automated_tags; +pub use nmap_xml_parsing::{parse_nmap_xml, XmlPortInfo}; \ No newline at end of file diff --git a/src/tag/nmap_parsing.rs b/src/tag/nmap_parsing.rs new file mode 100755 index 0000000..380ee48 --- /dev/null +++ b/src/tag/nmap_parsing.rs @@ -0,0 +1,230 @@ +// src/tag/nmap_parsing.rs +use std::collections::HashSet; +use regex::Regex; +use super::EnhancedTargetData; + +// Context: Enhanced target data contains parsed services, OS info, and domain information from nmap scans +// Operation: Extracts and formats tags directly from parsed nmap data including services, products, OS, and AD domains +pub fn extract_direct_nmap_tags(data: &EnhancedTargetData) -> HashSet { + let mut tags = HashSet::new(); + + // === SERVICE-BASED TAGS === + for service in &data.services { + // Service name (http, ssh, mysql, etc.) + if !service.service_name.is_empty() { + tags.insert(service.service_name.clone()); + } + + // Product name (apache, nginx, openssh, etc.) — lowercase for consistent tagging + if !service.product.is_empty() { + tags.insert(service.product.to_lowercase()); + } + } + + // === OS-BASED TAGS === + if let Some(ref os) = data.os_info { + if !os.family.is_empty() { + tags.insert(os.family.clone()); + } + if !os.name.is_empty() { + tags.insert(os.name.clone()); + } + } + + // Infer OS family from service product strings when nmap has no osmatch element. + // Hosts behind firewalls often lack an OS fingerprint but SMB/RDP product strings + // ("Windows Server 2016 Standard...") still reveal the OS family. + if !tags.contains("Windows") && !tags.contains("Linux") { + let products_lower: String = data.services.iter() + .map(|s| s.product.to_lowercase()) + .collect::>() + .join(" "); + if products_lower.contains("windows") { + tags.insert("Windows".to_string()); + } else if products_lower.contains("linux") { + tags.insert("Linux".to_string()); + } + } + + // === AD DOMAIN TAGS === + if let Some(ref domain) = data.ad_domain { + if !domain.is_empty() { + tags.insert(format!("domain-{}", domain.replace(".", "-"))); + } + } + + tags +} + +// Context: Raw domain names from nmap may have formatting issues, trailing characters, or LDAP formatting +// Operation: Cleans and normalizes domain names to standard format, handling LDAP DN format and trailing artifacts +pub fn normalize_domain_name(domain: &str) -> String { + let domain = domain.trim().to_lowercase(); + + // CRITICAL FIX: Remove trailing dots and numbers (like sevenkingdoms.local0.) + let domain = domain.trim_end_matches(|c: char| c == '.' || c.is_ascii_digit()); + + // Handle DC= LDAP format: "DC=essos,DC=local" -> "essos.local" + if domain.starts_with("dc=") { + return domain + .replace("dc=", "") + .replace(",", ".") + .trim() + .to_string(); + } + + // SPECIAL CASE: Handle certificate-style domains + // For domains extracted from certificates, ensure they're proper domains + let parts: Vec<&str> = domain.split('.').collect(); + if parts.len() >= 2 { + // Only take the last two parts for simple domains like "sevenkingdoms.local" + // But preserve longer domains like "north.sevenkingdoms.local" + const INTERNAL_TLDS: &[&str] = &[ + ".local", ".lan", ".corp", ".int", ".ad", ".home", ".internal", + ]; + + let clean_domain = if parts.len() == 2 { + domain.to_string() + } else if parts.len() >= 3 { + // For FQDNs like "north.sevenkingdoms.local", extract base domain (last 2 labels) + let potential_domain = parts[parts.len()-2..].join("."); + if INTERNAL_TLDS.iter().any(|tld| potential_domain.ends_with(tld)) { + potential_domain + } else { + domain.to_string() + } + } else { + domain.to_string() + }; + + // Final validation + match clean_domain.as_ref() { + "workgroup" | "" => String::new(), + _ if clean_domain.len() < 3 => String::new(), + _ if !clean_domain.contains('.') => String::new(), // Must be FQDN + _ => clean_domain, + } + } else { + String::new() + } +} + +// Context: Domain name validation is needed to filter out invalid strings extracted from nmap output +// Operation: Validates domain name format, length, and content to ensure it represents a legitimate domain +pub fn is_valid_domain(domain: &str) -> bool { + // Filter out obvious non-domains + if domain.is_empty() || + domain.len() < 2 || + domain.len() > 253 || + domain.contains("workgroup") || + domain.contains("localhost") || + domain.starts_with("127.") || + domain.parse::().is_ok() { + return false; + } + + // Basic domain validation + domain.chars().all(|c| c.is_alphanumeric() || c == '.' || c == '-') && + !domain.starts_with('-') && + !domain.ends_with('-') && + !domain.starts_with('.') && + !domain.ends_with('.') +} + +// Context: Raw nmap service names may have inconsistent formatting or uncertainty markers needing standardization +// Operation: Cleans and normalizes service names by removing uncertainty markers and mapping to consistent names +pub fn normalize_service_name(raw_service: &str) -> String { + // Remove uncertainty markers and clean up + let cleaned = raw_service + .trim_end_matches('?') // Remove trailing ? + .trim() + .to_lowercase(); + + // Normalize common service names + match cleaned.as_str() { + // Core Windows services + "microsoft-ds" => "smb".to_string(), + "cifs" => "smb".to_string(), // alternate SMB identifier on some hosts + "ms-sql-s" => "mssql".to_string(), + "ms-sql-m" => "mssql".to_string(), // SQL Server Browser (1434) + "ms-sql-monitor" => "mssql".to_string(), + "ms-sql-ds" => "mssql".to_string(), // SQL Server Discovery service + "ms-wbt-server" => "rdp".to_string(), + "wsman" => "winrm".to_string(), + "kerberos-sec" => "kerberos".to_string(), + "netbios-ssn" => "netbios".to_string(), + "netbios-ns" => "netbios".to_string(), // NetBIOS Name Service (UDP 137) + "msrpc" => "rpc".to_string(), + "epmap" => "rpc".to_string(), // DCE endpoint mapper (135) + "ncacn_http" => "rpc".to_string(), // RPC over HTTP (593) + "ncacn_ip_tcp" => "rpc".to_string(), // RPC over TCP (dynamic ports 49152+) + "msrpc-base" => "rpc".to_string(), + "domain" => "dns".to_string(), + // TLS variants — normalized to base service + "ldaps" => "ldap".to_string(), + "imaps" => "imap".to_string(), + "smtps" => "smtp".to_string(), + "pop3s" => "pop3".to_string(), + "ftps" => "ftp".to_string(), + "ftps-data" => "ftp".to_string(), + // HTTP on alternate ports + "http-alt" => "http".to_string(), + // Handle ssl/ prefixes + s if s.starts_with("ssl/") => s.strip_prefix("ssl/").unwrap_or(s).to_string(), + // Keep other services as-is (including http-proxy) + _ => cleaned, + } +} + +// Context: Some nmap service identifiers represent scan artifacts or unhelpful information rather than actual services +// Operation: Identifies and filters out non-informative service names that shouldn't become tags +pub fn is_noise_service(service: &str) -> bool { + matches!(service, + "syn-ack" | "ttl" | "filtered" | "closed" | + "tcpwrapped" | "unknown" | "" + ) || service.starts_with("cpe:") || service.len() < 2 +} + +// Context: OS detection strings from nmap need categorization into major OS families for consistent tagging +// Operation: Analyzes OS strings to determine and return the primary OS family (Windows, Linux, etc.) +pub fn extract_os_family(os_string: &str) -> String { + let os_lower = os_string.to_lowercase(); + + if os_lower.contains("windows") { + "Windows".to_string() + } else if os_lower.contains("linux") { + "Linux".to_string() + } else if os_lower.contains("freebsd") { + "FreeBSD".to_string() + } else if os_lower.contains("macos") || os_lower.contains("darwin") { + "macOS".to_string() + } else { + "unknown".to_string() + } +} + +// Context: Action tool outputs (enum4linux, nxc, etc.) contain domain information in various formats +// Operation: Parses action outputs using multiple regex patterns to extract and normalize AD domain names +pub fn extract_domain_from_action_output(output: &str) -> Option { + let domain_patterns = vec![ + Regex::new(r"(?i)Domain Name:\s*([A-Za-z0-9.-]+)").unwrap(), + Regex::new(r"(?i)Domain:\s*([A-Za-z0-9.-]+)").unwrap(), + Regex::new(r"(?i)Workgroup/Domain:\s*([A-Za-z0-9.-]+)").unwrap(), + Regex::new(r"(?i)NetBIOS domain:\s*([A-Za-z0-9.-]+)").unwrap(), + // From enum4linux output + Regex::new(r"(?i)Domain Name\s*\.\.\.\s*([A-Za-z0-9.-]+)").unwrap(), + // From nxc output + Regex::new(r"(?i)\[.*\]\s+([A-Za-z0-9.-]+\.[A-Za-z0-9.-]+)\s+\d+").unwrap(), + ]; + + for pattern in &domain_patterns { + if let Some(caps) = pattern.captures(output) { + let domain_candidate = caps[1].to_string().trim().to_lowercase(); + if is_valid_domain(&domain_candidate) && domain_candidate != "workgroup" { + return Some(normalize_domain_name(&domain_candidate)); + } + } + } + + None +} \ No newline at end of file diff --git a/src/tag/nmap_xml_parsing.rs b/src/tag/nmap_xml_parsing.rs new file mode 100644 index 0000000..e4b3f66 --- /dev/null +++ b/src/tag/nmap_xml_parsing.rs @@ -0,0 +1,468 @@ +// src/tag/nmap_xml_parsing.rs +// Context: Replaces fragile regex-on-text nmap parsing with structured XML parsing +// Operation: Parses nmap -oX XML output into typed structs used by tagging and port display + +use std::collections::HashMap; + +#[derive(Debug, Clone)] +pub struct XmlScript { + pub id: String, + pub elements: HashMap, +} + +#[derive(Debug, Clone)] +pub struct XmlPortInfo { + pub port: String, + pub protocol: String, + pub state: String, + pub service_name: String, + pub product: String, + pub version: String, + pub extrainfo: String, + pub cpe: Vec, + pub scripts: Vec, +} + +#[derive(Debug, Default)] +pub struct NmapXmlResult { + pub ports: Vec, + pub os_name: Option, + pub os_accuracy: u8, + pub hostname: Option, + pub ad_domain: Option, + pub host_scripts: Vec, +} + +// --- helpers ---------------------------------------------------------------- + +fn parse_script(node: roxmltree::Node) -> XmlScript { + let id = node.attribute("id").unwrap_or("").to_string(); + let mut elements: HashMap = HashMap::new(); + for child in node.children() { + if child.tag_name().name() == "elem" { + if let Some(key) = child.attribute("key") { + let text = child.text().unwrap_or("").trim().to_string(); + elements.insert(key.to_string(), text); + } + } + // Handle /
nested in ssl-cert + if child.tag_name().name() == "table" { + let table_key = child.attribute("key").unwrap_or(""); + for elem in child.children() { + if elem.tag_name().name() == "elem" { + if let Some(key) = elem.attribute("key") { + let full_key = if table_key.is_empty() { + key.to_string() + } else { + format!("{}/{}", table_key, key) + }; + let text = elem.text().unwrap_or("").trim().to_string(); + elements.insert(full_key, text); + } + } + } + } + } + XmlScript { id, elements } +} + +/// Convert "DC=sevenkingdoms,DC=local" -> "sevenkingdoms.local" +fn ldap_dn_to_domain(dn: &str) -> String { + let lower = dn.to_lowercase(); + if lower.contains("dc=") { + let parts: Vec<&str> = lower + .split(',') + .map(|p| p.trim()) + .filter(|p| p.starts_with("dc=")) + .map(|p| &p[3..]) + .collect(); + if !parts.is_empty() { + return parts.join("."); + } + } + String::new() +} + +fn is_valid_domain(d: &str) -> bool { + !d.is_empty() + && d.contains('.') + && d.len() >= 4 + && !d.contains("workgroup") + && !d.contains("localhost") + && d.chars().all(|c| c.is_alphanumeric() || c == '.' || c == '-') +} + +/// Extract domain from an ssl-cert commonName. +/// If the CN is a FQDN like "host.domain.tld", return "domain.tld". +/// If it already looks like a bare domain (no host prefix, contains a dot), return as-is. +fn domain_from_cn(cn: &str) -> Option { + let cn = cn.trim().to_lowercase(); + let parts: Vec<&str> = cn.split('.').collect(); + if parts.len() < 2 { + return None; + } + // If it's a multi-label name, drop the first label (hostname part) + let candidate = if parts.len() >= 3 { + parts[1..].join(".") + } else { + cn.clone() + }; + if is_valid_domain(&candidate) { + Some(candidate) + } else { + None + } +} + +// --- main entry point ------------------------------------------------------- + +/// Parse nmap XML output into NmapXmlResult. +/// Returns None if the XML is empty, invalid, or contains no element. +pub fn parse_nmap_xml(xml: &str) -> Option { + if xml.trim().is_empty() { + return None; + } + + // nmap XML always contains — enable DTD support so roxmltree + // doesn't reject the document (default allow_dtd is false → silent parse failure). + // nodes_limit: None = no cap on the number of XML nodes (service scans can be large). + let doc = roxmltree::Document::parse_with_options( + xml, + roxmltree::ParsingOptions { allow_dtd: true, nodes_limit: u32::MAX }, + ).ok()?; + let root = doc.root_element(); + + // Locate the first element + let host_node = root + .descendants() + .find(|n| n.tag_name().name() == "host")?; + + let mut result = NmapXmlResult::default(); + + // ---- hostname ----------------------------------------------------------- + if let Some(hostnames) = host_node + .children() + .find(|n| n.tag_name().name() == "hostnames") + { + for hn in hostnames.children() { + if hn.tag_name().name() == "hostname" { + let t = hn.attribute("type").unwrap_or(""); + // prefer "user" or "PTR" over generated names + if t == "user" || t == "PTR" || result.hostname.is_none() { + if let Some(name) = hn.attribute("name") { + let name = name.trim().to_string(); + if !name.is_empty() { + result.hostname = Some(name); + } + } + } + } + } + } + + // ---- hostname fallbacks (smb-os-discovery, ntlm-info) ------------------- + // If PTR/user hostname wasn't in , extract from script output. + // smb-os-discovery "Computer Name" gives the short NetBIOS name directly. + // ntlm-info "DNS_Computer_Name" gives the FQDN; strip_domain_suffix handles it. + if result.hostname.is_none() { + for script in &result.host_scripts { + if script.id == "smb-os-discovery" { + let name = script.elements.get("Computer Name") + .or_else(|| script.elements.get("NetBIOS Computer Name")) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + if let Some(n) = name { + result.hostname = Some(n); + break; + } + // fallback: FQDN elem (strip will happen later in the handler) + if let Some(fqdn) = script.elements.get("FQDN") { + let fqdn = fqdn.trim().to_string(); + if !fqdn.is_empty() { + result.hostname = Some(fqdn); + break; + } + } + } + } + } + if result.hostname.is_none() { + 'ntlm: for port in &result.ports { + for script in &port.scripts { + if script.id == "ntlm-info" { + let name = script.elements.get("DNS_Computer_Name") + .or_else(|| script.elements.get("NetBIOS_Computer_Name")) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + if let Some(n) = name { + result.hostname = Some(n); + break 'ntlm; + } + } + } + } + } + // Fallback D: ssl-cert commonName — covers RDP/LDAPS/HTTPS certs with real FQDNs + if result.hostname.is_none() { + 'sslcert: for port in &result.ports { + for script in &port.scripts { + if script.id == "ssl-cert" { + if let Some(subject) = script.elements.get("Subject") { + for part in subject.split(',') { + let part = part.trim(); + if let Some(cn) = part.strip_prefix("commonName=") { + let cn = cn.trim(); + if !cn.is_empty() + && cn != "SSL_Self_Signed_Fallback" + && !cn.starts_with('*') + && cn.contains('.') + && cn.parse::().is_err() + { + result.hostname = Some(cn.to_string()); + break 'sslcert; + } + } + } + } + } + } + } + } + + // ---- ports -------------------------------------------------------------- + if let Some(ports_node) = host_node + .children() + .find(|n| n.tag_name().name() == "ports") + { + for port_node in ports_node.children() { + if port_node.tag_name().name() != "port" { + continue; + } + let portid = port_node.attribute("portid").unwrap_or("").to_string(); + let protocol = port_node.attribute("protocol").unwrap_or("tcp").to_string(); + + // State + let state_node = port_node + .children() + .find(|n| n.tag_name().name() == "state"); + let state = state_node + .and_then(|n| n.attribute("state")) + .unwrap_or("") + .to_string(); + + if state != "open" { + continue; + } + + // Service attributes + let svc_node = port_node + .children() + .find(|n| n.tag_name().name() == "service"); + let service_name = svc_node + .and_then(|n| n.attribute("name")) + .unwrap_or("") + .to_string(); + let product = svc_node + .and_then(|n| n.attribute("product")) + .unwrap_or("") + .to_string(); + let version = svc_node + .and_then(|n| n.attribute("version")) + .unwrap_or("") + .to_string(); + let extrainfo = svc_node + .and_then(|n| n.attribute("extrainfo")) + .unwrap_or("") + .to_string(); + + // CPEs + let cpe: Vec = port_node + .descendants() + .filter(|n| n.tag_name().name() == "cpe") + .filter_map(|n| n.text().map(|t| t.trim().to_string())) + .filter(|s| !s.is_empty()) + .collect(); + + // Scripts + let scripts: Vec = port_node + .children() + .filter(|n| n.tag_name().name() == "script") + .map(parse_script) + .collect(); + + result.ports.push(XmlPortInfo { + port: portid, + protocol, + state, + service_name, + product, + version, + extrainfo, + cpe, + scripts, + }); + } + } + + // Sort ports numerically + result + .ports + .sort_by_key(|p| p.port.parse::().unwrap_or(0)); + + // ---- OS ----------------------------------------------------------------- + if let Some(os_node) = host_node + .children() + .find(|n| n.tag_name().name() == "os") + { + let mut best_accuracy: u32 = 0; + for osmatch in os_node + .children() + .filter(|n| n.tag_name().name() == "osmatch") + { + let accuracy: u32 = osmatch + .attribute("accuracy") + .and_then(|a| a.parse().ok()) + .unwrap_or(0); + if accuracy > best_accuracy { + best_accuracy = accuracy; + result.os_name = osmatch + .attribute("name") + .map(|s| s.trim().to_string()); + result.os_accuracy = accuracy.min(100) as u8; + } + } + } + + // ---- host scripts ------------------------------------------------------- + if let Some(hsr) = host_node + .children() + .find(|n| n.tag_name().name() == "hostscript") + { + for script_node in hsr + .children() + .filter(|n| n.tag_name().name() == "script") + { + result.host_scripts.push(parse_script(script_node)); + } + } + + // ---- OS fallback A: smb-os-discovery "OS" elem -------------------------- + // Firewalled hosts often lack an osmatch element but smb-os-discovery emits an "OS" field. + if result.os_name.is_none() { + for script in &result.host_scripts { + if script.id == "smb-os-discovery" { + if let Some(os) = script.elements.get("OS") { + let os = os.trim().to_string(); + if !os.is_empty() { + result.os_name = Some(os); + break; + } + } + } + } + } + + // ---- OS fallback B: ntlm-info "OS" elem (per-port) ---------------------- + if result.os_name.is_none() { + 'os_search: for port in &result.ports { + for script in &port.scripts { + if script.id == "ntlm-info" { + if let Some(os) = script.elements.get("OS") { + let os = os.trim().to_string(); + if !os.is_empty() { + result.os_name = Some(os); + break 'os_search; + } + } + } + } + } + } + + // ---- AD domain extraction (priority order) ------------------------------ + // + // 1. smb-os-discovery elem[key="domain"] + // 2. smb-os-discovery elem[key="forest"] + // 3. Per-port ntlm-info elem[key="DNS_Domain_Name"] + // 4. ssl-cert subject/commonName — domain part + // 5. ldap-rootdse elem[key="defaultNamingContext"] — convert DC= format + + let mut domain: Option = None; + + // (1) + (2) from host-level smb-os-discovery + if domain.is_none() { + for script in &result.host_scripts { + if script.id == "smb-os-discovery" { + if let Some(d) = script.elements.get("domain") { + let d = d.trim().to_lowercase(); + if is_valid_domain(&d) { + domain = Some(d); + break; + } + } + if domain.is_none() { + if let Some(f) = script.elements.get("forest") { + let f = f.trim().to_lowercase(); + if is_valid_domain(&f) { + domain = Some(f); + } + } + } + } + } + } + + // (3) ntlm-info DNS_Domain_Name (per-port scripts) + if domain.is_none() { + 'outer: for port in &result.ports { + for script in &port.scripts { + if script.id == "ntlm-info" { + if let Some(d) = script.elements.get("DNS_Domain_Name") { + let d = d.trim().to_lowercase(); + if is_valid_domain(&d) { + domain = Some(d); + break 'outer; + } + } + } + } + } + } + + // (4) ssl-cert subject commonName → domain part + if domain.is_none() { + 'outer2: for port in &result.ports { + for script in &port.scripts { + if script.id == "ssl-cert" { + if let Some(cn) = script.elements.get("subject/commonName") { + if let Some(d) = domain_from_cn(cn) { + domain = Some(d); + break 'outer2; + } + } + } + } + } + } + + // (5) ldap-rootdse defaultNamingContext + if domain.is_none() { + 'outer3: for port in &result.ports { + for script in &port.scripts { + if script.id == "ldap-rootdse" { + if let Some(dn) = script.elements.get("defaultNamingContext") { + let converted = ldap_dn_to_domain(dn); + if is_valid_domain(&converted) { + domain = Some(converted); + break 'outer3; + } + } + } + } + } + } + + result.ad_domain = domain; + + Some(result) +} diff --git a/src/tag/refinement.rs b/src/tag/refinement.rs new file mode 100755 index 0000000..6eb298a --- /dev/null +++ b/src/tag/refinement.rs @@ -0,0 +1,371 @@ +// src/tag/refinement.rs +use std::collections::{HashSet, HashMap}; +use std::error::Error; +use crate::sql::ProjectContext; +use crate::utilities::tag_debug; +use crate::VERBOSE; + +// Context: Tag deduplication requires similarity measurement to identify near-duplicate tags +// Operation: Implements Jaro-Winkler algorithm to calculate similarity score between two strings with prefix weighting +pub fn jaro_winkler_similarity(s1: &str, s2: &str) -> f64 { + if s1 == s2 { + return 1.0; + } + + if s1.is_empty() || s2.is_empty() { + return 0.0; + } + + let jaro_sim = jaro_similarity(s1, s2); + if jaro_sim < 0.7 { + return jaro_sim; + } + + // Calculate common prefix (up to 4 characters) + let prefix_length = s1.chars() + .zip(s2.chars()) + .take(4) + .take_while(|(c1, c2)| c1 == c2) + .count(); + + // Jaro-Winkler formula with standard prefix weight of 0.1 + jaro_sim + (0.1 * prefix_length as f64 * (1.0 - jaro_sim)) +} + +// Context: Jaro-Winkler builds upon the basic Jaro similarity calculation for string comparison +// Operation: Calculates Jaro similarity based on matching characters and transpositions between two strings +pub fn jaro_similarity(s1: &str, s2: &str) -> f64 { + let s1_chars: Vec = s1.chars().collect(); + let s2_chars: Vec = s2.chars().collect(); + + let len1 = s1_chars.len(); + let len2 = s2_chars.len(); + + if len1 == 0 && len2 == 0 { + return 1.0; + } + + let match_distance = (std::cmp::max(len1, len2) / 2).saturating_sub(1); + + let mut s1_matches = vec![false; len1]; + let mut s2_matches = vec![false; len2]; + + let mut matches = 0; + + // Find matches + for i in 0..len1 { + let start = if i >= match_distance { i - match_distance } else { 0 }; + let end = std::cmp::min(i + match_distance + 1, len2); + + for j in start..end { + if s2_matches[j] || s1_chars[i] != s2_chars[j] { + continue; + } + s1_matches[i] = true; + s2_matches[j] = true; + matches += 1; + break; + } + } + + if matches == 0 { + return 0.0; + } + + // Count transpositions + let mut transpositions = 0; + let mut k = 0; + + for i in 0..len1 { + if !s1_matches[i] { + continue; + } + + while !s2_matches[k] { + k += 1; + } + + if s1_chars[i] != s2_chars[k] { + transpositions += 1; + } + k += 1; + } + + let jaro = (matches as f64 / len1 as f64 + + matches as f64 / len2 as f64 + + (matches as f64 - transpositions as f64 / 2.0) / matches as f64) / 3.0; + + jaro +} + +fn tag_contains_word(haystack: &str, needle: &str) -> bool { + haystack.split_whitespace().any(|word| word == needle) +} + +// Context: Tags may contain exact duplicates or substring relationships that should be consolidated +// Operation: Removes exact duplicates and consolidates substring-contained tags, preferring the shortest version +pub fn exact_deduplication(tags: &[String]) -> HashSet { + let mut deduplicated = HashSet::new(); + let mut processed = HashSet::new(); + + // Normalize tags for comparison (case-insensitive, trim whitespace) + let normalized_tags: Vec<(String, String)> = tags.iter() + .map(|tag| (tag.clone(), tag.trim().to_lowercase())) + .collect(); + + for (original_tag, normalized_tag) in &normalized_tags { + if processed.contains(normalized_tag) { + continue; + } + + // Check for substring containment + let mut is_substring_of_shorter = false; + let mut shortest_containing = original_tag.clone(); + + for (other_original, other_normalized) in &normalized_tags { + if normalized_tag == other_normalized { + continue; + } + + // Check if current tag is contained in another tag (word-boundary match only) + if tag_contains_word(other_normalized, normalized_tag) && other_original.len() < shortest_containing.len() { + shortest_containing = other_original.clone(); + is_substring_of_shorter = true; + } + // Check if another tag is contained in current tag (word-boundary match only) + else if tag_contains_word(normalized_tag, other_normalized) && original_tag.len() > other_original.len() { + is_substring_of_shorter = true; + shortest_containing = other_original.clone(); + } + } + + if !is_substring_of_shorter { + deduplicated.insert(original_tag.clone()); + processed.insert(normalized_tag.clone()); + } else { + deduplicated.insert(shortest_containing); + processed.insert(normalized_tag.clone()); + } + } + + deduplicated +} + +// Context: Similar but not identical tags need consolidation using fuzzy string matching techniques +// Operation: Groups similar tags using Jaro-Winkler similarity thresholds and selects the shortest representative +pub fn jaro_winkler_deduplication(tags: &[String], high_threshold: f64, moderate_threshold: f64) -> HashSet { + let mut deduplicated = HashSet::new(); + let mut processed = HashSet::new(); + + for tag1 in tags { + if processed.contains(tag1) { + continue; + } + + let mut best_match = tag1.clone(); + let mut highest_similarity = 0.0; + + for tag2 in tags { + if tag1 == tag2 || processed.contains(tag2) { + continue; + } + + let similarity = jaro_winkler_similarity(tag1, tag2); + + // High threshold tier (≥0.90) + if similarity >= high_threshold { + if similarity > highest_similarity { + highest_similarity = similarity; + // Shortest wins + best_match = if tag2.len() < best_match.len() { tag2.clone() } else { best_match }; + } + } + // Moderate threshold tier (≥0.85) + else if similarity >= moderate_threshold { + if similarity > highest_similarity { + highest_similarity = similarity; + // Shortest wins + best_match = if tag2.len() < best_match.len() { tag2.clone() } else { best_match }; + } + } + } + + // Mark all similar tags as processed + if highest_similarity >= moderate_threshold { + for tag2 in tags { + let similarity = jaro_winkler_similarity(&best_match, tag2); + if similarity >= moderate_threshold { + processed.insert(tag2.clone()); + } + } + } else { + processed.insert(tag1.clone()); + } + + deduplicated.insert(best_match); + } + + deduplicated +} + +// Context: Pentesting contexts have known service variants that should be consolidated to canonical forms +// Operation: Applies predefined rules to consolidate service variants into canonical pentesting-relevant tags +pub fn apply_hardcoded_rules(tags: &[String]) -> HashSet { + // Define consolidation rules + let windows_rules: HashMap<&str, Vec<&str>> = [ + ("smb", vec!["smb", "microsoft windows netbios-ssn", "netbios-ssn", "cifs", "netbios"]), + ("rdp", vec!["rdp", "microsoft terminal services", "ms-wbt-server", "terminal services"]), + ("winrm", vec!["winrm", "microsoft httpapi httpd", "ms-httpapi"]), + ("dns", vec!["dns", "microsoft dns", "simple dns plus", "domain name system"]), + ("kerberos", vec!["kerberos", "microsoft windows kerberos", "kerberos-sec", "krb5"]), + ("ldap", vec!["ldap", "microsoft windows active directory", "active directory", "ldaps", "microsoft windows active"]), + ("iis", vec!["iis", "microsoft iis httpd"]), + ("rpc", vec!["rpc", "microsoft windows rpc", "msrpc"]), + ("mssql", vec!["mssql", "ms-sql-s", "microsoft sql server", "sql server"]), + ].iter().cloned().collect(); + + let linux_rules: HashMap<&str, Vec<&str>> = [ + ("ssh", vec!["ssh", "openssh", "dropbear sshd"]), + ("apache", vec!["apache", "apache httpd", "httpd"]), + ("tomcat", vec!["tomcat", "apache tomcat", "tomcat server"]), + ("nginx", vec!["nginx", "nginx httpd"]), + ("mysql", vec!["mysql", "mysql server"]), + ("mariadb", vec!["mariadb", "mariadb server"]), + ("postgresql", vec!["postgresql", "postgres", "pgsql"]), + ("smb", vec!["smb", "samba", "samba smbd"]), + ].iter().cloned().collect(); + + // Combine all rules + let mut all_rules = windows_rules; + all_rules.extend(linux_rules); + + // Apply consolidation rules + apply_consolidation_rules(tags, &all_rules) +} + +// Context: Generic rule application mechanism for consolidating tag variants according to defined mappings +// Operation: Matches tags against consolidation rules and replaces variants with their canonical forms +pub fn apply_consolidation_rules(tags: &[String], rules: &HashMap<&str, Vec<&str>>) -> HashSet { + let mut result = HashSet::new(); + let mut processed = HashSet::new(); + + // Normalize tags for comparison + let tag_map: HashMap = tags.iter() + .map(|tag| (tag.to_lowercase(), tag.clone())) + .collect(); + + for tag in tags { + let tag_lower = tag.to_lowercase(); + + if processed.contains(&tag_lower) { + continue; + } + + let mut matched_rule = None; + + // Check if this tag matches any consolidation rule + for (canonical, variants) in rules { + if variants.iter().any(|variant| variant.to_lowercase() == tag_lower) { + matched_rule = Some(*canonical); + break; + } + } + + if let Some(canonical) = matched_rule { + // Mark all variants of this rule as processed + if let Some(variants) = rules.get(canonical) { + for variant in variants { + let variant_lower = variant.to_lowercase(); + if tag_map.contains_key(&variant_lower) { + processed.insert(variant_lower); + } + } + } + result.insert(canonical.to_string()); + } else { + // No rule matched, keep original tag + result.insert(tag.clone()); + processed.insert(tag_lower); + } + } + + result +} + +// Context: Complete tag deduplication requires multiple passes with different techniques for optimal results +// Operation: Applies multi-layer deduplication process (exact, similarity-based, rule-based) to clean tag lists +pub fn deduplicate_tags(tags: Vec) -> Vec { + if tags.is_empty() { + return tags; + } + + // Filter out uncertain services (those with ?) + let filtered_tags = filter_uncertain_services(&tags); + + // Layer 1: Hardcoded rules — must run BEFORE exact_deduplication. + // exact_deduplication uses substring containment, which would collapse + // "microsoft iis httpd" → "http" before the iis rule ever fires. + let after_rules = apply_hardcoded_rules(&filtered_tags); + let tags_vec: Vec = after_rules.into_iter().collect(); + + // Layer 2: Exact deduplication — operates on canonical names from Layer 1 + let after_exact = exact_deduplication(&tags_vec); + let tags_vec2: Vec = after_exact.into_iter().collect(); + + // Layer 3: Jaro-Winkler similarity (0.90 high, 0.85 moderate) + let after_similarity = jaro_winkler_deduplication(&tags_vec2, 0.90, 0.85); + + // Return sorted for consistent output + let mut final_tags: Vec = after_similarity.into_iter().collect(); + final_tags.sort(); + final_tags +} + +// Context: Automated tags need refinement to remove duplicates and consolidate similar tags for clean output +// Operation: Retrieves automated tags, applies full deduplication process, and stores refined results in database +pub async fn refine_automated_tags( + ctx: &ProjectContext, + ip: &str, +) -> Result<(), Box> { + // Get current automated tags + let automated_tags = ctx.get_automated_tags(ip).await?; + + if automated_tags.is_empty() { + return Ok(()); + } + + // Convert to Vec for deduplication + let tags_vec: Vec = automated_tags.into_iter().collect(); + + // Apply deduplication algorithm + let refined_tags_vec = deduplicate_tags(tags_vec); + + // Convert back to HashSet + let refined_tags: HashSet = refined_tags_vec.into_iter().collect(); + + // Update refined tags in database + ctx.set_refined_tags(ip, refined_tags.clone()).await?; + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Refined tags for {}: {}", ip, + refined_tags.iter().cloned().collect::>().join(","))); + } + + Ok(()) +} + +// Context: Some nmap services are marked as uncertain or unknown and should be filtered from final tag sets +// Operation: Filters out tags with uncertainty markers, unknown services, and other low-confidence identifiers +pub fn filter_uncertain_services(tags: &[String]) -> Vec { + tags.iter() + .filter(|tag| { + let tag_lower = tag.to_lowercase(); + // Keep tags that don't end with ? and aren't uncertain service patterns + !tag_lower.ends_with('?') && + !tag_lower.contains("unknown") && + !tag_lower.contains("tcpwrapped") && + !tag_lower.contains("filtered") + }) + .cloned() + .collect() +} \ No newline at end of file diff --git a/src/utilities/mod.rs b/src/utilities/mod.rs new file mode 100755 index 0000000..45733be --- /dev/null +++ b/src/utilities/mod.rs @@ -0,0 +1,3 @@ +pub mod outputs; + +pub use outputs::*; \ No newline at end of file diff --git a/src/utilities/outputs.rs b/src/utilities/outputs.rs new file mode 100755 index 0000000..b915d32 --- /dev/null +++ b/src/utilities/outputs.rs @@ -0,0 +1,160 @@ +// src/outputs.rs - ROBUST PATCHED: Critical message handling with colors + +use once_cell::sync::OnceCell; +use crate::sql::ProjectContext; +use crate::VERBOSE; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Global database context for logging (replaces TUI channel) +static DB_LOGGER: OnceCell = OnceCell::new(); + +/// Flag to enable dual output mode during startup (terminal + database) +static STARTUP_MODE: AtomicBool = AtomicBool::new(true); + +// Context: Setting up centralized logging system to database for persistent log storage +// Operation: Initializes global database logger context and disables startup mode +pub fn init_db_logger(ctx: ProjectContext) { + DB_LOGGER + .set(ctx) + .expect("init_db_logger called more than once"); + + // Disable startup mode once database logging is available + STARTUP_MODE.store(false, Ordering::SeqCst); +} + +// Context: Providing color-coded terminal output for better log visibility and readability +// Operation: Adds ANSI color codes to log messages based on their level for terminal display +fn format_terminal_output(line: &str) -> String { + let (level, message) = extract_log_level(line); + + match level { + "INFO" => format!("\x1b[32m[INFO]\x1b[0m {}", message), // Green + "STEP" => format!("\x1b[36m[STEP]\x1b[0m {}", message), // Cyan + "ACTION" => format!("\x1b[35m[ACTION]\x1b[0m {}", message), // Magenta + "ENUM" => format!("\x1b[34m[ENUM]\x1b[0m {}", message), // Blue + "CMD" => format!("\x1b[33m[CMD]\x1b[0m {}", message), // Yellow + "DEBUG" => format!("\x1b[37m[DEBUG]\x1b[0m {}", message), // Gray + "STARTUP" => format!("\x1b[92m[STARTUP]\x1b[0m {}", message), // Bright Green + "INIT" => format!("\x1b[96m[INIT]\x1b[0m {}", message), // Bright Cyan + "SUDO" => format!("\x1b[93m[SUDO]\x1b[0m {}", message), // Bright Yellow + "CRITICAL" => format!("\x1b[91m[CRITICAL]\x1b[0m {}", message), // Bright Red + _ => line.to_string(), // Keep original formatting for other levels + } +} + +// Context: Central log routing mechanism that handles both terminal and database output +// Operation: Writes log messages to terminal during startup and to database when available +fn log_line(line: String) { + // During startup mode, always print to terminal with colors + if STARTUP_MODE.load(Ordering::SeqCst) { + println!("{}", format_terminal_output(&line)); + } + + // If database logger is available, also log to database + if let Some(ctx) = DB_LOGGER.get() { + // Extract log level from message format like "[INFO] message" or "[DEBUG] message" + let (level, message) = extract_log_level(&line); + + // Write to database (non-blocking, ignore errors to avoid workflow disruption) + let ctx_clone = ctx.clone(); + let level_clone = level.to_string(); + let message_clone = message.to_string(); + + tokio::spawn(async move { + if let Err(_) = ctx_clone.insert_log(&level_clone, &message_clone).await { + // Silently ignore database logging errors to avoid breaking workflow + } + }); + + // In non-startup mode, only show messages in verbose mode + if !STARTUP_MODE.load(Ordering::SeqCst) && VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + println!("{}", format_terminal_output(&line)); + } + } +} + +// Context: Parsing log messages to separate level indicators from message content +// Operation: Extracts log level from bracketed format or defaults to INFO for plain messages +fn extract_log_level(line: &str) -> (&str, &str) { + let trimmed = line.trim(); + + // Match patterns like "[INFO]", "[DEBUG]", "[ENUM]", etc. + if trimmed.starts_with('[') { + if let Some(end_bracket) = trimmed.find(']') { + if end_bracket > 1 { + let level = &trimmed[1..end_bracket]; + let message = trimmed[end_bracket + 1..].trim(); + return (level, message); + } + } + } + + // Default to INFO level for untagged messages + ("INFO", trimmed) +} + +// Context: Ensuring critical startup information is always visible during application initialization +// Operation: Displays startup messages with bright green formatting and logs to database +pub fn print_startup(msg: &str) { + let formatted = format!("\x1b[92m[STARTUP]\x1b[0m {}", msg); // Bright green + println!("{}", formatted); + + // Also log to database if available (without color codes) + if let Some(ctx) = DB_LOGGER.get() { + let ctx_clone = ctx.clone(); + let msg_clone = msg.to_string(); + tokio::spawn(async move { + let _ = ctx_clone.insert_log("STARTUP", &msg_clone).await; + }); + } +} + +// Context: Handling high-priority messages that must be seen regardless of verbosity settings +// Operation: Displays critical messages with red formatting always to terminal and database +pub fn tag_critical(msg: &str) { + let line = format!("[CRITICAL] {}", msg); + + // Always print to terminal with colors, regardless of mode + println!("{}", format_terminal_output(&line)); + + // Also log to database if available + if let Some(ctx) = DB_LOGGER.get() { + let ctx_clone = ctx.clone(); + let msg_clone = msg.to_string(); + tokio::spawn(async move { + let _ = ctx_clone.insert_log("CRITICAL", &msg_clone).await; + }); + } +} + +// Context: Standard information logging for general application events and status updates +// Operation: Logs informational messages with INFO level through the central logging system +pub fn tag_info(msg: &str) { + log_line(format!("[INFO] {}", msg)); +} + +// Context: Conditional debug logging that respects verbosity settings for development troubleshooting +// Operation: Logs debug messages only when verbose mode is enabled to reduce output noise +pub fn tag_debug(msg: &str) { + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + log_line(format!("[DEBUG] {}", msg)); + } +} + +// Context: Tracking enumeration and discovery processes during scanning operations +// Operation: Logs enumeration steps with ENUM level to show scanning progress +pub fn print_enum(msg: &str) { + log_line(format!("[ENUM] {}", msg)); +} + +// Context: Documenting executed commands for audit trail and debugging purposes +// Operation: Logs command execution with CMD level to track what commands were run +pub fn print_cmd(msg: &str) { + log_line(format!("[CMD] {}", msg)); +} + +// Context: Reporting significant actions and operations performed by the application +// Operation: Logs action notifications with ACTION level to track major operations +pub fn print_action(msg: &str) { + log_line(format!("[ACTION] {}", msg)); +} \ No newline at end of file diff --git a/src/web/auth.rs b/src/web/auth.rs new file mode 100755 index 0000000..8d1a122 --- /dev/null +++ b/src/web/auth.rs @@ -0,0 +1,370 @@ +// src/web/auth.rs - Enhanced Authentication with Secure Session Management + +use askama::Template; +use axum::{ + extract::{Request, State}, + middleware::Next, + response::{Html, IntoResponse, Redirect, Response}, + http::{HeaderMap, header::{COOKIE, SET_COOKIE}}, + Form, +}; +use lazy_static; +use serde::{Deserialize, Serialize}; +use serde_json; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; +use rand::Rng; +use bcrypt::{hash, verify, DEFAULT_COST}; +use sqlx::Row; +use crate::sql::ProjectContext; + +/// Admin username constant +pub const ADMIN_USERNAME: &str = "admin"; + +/// User role enumeration +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum UserRole { + Admin, + Player, +} + +impl UserRole { + pub fn as_str(&self) -> &'static str { + match self { + UserRole::Admin => "admin", + UserRole::Player => "player", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "admin" => Some(UserRole::Admin), + "player" => Some(UserRole::Player), + _ => None, + } + } +} + +impl std::fmt::Display for UserRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +/// Authenticated user structure +#[derive(Clone, Debug)] +pub struct AuthenticatedUser { + pub username: String, + pub role: UserRole, + pub login_time: u64, +} + +/// Session cookie name +const SESSION_COOKIE: &str = "rofmad_session"; + +/// Session expiration time (24 hours in seconds) +const SESSION_EXPIRY: u64 = 24 * 60 * 60; + +// Active sessions storage with user data +lazy_static::lazy_static! { + static ref ACTIVE_SESSIONS: Mutex> = Mutex::new(HashMap::new()); +} + +/// Login form data +#[derive(Deserialize)] +pub struct LoginForm { + pub username: String, + pub password: String, +} + +/// Login page template +#[derive(Template)] +#[template(path = "login.html")] +pub struct LoginTemplate { + pub error: String, +} + +// Context: Provides secure session token generation for authenticated access +// Operation: Creates 64-character hexadecimal token using cryptographically secure random generator +fn generate_session_token() -> String { + let mut rng = rand::thread_rng(); + (0..64) + .map(|_| format!("{:02x}", rng.r#gen::())) + .collect() +} + +// Context: Provides time reference for session expiration tracking +// Operation: Returns current Unix timestamp in seconds for session management +fn current_timestamp() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +// Context: Generates secure random password for player accounts +// Operation: Creates 16-character password with mixed case, numbers, and symbols +pub fn generate_secure_password() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%&*"; + let mut rng = rand::thread_rng(); + (0..16) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() +} + +// Context: Secure password hashing for database storage +// Operation: Uses bcrypt with default cost for password hashing +pub fn hash_password(password: &str) -> Result> { + hash(password, DEFAULT_COST).map_err(|e| e.into()) +} + +// Context: Password verification for authentication +// Operation: Compares plaintext password with bcrypt hash +pub fn verify_password(password: &str, hash: &str) -> bool { + verify(password, hash).unwrap_or(false) +} + +// Context: Maintains session storage hygiene by removing expired entries +// Operation: Filters out sessions with timestamps older than current time +fn clean_expired_sessions() { + let current_time = current_timestamp(); + let mut sessions = ACTIVE_SESSIONS.lock().unwrap(); + sessions.retain(|_, user| (current_time - user.login_time) < SESSION_EXPIRY); +} + +// Context: Verifies user authentication status for protected routes +// Operation: Checks if session token exists and hasn't expired in active sessions +fn is_valid_session(token: &str) -> bool { + clean_expired_sessions(); + let sessions = ACTIVE_SESSIONS.lock().unwrap(); + sessions.contains_key(token) +} + +// Context: Retrieves authenticated user from session token +// Operation: Returns user data if session is valid, None otherwise +pub fn get_session_user(token: &str) -> Option { + clean_expired_sessions(); + let sessions = ACTIVE_SESSIONS.lock().unwrap(); + sessions.get(token).cloned() +} + +// Context: Checks if user has admin role privileges +// Operation: Validates session and returns true if user is admin +pub fn is_admin_session(token: &str) -> bool { + if let Some(user) = get_session_user(token) { + user.role == UserRole::Admin + } else { + false + } +} + +// Context: Gets current username from session for audit logging +// Operation: Returns username if session is valid, "unknown" otherwise +pub fn get_current_username(token: &str) -> String { + if let Some(user) = get_session_user(token) { + user.username.clone() + } else { + "unknown".to_string() + } +} + +// Context: Establishes new user session after successful authentication +// Operation: Generates token and stores it with user data in session map +fn create_session(username: String, role: UserRole) -> String { + clean_expired_sessions(); + let token = generate_session_token(); + let user = AuthenticatedUser { + username, + role, + login_time: current_timestamp(), + }; + let mut sessions = ACTIVE_SESSIONS.lock().unwrap(); + sessions.insert(token.clone(), user); + token +} + + +// Context: Provides login interface for unauthenticated users +// Operation: Renders login form template with empty error message +pub async fn login_page() -> impl IntoResponse { + let template = LoginTemplate { error: String::new() }; + Html(template.render().unwrap_or_else(|_| "Template error".to_string())) +} + +// Context: Processes user login attempts and manages session creation +// Operation: Validates username/password and creates session cookie or returns error +pub async fn login_submit(State(ctx): State, Form(form): Form) -> impl IntoResponse { + let username = form.username.trim(); + let password = form.password.trim(); + + // Check admin login — verified via bcrypt hash stored in DB + if username == ADMIN_USERNAME { + if ctx.verify_admin_password(password).await.unwrap_or(false) { + let session_token = create_session(ADMIN_USERNAME.to_string(), UserRole::Admin); + let cookie = format!("{}={}; Path=/; HttpOnly; SameSite=Strict; Max-Age={}", + SESSION_COOKIE, session_token, SESSION_EXPIRY); + let mut response = Redirect::to("/").into_response(); + if let Ok(header_value) = cookie.parse() { + response.headers_mut().insert(SET_COOKIE, header_value); + } + return response; + } + } + + // Check player login from database + if let Ok(player_data) = sqlx::query("SELECT password_hash FROM users WHERE username = ? AND role = ? AND enabled = TRUE") + .bind(username) + .bind(UserRole::Player.as_str()) + .fetch_optional(&ctx.pool) + .await + { + if let Some(row) = player_data { + let password_hash: String = row.get("password_hash"); + if verify_password(password, &password_hash) { + // Successful player login + let session_token = create_session(username.to_string(), UserRole::Player); + let cookie = format!("{}={}; Path=/; HttpOnly; SameSite=Strict; Max-Age={}", + SESSION_COOKIE, session_token, SESSION_EXPIRY); + + let mut response = Redirect::to("/").into_response(); + if let Ok(header_value) = cookie.parse() { + response.headers_mut().insert(SET_COOKIE, header_value); + } + return response; + } + } + } + + // Failed login - show login form with error + let template = LoginTemplate { + error: "Invalid username or password. Please try again.".to_string() + }; + Html(template.render().unwrap_or_else(|_| "Template error".to_string())).into_response() +} + +// Context: Admin-only authentication middleware for command execution routes +// Operation: Validates session and ensures user has admin role before allowing access +pub async fn admin_required_middleware( + State(ctx): State, + headers: HeaderMap, + request: Request, + next: Next, +) -> Response { + // Redirect to setup if admin account has not been created yet + if !ctx.admin_exists().await.unwrap_or(true) { + return Redirect::to("/setup").into_response(); + } + + if let Some(cookie_header) = headers.get(COOKIE) { + if let Ok(cookie_str) = cookie_header.to_str() { + for cookie in cookie_str.split(';') { + let cookie = cookie.trim(); + if let Some(value) = cookie.strip_prefix(&format!("{}=", SESSION_COOKIE)) { + if is_admin_session(value) { + // Admin authenticated - allow request to proceed + return next.run(request).await; + } + } + } + } + } + + // Check if this is an API request - be more comprehensive + let path = request.uri().path(); + let method = request.method(); + let is_api_request = + // Explicit API paths + path.starts_with("/api/") || + // Form submission endpoints + path.starts_with("/create-") || path.starts_with("/manage-") || + path.starts_with("/add-") || path.starts_with("/remove-") || + // Action endpoints + path.starts_with("/actions/") || path.starts_with("/launch-") || + path.starts_with("/run-") || + // Host management endpoints + path.starts_with("/host/") || + // Most POST requests expect JSON responses + method == "POST" || + // Client explicitly requests JSON + headers.get("accept") + .and_then(|h| h.to_str().ok()) + .map_or(false, |s| s.contains("application/json")); + + if is_api_request { + // Return JSON error for API requests + let json_response = serde_json::json!({ + "success": false, + "message": "Admin privileges required" + }); + return axum::response::Json(json_response).into_response(); + } + + // Redirect to login for regular page requests + Redirect::to("/login").into_response() +} + +// Context: Basic authentication middleware for general protected routes +// Operation: Validates session for any authenticated user (admin or player) +pub async fn authenticated_middleware( + State(ctx): State, + headers: HeaderMap, + request: Request, + next: Next, +) -> Response { + // Redirect to setup if admin account has not been created yet + if !ctx.admin_exists().await.unwrap_or(true) { + return Redirect::to("/setup").into_response(); + } + + // Check authentication for all protected routes + if let Some(cookie_header) = headers.get(COOKIE) { + if let Ok(cookie_str) = cookie_header.to_str() { + for cookie in cookie_str.split(';') { + let cookie = cookie.trim(); + if let Some(value) = cookie.strip_prefix(&format!("{}=", SESSION_COOKIE)) { + if is_valid_session(value) { + // Authenticated - allow request to proceed + return next.run(request).await; + } + } + } + } + } + + // Check if this is an API request - be more comprehensive + let path = request.uri().path(); + let method = request.method(); + let is_api_request = + // Explicit API paths + path.starts_with("/api/") || + // Form submission endpoints + path.starts_with("/create-") || path.starts_with("/manage-") || + path.starts_with("/add-") || path.starts_with("/remove-") || + // Action endpoints + path.starts_with("/actions/") || path.starts_with("/launch-") || + path.starts_with("/run-") || + // Host management endpoints + path.starts_with("/host/") || + // Most POST requests expect JSON responses + method == "POST" || + // Client explicitly requests JSON + headers.get("accept") + .and_then(|h| h.to_str().ok()) + .map_or(false, |s| s.contains("application/json")); + + if is_api_request { + // Return JSON error for API requests + let json_response = serde_json::json!({ + "success": false, + "message": "Authentication required" + }); + return axum::response::Json(json_response).into_response(); + } + + // Redirect to login for regular page requests + Redirect::to("/login").into_response() +} \ No newline at end of file diff --git a/src/web/handlers/actions.rs b/src/web/handlers/actions.rs new file mode 100755 index 0000000..6c2c888 --- /dev/null +++ b/src/web/handlers/actions.rs @@ -0,0 +1,197 @@ +// src/web/handlers/actions.rs - Action builder handlers + +use askama::Template; +use axum::{ + extract::State, + response::{Html, IntoResponse}, + Json, +}; +use serde::{Deserialize, Serialize}; +use sqlx::Row; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError}; +use crate::workflow::{SpecificActionStep, Condition, TagPattern}; +use crate::sql::hosts::HostSnapshot; +use crate::utilities::outputs::tag_info; + +// Template struct for action builder page +#[derive(Template)] +#[template(path = "action_builder.html")] +pub struct ActionBuilderTemplate {} + +#[derive(Deserialize)] +pub struct ActionExecuteForm { + pub name: String, + pub command: String, + pub timeout_s: Option, + pub condition: Option, + pub tags: Option>, + pub save_to_workflow: Option, +} + +#[derive(Deserialize)] +pub struct ActionPreviewForm { + pub condition: Option, +} + +#[derive(Serialize)] +pub struct PreviewHost { + pub ip: String, + pub hostname: String, + pub tags: Vec, +} + +// Context: Provides interface for creating custom automation actions +// Operation: Renders action builder form template for workflow management +pub async fn action_builder_page() -> WebResult { + let template = ActionBuilderTemplate {}; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Shows preview of hosts that match specified action conditions +// Operation: Evaluates conditions against all hosts and returns matching list +pub async fn action_preview( + State(ctx): State, + Json(form): Json +) -> WebResult { + let mut matching_hosts = Vec::new(); + + // Get all host state in one query — avoids per-host DB calls for condition evaluation + let rows = sqlx::query( + "SELECT ip, + COALESCE(hostname, '') as hostname, + COALESCE(ports, '') as ports, + COALESCE(manual_tags, '') as manual_tags, + COALESCE(refined_tags, '') as refined_tags, + COALESCE(automated_tags, '') as automated_tags, + COALESCE(action_tags, '') as action_tags, + COALESCE(full_scan_complete, 0) as full_scan_complete, + COALESCE(service_scan_complete, 0) as service_scan_complete, + COALESCE(http_discovery_complete, 0) as http_discovery_complete, + COALESCE(port_services, '{}') as port_services + FROM targets WHERE run_id = ? ORDER BY ip" + ) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + // Fetch IPs that have HTTP services in one query + let http_ips: std::collections::HashSet = sqlx::query( + "SELECT DISTINCT ip FROM http_services WHERE run_id = ?" + ) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))? + .into_iter() + .map(|r| r.get::("ip")) + .collect(); + + for row in rows { + let ip: String = row.get("ip"); + let hostname: String = row.get("hostname"); + let ports_str: String = row.get("ports"); + let manual_tags: String = row.get("manual_tags"); + let refined_tags: String = row.get("refined_tags"); + let automated_tags: String = row.get("automated_tags"); + + let ports: Vec = ports_str.split(',').map(str::trim) + .filter(|s| !s.is_empty()).map(String::from).collect(); + + let action_tags_str: String = row.try_get("action_tags").unwrap_or_default(); + let mut tag_set = std::collections::HashSet::new(); + for t in action_tags_str.split(',').chain(manual_tags.split(',')).chain(refined_tags.split(',')).chain(automated_tags.split(',')) { + let t = t.trim(); + if !t.is_empty() { tag_set.insert(t.to_string()); } + } + let tag_vec: Vec = tag_set.into_iter().collect(); + + let snapshot = HostSnapshot { + ip: ip.clone(), + ports: ports_str, + automated_tags, + manual_tags, + refined_tags, + action_tags: row.try_get("action_tags").unwrap_or_default(), + full_scan_complete: row.get::("full_scan_complete"), + service_scan_complete: row.get::("service_scan_complete"), + http_discovery_complete: row.get::("http_discovery_complete"), + has_http_services: http_ips.contains(&ip), + port_services: row.try_get("port_services").unwrap_or_else(|_| "{}".to_string()), + http_service_list: vec![], + }; + + let matches = if let Some(condition) = &form.condition { + crate::workflow::evaluate_condition(condition, &ports, &tag_vec, &snapshot) + } else { + true // No condition means all hosts match + }; + + if matches { + matching_hosts.push(PreviewHost { + ip, + hostname, + tags: tag_vec, + }); + } + } + + Ok(axum::Json(serde_json::json!({ + "hosts": matching_hosts + }))) +} + +// Context: Executes custom actions against hosts and optionally saves to workflow +// Operation: Creates action steps, saves to workflow file if requested, and queues for execution +pub async fn action_execute( + State(ctx): State, + Json(form): Json +) -> WebResult { + let should_save = form.save_to_workflow.unwrap_or(false); + + // Create the SpecificActionStep + let action = SpecificActionStep { + name: form.name.clone(), + condition: form.condition.clone(), + command: form.command.clone(), + timeout_s: form.timeout_s, + tags: form.tags.clone(), + }; + + // Use new workflow function for YAML persistence - SAME LOGIC, NO DUPLICATION + if should_save { + let workflow_path = sqlx::query_scalar::<_, String>("SELECT workflow_path FROM settings WHERE run_id = ?") + .bind(&ctx.run_id) + .fetch_optional(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))? + .ok_or_else(|| WebError::NotFound("No workflow path configured for this run".to_string()))?; + + crate::workflow::save_action_to_workflow(&workflow_path, &action) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + tag_info(&format!("Action '{}' saved to workflow file", form.name)); + } + + // Use existing workflow runtime actions - NO DIRECT MUTEX ACCESS + { + let mut runtime_actions = crate::workflow::RUNTIME_ACTIONS.lock().unwrap(); + runtime_actions.push(action); + } + + tag_info(&format!("Runtime action '{}' added to workflow", form.name)); + + let message = if should_save { + format!("Action '{}' executed and saved to workflow file", form.name) + } else { + format!("Action '{}' queued for execution", form.name) + }; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": message + }))) +} + diff --git a/src/web/handlers/api.rs b/src/web/handlers/api.rs new file mode 100755 index 0000000..da3887d --- /dev/null +++ b/src/web/handlers/api.rs @@ -0,0 +1,230 @@ +// src/web/handlers/api.rs - API endpoints + +use axum::{ + extract::{Path, State}, + response::IntoResponse, +}; +use std::sync::atomic::Ordering; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError, convert_task_info, WebTaskInfo}; +use crate::workflow::{ACTIVE_TASKS, QUEUED_TASKS, QUEUED_TASK_INFOS}; + +// Context: Provides real-time task queue status for web interface updates +// Operation: Returns JSON with current active and queued task counts +pub async fn api_task_status() -> impl IntoResponse { + let active_count = ACTIVE_TASKS.lock().unwrap().len(); + let queued_count = QUEUED_TASKS.load(Ordering::SeqCst); + + axum::Json(serde_json::json!({ + "active_count": active_count, + "queued_count": queued_count + })) +} + +// Context: Provides list of hosts that have completed scanning and are ready for analysis +// Operation: Queries database for hosts with scan data but no analysis status +pub async fn api_hosts_ready_analysis(State(ctx): State) -> WebResult { + let hosts = ctx.get_hosts_ready_for_analysis().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "hosts": hosts + }))) +} + +// Context: Provides list of hosts that have nuclei scan results for dashboard live update +// Operation: Queries action_outputs for nuclei_% entries and returns matching host summaries +pub async fn api_nuclei_findings(State(ctx): State) -> WebResult { + let hosts = ctx.get_hosts_with_nuclei_findings().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + Ok(axum::Json(serde_json::json!({ "hosts": hosts }))) +} + +// Context: Provides current concurrency limits and usage statistics +// Operation: Returns JSON with workflow execution concurrency information +pub async fn api_concurrency_status() -> impl IntoResponse { + let concurrency_stats = crate::workflow::concurrency::get_concurrency_stats(); + axum::Json(concurrency_stats) +} + +// Context: Provides real-time discovery statistics for dashboard updates +// Operation: Returns JSON with current scanning progress metrics +pub async fn api_discovery_stats(State(ctx): State) -> WebResult { + let stats = ctx.get_dashboard_statistics().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "total_hosts": stats.total_hosts, + "hosts_with_open_ports": stats.hosts_with_open_ports, + "total_open_ports": stats.total_open_ports, + "hosts_with_http_services": stats.hosts_with_http_services, + "total_screenshots": stats.total_screenshots + }))) +} + +// Context: Provides task termination capability for workflow management +// Operation: Cancels active or queued task by ID and updates task counters +pub async fn kill_task( + Path(id): Path, +) -> WebResult { + // Check active tasks first + { + let mut active_tasks = crate::workflow::ACTIVE_TASKS.lock().unwrap(); + if let Some(task) = active_tasks.remove(&id) { + task.cancel_token.cancel(); + return Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Active task '{}' killed", task.name) + }))); + } + } + + // Check queued tasks + { + let mut queued_tasks = crate::workflow::QUEUED_TASK_INFOS.lock().unwrap(); + if let Some(task) = queued_tasks.remove(&id) { + // Only cancel — spawning.rs cancel handler decrements QUEUED_TASKS and calls + // unregister_queued_task. Decrementing here too would double-count. + task.cancel_token.cancel(); + return Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Queued task '{}' killed", task.name) + }))); + } + } + + Err(WebError::NotFound("Task not found".to_string())) +} + +// Context: Provides current active task information for real-time monitoring +// Operation: Returns JSON list of currently executing workflow tasks +pub async fn api_active_tasks() -> impl IntoResponse { + let active_tasks_raw: Vec<_> = ACTIVE_TASKS.lock().unwrap().values().cloned().collect(); + let tasks: Vec = active_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + axum::Json(serde_json::json!({ + "tasks": tasks + })) +} + +// Context: Provides comprehensive task queue status for monitoring interface +// Operation: Returns JSON with both active and queued task lists +pub async fn api_queue_tasks() -> impl IntoResponse { + let active_tasks_raw: Vec<_> = ACTIVE_TASKS.lock().unwrap().values().cloned().collect(); + let queued_tasks_raw: Vec<_> = QUEUED_TASK_INFOS.lock().unwrap().values().cloned().collect(); + + let active_tasks: Vec = active_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + let queued_tasks: Vec = queued_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + axum::Json(serde_json::json!({ + "active_tasks": active_tasks, + "queued_tasks": queued_tasks + })) +} + +// Context: Provides pending task information for queue management +// Operation: Returns JSON list of tasks waiting for execution +pub async fn api_queued_tasks() -> impl IntoResponse { + let queued_tasks_raw: Vec<_> = QUEUED_TASK_INFOS.lock().unwrap().values().cloned().collect(); + let tasks: Vec = queued_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + axum::Json(serde_json::json!({ + "tasks": tasks + })) +} +// Context: Manual HTTP probe retry for a specific host:port +// Operation: Spawns a visible task (no semaphore) that re-probes with curl and takes a +// screenshot on success. Returns immediately with a task_id the caller can poll. +pub async fn retry_http_probe( + Path((ip, port)): Path<(String, String)>, + State(ctx): State, +) -> WebResult { + use crate::http::{probe_with_curl, effective_probe_order}; + use crate::workflow::tasks::{register_task, finalize_task_output}; + use tokio_util::sync::CancellationToken; + + let cancel_token = CancellationToken::new(); + let (task_info, _, collector_handle) = register_task( + &format!("http_retry_probe:{}:{}", ip, port), + &ip, + cancel_token, + ); + let task_id = task_info.id.clone(); + + // Spawn without acquiring any semaphore — user-initiated, must not queue behind + // automated scan workload. Visible in the queue monitor via ACTIVE_TASKS. + tokio::spawn(async move { + let raw_tx = task_info.raw_tx.clone(); + let _ = raw_tx.send(format!("Retry probe starting on {}:{}", ip, port)); + + let protocols = effective_probe_order(&port, ""); + let mut found: Option<(String, crate::http::HttpServiceInfo)> = None; + + for protocol in &protocols { + let _ = raw_tx.send(format!("Trying {}://{}:{}/", protocol, ip, port)); + match probe_with_curl(&ip, &port, protocol, Some(&raw_tx)).await { + Ok(info) => { found = Some((protocol.clone(), info)); break; } + Err((_, msg)) => { + let _ = raw_tx.send(format!(" {} failed: {}", protocol, msg)); + } + } + } + + match found { + None => { + let _ = raw_tx.send("All protocols failed".to_string()); + } + Some((_protocol, service_info)) => { + if let Ok(json) = serde_json::to_string(&service_info) { + ctx.insert_http_service(&ip, &port, &json).await.ok(); + let _ = raw_tx.send(format!( + "Service stored: {} {} ({}ms)", + service_info.status_code, + service_info.server_header, + service_info.response_time_ms + )); + } + // Use the same sequential screenshot function as the automated phase (no semaphore) + if let Ok(output_dir) = crate::config::paths::ensure_screenshots_directory_for_run(&ctx.db_path, &ctx.run_id) { + let _ = raw_tx.send("Taking screenshot (sequential, 30s limit)...".to_string()); + let results = crate::http::screenshot_http_services(&ctx, &ip, &output_dir, Some(raw_tx.clone())) + .await + .unwrap_or_default(); + for result in &results { + if result.success && !result.file_path.is_empty() { + if let Some(p) = crate::workflow::screenshots::extract_port_from_url(&result.url) { + if let Ok(mut services) = ctx.get_http_services(&ip).await { + if let Some(svc_json) = services.get_mut(&p) { + if let Ok(mut info) = serde_json::from_str::(svc_json) { + info.screenshot_path = Some(result.file_path.clone()); + if let Ok(updated) = serde_json::to_string(&info) { + ctx.insert_http_service(&ip, &p, &updated).await.ok(); + } + } + } + } + } + } + } + } + } + } + + drop(raw_tx); + finalize_task_output(task_info, collector_handle, "completed").await; + }); + + Ok(axum::Json(serde_json::json!({ + "task_id": task_id, + "started": true + }))) +} diff --git a/src/web/handlers/dashboard.rs b/src/web/handlers/dashboard.rs new file mode 100755 index 0000000..3826800 --- /dev/null +++ b/src/web/handlers/dashboard.rs @@ -0,0 +1,80 @@ +// src/web/handlers/dashboard.rs - Dashboard page handlers + +use askama::Template; +use axum::{ + extract::{State}, + response::{Html, IntoResponse}, +}; +use serde::Serialize; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError, HostSummary, convert_task_info, WebTaskInfo}; +use crate::workflow::{ACTIVE_TASKS, QUEUED_TASKS}; +use std::sync::atomic::Ordering; + +#[derive(Serialize)] +pub struct DashboardStats { + pub total_hosts: usize, + pub total_subnets: usize, + pub active_tasks: Vec, + pub queued_tasks_count: usize, + pub hosts_with_open_ports: usize, + pub total_open_ports: usize, + pub hosts_with_http_services: usize, + pub total_screenshots: usize, + + // Ready for analysis + pub hosts_ready_for_analysis: Vec, + pub concurrency_stats: crate::workflow::concurrency::ConcurrencyStats, + + // Nuclei findings + pub hosts_with_nuclei_findings: Vec, +} + +#[derive(Template)] +#[template(path = "dashboard.html")] +pub struct DashboardTemplate { + pub stats: DashboardStats, +} + +// Context: Provides main dashboard view with comprehensive system statistics +// Operation: Aggregates host, task, and workflow data into dashboard template +pub async fn dashboard(State(ctx): State) -> WebResult { + // Use new core function for all statistics + let core_stats = ctx.get_dashboard_statistics().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + // Get live task data and convert to web format + let active_tasks_raw: Vec<_> = ACTIVE_TASKS.lock().unwrap().values().cloned().collect(); + let active_tasks: Vec = active_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + // Get hosts ready for analysis + let hosts_ready_for_analysis = ctx.get_hosts_ready_for_analysis().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let queued_count = QUEUED_TASKS.load(Ordering::SeqCst); + + let concurrency_stats = crate::workflow::concurrency::get_concurrency_stats(); + + let hosts_with_nuclei_findings = ctx.get_hosts_with_nuclei_findings().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + // Combine core statistics with task/workflow data + let stats = DashboardStats { + total_hosts: core_stats.total_hosts, + total_subnets: core_stats.total_subnets, + active_tasks, + queued_tasks_count: queued_count, + hosts_with_open_ports: core_stats.hosts_with_open_ports, + total_open_ports: core_stats.total_open_ports, + hosts_with_http_services: core_stats.hosts_with_http_services, + total_screenshots: core_stats.total_screenshots, + hosts_ready_for_analysis, + concurrency_stats, + hosts_with_nuclei_findings, + }; + + let template = DashboardTemplate { stats }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} \ No newline at end of file diff --git a/src/web/handlers/forms.rs b/src/web/handlers/forms.rs new file mode 100755 index 0000000..9d937ac --- /dev/null +++ b/src/web/handlers/forms.rs @@ -0,0 +1,874 @@ +// src/web/handlers/forms.rs - Form processing handlers + +use axum::{ + extract::{State, Form, Query, Multipart}, + response::IntoResponse, +}; +use axum::http::HeaderValue; +use serde::Deserialize; +use sqlx::Row; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError}; +use crate::utilities::outputs::tag_debug; + +#[derive(Deserialize)] +pub struct AddTargetForm { + pub target: String, +} + +#[derive(Deserialize)] +pub struct AddSubnetForm { + pub subnet: String, +} + +#[derive(Deserialize)] +pub struct RunCommandForm { + pub ip: String, + pub command: String, + pub timeout: Option, +} + +#[derive(Deserialize)] +pub struct ExportTagsForm { + pub tags: String, + pub format: String, // "txt", "csv", "json" +} + +#[derive(Deserialize)] +pub struct NucleiBulkForm { + pub include_tags: String, // comma-separated, may be empty + pub exclude_tags: String, // comma-separated, may be empty + #[serde(default)] + pub include_ips: String, // comma-separated IPs/CIDRs, may be empty + #[serde(default)] + pub exclude_ips: String, // comma-separated IPs/CIDRs, may be empty +} + +#[derive(Deserialize)] +pub struct NucleiBulkQuery { + #[serde(default)] + pub include_tags: String, + #[serde(default)] + pub exclude_tags: String, + #[serde(default)] + pub include_ips: String, + #[serde(default)] + pub exclude_ips: String, +} + +// Context: Shared comma-separated tag list parsing for bulk tag-filtered actions +fn parse_tag_list(raw: &str) -> Vec { + raw.split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + +// Context: Shared comma-separated IP/CIDR list parsing for bulk nuclei host filters +// Operation: Splits/trims/filters empty entries, then parses each as an IpNetwork, +// returning the first invalid entry as an error +fn parse_ip_list(raw: &str) -> Result, String> { + raw.split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .map(|s| s.parse::().map_err(|_| s)) + .collect() +} + +#[derive(Deserialize)] +pub struct AddBlacklistForm { + pub target: String, +} + +#[derive(Deserialize)] +pub struct RemoveBlacklistForm { + pub target: String, +} + +#[derive(Deserialize)] +pub struct ResetHostForm { + pub ip: String, +} + +#[derive(Deserialize)] +pub struct RetagHostForm { + pub ip: String, +} + +#[derive(Deserialize)] +pub struct RetakeScreenshotForm { + pub ip: String, + pub port: String, + pub protocol: String, +} + +#[derive(Deserialize)] +pub struct NucleiScanForm { + pub ip: String, + pub port: String, + pub protocol: String, +} + +// Context: Allows adding single IP addresses or hostnames to scanning targets +// Operation: Validates and resolves input, checks blacklist, inserts into database and triggers scan +pub async fn add_individual_target( + State(ctx): State, + Form(form): Form +) -> WebResult { + let target = form.target.trim(); + + // Validate input (IP or hostname) + if target.is_empty() { + return Err(WebError::InvalidData("Target cannot be empty".to_string())); + } + + // Try to parse as IP first + if let Ok(ip) = target.parse::() { + let ip_str = ip.to_string(); + + // Check if already blacklisted + if ctx.is_blacklisted(&ip_str).await.map_err(|e| WebError::DatabaseError(e.to_string()))? { + return Err(WebError::InvalidData("Target is blacklisted".to_string())); + } + + // Insert IP and trigger workflow + if ctx.insert_ip(&ip_str).await.map_err(|e| WebError::DatabaseError(e.to_string()))? { + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Target {} added successfully - scanning will begin shortly", ip_str), + "target": ip_str + }))) + } else { + Err(WebError::InvalidData("Target already exists".to_string())) + } + } else { + // Try as hostname - resolve to IP first + match tokio::net::lookup_host(format!("{}:80", target)).await { + Ok(mut addrs) => { + if let Some(addr) = addrs.next() { + let ip_str = addr.ip().to_string(); + + if ctx.is_blacklisted(&ip_str).await.map_err(|e| WebError::DatabaseError(e.to_string()))? { + return Err(WebError::InvalidData("Resolved IP is blacklisted".to_string())); + } + + if ctx.insert_ip(&ip_str).await.map_err(|e| WebError::DatabaseError(e.to_string()))? { + // Also store the hostname + let _ = ctx.insert_hostname(&ip_str, target).await; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Target {} ({}) added successfully - scanning will begin shortly", target, ip_str), + "target": ip_str, + "hostname": target + }))) + } else { + Err(WebError::InvalidData("Target already exists".to_string())) + } + } else { + Err(WebError::InvalidData("Could not resolve hostname".to_string())) + } + } + Err(_) => Err(WebError::InvalidData("Invalid target format".to_string())) + } + } +} + +// Context: Provides subnet discovery functionality using masscan for bulk scanning +// Operation: Validates CIDR notation, launches masscan task and tracks progress +pub async fn add_subnet_with_masscan( + State(ctx): State, + Form(form): Form +) -> WebResult { + let subnet = form.subnet.trim(); + + if subnet.is_empty() { + return Ok(axum::Json(serde_json::json!({ + "success": false, + "message": "Subnet cannot be empty" + }))); + } + + // Validate subnet format + if !subnet.contains('/') { + return Ok(axum::Json(serde_json::json!({ + "success": false, + "message": "Invalid subnet format. Use CIDR notation (e.g., 192.168.1.0/24)" + }))); + } + + // Check if already exists + let existing = sqlx::query("SELECT target FROM subnets WHERE run_id = ? AND target = ?") + .bind(&ctx.run_id) + .bind(subnet) + .fetch_optional(&ctx.pool) + .await + .map_err(|e| WebError::Database(e.to_string()))?; + + if existing.is_some() { + return Ok(axum::Json(serde_json::json!({ + "success": false, + "message": format!("Subnet {} already exists", subnet) + }))); + } + + // Insert subnet without firing Hook::Subnet — masscan owns the Phase 1 → Phase 2 chain. + // Hook::Subnet fires after masscan completes (from masscan.rs), ensuring Phase 1 runs + // only after masscan-discovered hosts are already excluded from its exclusion list. + ctx.insert_subnet_no_hook(subnet).await + .map_err(|e| WebError::Database(e.to_string()))?; + + // Launch masscan + let subnets = vec![subnet.to_string()]; + let ctx_clone = ctx.clone(); + + crate::workflow::default_spawn( + "masscan_discovery", + subnet.to_string(), + crate::workflow::concurrency::OperationType::Global, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx_clone.clone(); + let subnets_clone = subnets.clone(); + async move { + if let Err(e) = crate::enumeration::masscan::masscan_discovery_non_interactive(&ctx2, subnets_clone.clone(), tx_opt).await { + crate::utilities::outputs::tag_info(&format!("Masscan failed: {}", e)); + // Masscan failed before completing — fall back to nmap Phase 1 directly + for s in &subnets_clone { + crate::workflow::dispatch_event(crate::workflow::Hook::Subnet, s, &ctx2); + } + } + } + }, + None, + crate::workflow::GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + + // Wait for task registration to complete + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + + // Get the task ID + let task_id = { + let active_tasks = crate::workflow::ACTIVE_TASKS.lock().unwrap(); + active_tasks.values() + .find(|t| t.name == "masscan_discovery" && t.target == subnet) + .map(|t| t.id.clone()) + .unwrap_or_else(|| "unknown".to_string()) + }; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Subnet {} added and masscan launched", subnet), + "masscan_task_id": task_id + }))) +} + +// Context: Enables execution of custom commands using proper workflow architecture +// Operation: Uses default_spawn with full concurrency management and task tracking +pub async fn run_custom_command( + State(_ctx): State, + headers: axum::http::HeaderMap, + Form(form): Form +) -> WebResult { + let ip = form.ip.trim(); + let command = form.command.trim(); + + // Get username for audit logging + let username = if let Some(token) = super::utils::get_session_token_from_headers(&headers) { + crate::web::auth::get_current_username(&token) + } else { + "unknown".to_string() + }; + + if command.is_empty() { + return Err(WebError::InvalidData("Command cannot be empty".to_string())); + } + + // Security audit log + crate::utilities::tag_info(&format!("User '{}' executing command '{}' on {}", username, command, ip)); + + + // Replace placeholder with actual IP first + let final_command = command.replace("__IP__", ip); + + // Execute command immediately and return output + let timeout_duration = std::time::Duration::from_secs(form.timeout.unwrap_or(300)); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let (tx, _rx) = tokio::sync::mpsc::unbounded_channel(); + + match crate::workflow::execution::execute_simple_command(&final_command, timeout_duration, cancel_token, tx).await { + Ok((exit_code, output)) => { + let result_output = if exit_code == 0 { + output + } else { + format!("Command failed (exit code: {})\n{}", exit_code, output) + }; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "output": result_output, + "command": final_command, + "exit_code": exit_code + }))) + } + Err(e) => { + Ok(axum::Json(serde_json::json!({ + "success": false, + "message": format!("Command execution failed: {}", e), + "command": final_command + }))) + } + } +} + +// Context: Provides data export functionality filtered by host tags +// Operation: Queries hosts matching tags and formats output as TXT, CSV, or JSON +pub async fn export_hosts_by_tags( + State(ctx): State, + Form(form): Form +) -> WebResult { + // Parse comma-separated tags from string + let tags = parse_tag_list(&form.tags); + + if tags.is_empty() { + return Err(WebError::InvalidData("No tags selected".to_string())); + } + + let mut all_hosts = std::collections::HashSet::new(); + + // Get hosts for each tag - search across all three tag columns + for tag in &tags { + let rows = sqlx::query( + "SELECT ip, hostname, ports, manual_tags, refined_tags, automated_tags, action_tags FROM targets + WHERE run_id = ? AND ( + manual_tags LIKE ? OR + refined_tags LIKE ? OR + automated_tags LIKE ? OR + action_tags LIKE ? + )" + ) + .bind(&ctx.run_id) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .bind(format!("%{}%", tag)) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + for row in rows { + let ip: String = row.get("ip"); + let hostname: Option = row.get("hostname"); + let ports: String = row.get("ports"); + let manual_tags: Option = row.get("manual_tags"); + let refined_tags: Option = row.get("refined_tags"); + let automated_tags: Option = row.get("automated_tags"); + let action_tags: Option = row.get("action_tags"); + + // Combine all tags for verification and display + let mut combined_tags = std::collections::HashSet::new(); + + for tags_opt in [manual_tags, refined_tags, automated_tags, action_tags] { + if let Some(tags_str) = tags_opt { + for t in tags_str.split(',').map(str::trim).filter(|s| !s.is_empty()) { + combined_tags.insert(t.to_string()); + } + } + } + + // Verify tag actually exists in the combined tags (guards against LIKE substring false positives) + if combined_tags.contains(tag) { + let combined_tags_str = { + let mut tag_vec: Vec = combined_tags.into_iter().collect(); + tag_vec.sort(); + tag_vec.join(",") + }; + + all_hosts.insert((ip, hostname.unwrap_or_default(), ports, combined_tags_str)); + } + } + } + + let content = match form.format.as_str() { + "txt" => { + // Simple IP list + all_hosts.iter() + .map(|(ip, _, _, _)| ip.clone()) + .collect::>() + .join("\n") + }, + "csv" => { + // CSV format with headers + let mut lines = vec!["IP,Hostname,Ports,Tags".to_string()]; + for (ip, hostname, ports, tags) in &all_hosts { + lines.push(format!("{},{},{},\"{}\"", ip, hostname, ports, tags)); + } + lines.join("\n") + }, + "json" => { + // JSON format + let json_data: Vec = all_hosts.iter() + .map(|(ip, hostname, ports, tags)| { + serde_json::json!({ + "ip": ip, + "hostname": hostname, + "ports": ports.split(',').filter(|p| !p.trim().is_empty()).collect::>(), + "tags": tags.split(',').filter(|t| !t.trim().is_empty()).collect::>() + }) + }) + .collect(); + serde_json::to_string_pretty(&json_data).unwrap_or_default() + }, + _ => return Err(WebError::InvalidData("Invalid format".to_string())) + }; + + let filename = format!("hosts_export_{}.{}", + chrono::Utc::now().format("%Y%m%d_%H%M%S"), + form.format); + + let content_type = match form.format.as_str() { + "txt" => "text/plain", + "csv" => "text/csv", + "json" => "application/json", + _ => "text/plain" + }; + + let headers = [ + (axum::http::header::CONTENT_TYPE, HeaderValue::from_static(content_type)), + (axum::http::header::CONTENT_DISPOSITION, HeaderValue::from_str(&format!("attachment; filename=\"{}\"", filename)).unwrap()) + ]; + + Ok((headers, content)) +} + +// Context: Shared resolution for bulk nuclei actions — finds hosts with HTTP services matching +// the tag and IP/CIDR filters, paired with each host's (port, protocol) services +// Operation: Returns one entry per matching host, each carrying its (port, protocol) list +async fn resolve_nuclei_targets( + ctx: &ProjectContext, + include_tags: &[String], + exclude_tags: &[String], + include_ips: &[ipnetwork::IpNetwork], + exclude_ips: &[ipnetwork::IpNetwork], +) -> Result)>, Box> { + let hosts = ctx.get_hosts_with_http_services_filtered(include_tags, exclude_tags, include_ips, exclude_ips).await?; + + let mut targets = Vec::new(); + for ip in hosts { + let services = ctx.get_http_services(&ip).await.unwrap_or_default(); + let services: Vec<(String, String)> = services.into_iter() + .map(|(port, service_json)| { + let protocol = serde_json::from_str::(&service_json) + .map(|i| i.protocol) + .unwrap_or_else(|_| if port == "443" || port.ends_with("443") { "https".to_string() } else { "http".to_string() }); + (port, protocol) + }) + .collect(); + targets.push((ip, services)); + } + + Ok(targets) +} + +// Context: Preview for the bulk nuclei action — lets the UI show exact host/service counts +// and the active tag filters in a confirmation prompt before launching +// Operation: Resolves the same target set as run_nuclei_bulk without dispatching anything +pub async fn nuclei_bulk_preview( + State(ctx): State, + Query(query): Query, +) -> WebResult { + let include_tags = parse_tag_list(&query.include_tags); + let exclude_tags = parse_tag_list(&query.exclude_tags); + let include_ips = parse_ip_list(&query.include_ips) + .map_err(|bad| WebError::InvalidData(format!("Invalid IP/CIDR: {bad}")))?; + let exclude_ips = parse_ip_list(&query.exclude_ips) + .map_err(|bad| WebError::InvalidData(format!("Invalid IP/CIDR: {bad}")))?; + + let targets = resolve_nuclei_targets(&ctx, &include_tags, &exclude_tags, &include_ips, &exclude_ips).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let service_count: usize = targets.iter().map(|(_, services)| services.len()).sum(); + + Ok(axum::Json(serde_json::json!({ + "success": true, + "host_count": targets.len(), + "service_count": service_count, + "include_tags": include_tags, + "exclude_tags": exclude_tags, + "include_ips": query.include_ips.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect::>(), + "exclude_ips": query.exclude_ips.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect::>(), + }))) +} + +// Context: Bulk vulnerability scanning from the Scanners page — runs nuclei against every +// discovered HTTP service, optionally filtered by tag include/exclude lists +// Operation: Resolves matching hosts, then dispatches a nuclei scan per (ip, port, protocol) +pub async fn run_nuclei_bulk( + State(ctx): State, + Form(form): Form, +) -> WebResult { + let include_tags = parse_tag_list(&form.include_tags); + let exclude_tags = parse_tag_list(&form.exclude_tags); + let include_ips = parse_ip_list(&form.include_ips) + .map_err(|bad| WebError::InvalidData(format!("Invalid IP/CIDR: {bad}")))?; + let exclude_ips = parse_ip_list(&form.exclude_ips) + .map_err(|bad| WebError::InvalidData(format!("Invalid IP/CIDR: {bad}")))?; + + let targets = resolve_nuclei_targets(&ctx, &include_tags, &exclude_tags, &include_ips, &exclude_ips).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let mut service_count = 0; + for (ip, services) in &targets { + for (port, protocol) in services { + crate::workflow::nuclei::dispatch_nuclei_scan(ip, port, protocol, &ctx); + service_count += 1; + } + } + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Queued nuclei scans for {} service(s) across {} host(s)", service_count, targets.len()) + }))) +} + +// Context: Provides host workflow restart capability for troubleshooting +// Operation: Resets host database state and triggers complete workflow restart +pub async fn reset_host_state( + State(ctx): State, + Form(form): Form +) -> WebResult { + // Use the new restart_host_workflow method which handles everything + ctx.restart_host_workflow(&form.ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Host reset and workflow restarted - nmap scan will begin shortly", + "ip": form.ip + }))) +} + +// Context: Allows manual retrigger of host tagging system for classification updates +// Operation: Clears automated tags and runs complete tagging workflow for host +pub async fn force_retag_host( + State(ctx): State, + Form(form): Form +) -> WebResult { + // Clear existing automated and refined tags (leave manual tags intact) + ctx.set_automated_tags(&form.ip, std::collections::HashSet::new()).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + ctx.set_refined_tags(&form.ip, std::collections::HashSet::new()).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + // Trigger nmap auto-tagging + if let Err(e) = crate::tag::automatic_tag(&ctx, &form.ip, crate::tag::TriggerContext::ServiceScanComplete).await { + return Err(WebError::DatabaseError(format!("Failed to retag: {}", e))); + } + + // Trigger HTTP auto-tagging if has HTTP services + if ctx.has_http_services(&form.ip).await.unwrap_or(false) { + if let Err(e) = crate::tag::auto_tag_http(&ctx, &form.ip).await { + tag_debug(&format!("HTTP retagging failed for {}: {}", form.ip, e)); + } + } + + // Now trigger refinement on the combined automated tags + if let Err(e) = crate::tag::refine_automated_tags(&ctx, &form.ip).await { + tag_debug(&format!("Tag refinement failed for {}: {}", form.ip, e)); + } + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Host retagged successfully", + "ip": form.ip + }))) +} + +// Context: Enables manual screenshot capture for web services verification +// Operation: Launches browser automation to capture screenshot and update service records +pub async fn retake_screenshot( + State(ctx): State, + Form(form): Form +) -> WebResult { + let url = format!("{}://{}:{}/", form.protocol, form.ip, form.port); + let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string(); + let filename = format!("{}_{}_{}_{}.png", + form.ip.replace(".", "_"), + form.port, + form.protocol, + timestamp); + // Ensure project-specific screenshot directory exists and get output path + let _screenshots_dir = crate::config::paths::ensure_screenshots_directory_for_run(&ctx.db_path, &ctx.run_id) + .map_err(|e| WebError::DatabaseError(format!("Failed to create screenshots directory: {}", e)))?; + let output_path = crate::config::paths::get_screenshot_file_path(&ctx.db_path, &ctx.run_id, &filename); + + crate::workflow::screenshots::dispatch_retake_screenshot( + &form.ip, + url, + output_path, + form.port.clone(), + &ctx, + ); + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Screenshot queued", + "filename": filename + }))) +} + +// Context: Manual nuclei vulnerability scan for a specific HTTP service +// Operation: Dispatches nuclei scan task visible in the task queue +pub async fn run_nuclei_scan( + State(ctx): State, + Form(form): Form, +) -> WebResult { + crate::workflow::nuclei::dispatch_nuclei_scan(&form.ip, &form.port, &form.protocol, &ctx); + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Nuclei scan queued" + }))) +} + +// Context: Provides blacklist management for excluding specific targets from scans +// Operation: Validates IP/CIDR format and adds entry to persistent blacklist database +pub async fn add_blacklist_entry( + State(ctx): State, + Form(form): Form +) -> WebResult { + let target = form.target.trim(); + + // Validate input + if target.is_empty() { + return Err(WebError::InvalidData("Target cannot be empty".to_string())); + } + + // Validate IP or CIDR format + let normalized_target = if target.contains('/') { + // Validate CIDR format + match target.parse::() { + Ok(net) => match net { + ipnetwork::IpNetwork::V4(v4) => format!("{}/{}", v4.network(), v4.prefix()), + _ => return Err(WebError::InvalidData("Only IPv4 networks are supported".to_string())), + }, + Err(_) => return Err(WebError::InvalidData("Invalid CIDR format".to_string())), + } + } else { + // Validate IP format and convert to /32 + match target.parse::() { + Ok(std::net::IpAddr::V4(ip)) => format!("{}/32", ip), + Ok(std::net::IpAddr::V6(_)) => return Err(WebError::InvalidData("IPv6 not supported".to_string())), + Err(_) => return Err(WebError::InvalidData("Invalid IP address format".to_string())), + } + }; + + // Use existing insert_blacklist method + match ctx.insert_blacklist(&normalized_target).await { + Ok(_) => { + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Successfully added {} to blacklist", normalized_target), + "target": normalized_target + }))) + } + Err(e) => { + Err(WebError::DatabaseError(format!("Failed to add to blacklist: {}", e))) + } + } +} + +// Context: Retrieves current blacklist entries for management interface display +// Operation: Queries blacklist table and returns sorted list of blocked CIDR ranges +pub async fn get_blacklist_entries(State(ctx): State) -> WebResult { + let rows = sqlx::query("SELECT cidr FROM blacklist WHERE run_id = ? ORDER BY cidr") + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let entries: Vec = rows.into_iter().map(|row| row.get("cidr")).collect(); + + Ok(axum::Json(serde_json::json!({ + "entries": entries + }))) +} + +// Context: Enables removal of entries from target blacklist for scope management +// Operation: Deletes specified CIDR entry from blacklist database and confirms action +pub async fn remove_blacklist_entry( + State(ctx): State, + Form(form): Form +) -> WebResult { + let target = form.target.trim(); + + if target.is_empty() { + return Err(WebError::InvalidData("Target cannot be empty".to_string())); + } + + // Remove from blacklist table + let rows_affected = sqlx::query("DELETE FROM blacklist WHERE run_id = ? AND cidr = ?") + .bind(&ctx.run_id) + .bind(target) + .execute(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))? + .rows_affected(); + + if rows_affected > 0 { + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("Successfully removed {} from blacklist", target), + "target": target + }))) + } else { + Err(WebError::NotFound(format!("Blacklist entry {} not found", target))) + } +} + +// Context: Parses uploaded text file containing targets or subnets +// Operation: Reads file, splits by separators, validates entries, returns for modal display +pub async fn parse_targets_file( + mut multipart: Multipart, +) -> WebResult { + let mut file_content = String::new(); + let mut file_type = String::from("targets"); + + // Read multipart form data + while let Some(field) = multipart.next_field().await.map_err(|e| WebError::InvalidData(e.to_string()))? { + let name = field.name().unwrap_or("").to_string(); + + if name == "file" { + let data = field.bytes().await.map_err(|e| WebError::InvalidData(e.to_string()))?; + // Security: Limit file size to 1MB + if data.len() > 1_048_576 { + return Err(WebError::InvalidData("File too large (max 1MB)".to_string())); + } + file_content = String::from_utf8_lossy(&data).to_string(); + } else if name == "file_type" { + let data = field.bytes().await.map_err(|e| WebError::InvalidData(e.to_string()))?; + file_type = String::from_utf8_lossy(&data).to_string(); + } + } + + // Security: File is NOT stored - only read and parsed + if file_content.is_empty() { + return Err(WebError::InvalidData("File is empty".to_string())); + } + + // Parse entries (split by , ; space newline) + let entries: Vec = file_content + .split(&[',', ';', ' ', '\n', '\r', '\t'][..]) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty() && !s.starts_with('#')) // Filter empty and comments + .collect(); + + if entries.is_empty() { + return Err(WebError::InvalidData("No valid entries found in file".to_string())); + } + + // Security: Limit to 1000 entries + if entries.len() > 1000 { + return Err(WebError::InvalidData(format!("Too many entries: {} (max 1000)", entries.len()))); + } + + // Smart categorization: detect both IPs and subnets regardless of file_type + let (mut ips, mut subnets, invalid) = categorize_and_validate_entries(entries); + + // Deduplicate entries + ips = deduplicate_entries(ips); + subnets = deduplicate_entries(subnets); + + // Check if we found anything + if ips.is_empty() && subnets.is_empty() { + return Err(WebError::InvalidData("No valid entries found in file".to_string())); + } + + // Type mismatch detection and warning messages + let mut warnings = Vec::new(); + + if file_type == "targets" && !subnets.is_empty() && ips.is_empty() { + warnings.push("No individual hosts detected, only subnets found. Use the Subnet Upload tab instead.".to_string()); + } else if file_type == "subnets" && !ips.is_empty() && subnets.is_empty() { + warnings.push("No subnets detected, only individual hosts found. Use the Target Upload tab instead.".to_string()); + } else if !ips.is_empty() && !subnets.is_empty() { + warnings.push(format!("Mixed entries detected: {} host(s) and {} subnet(s)", ips.len(), subnets.len())); + } + + // Build success message + let mut message_parts = vec![]; + if !ips.is_empty() { + message_parts.push(format!("{} host(s)", ips.len())); + } + if !subnets.is_empty() { + message_parts.push(format!("{} subnet(s)", subnets.len())); + } + let message = format!("Parsed {} (after deduplication)", message_parts.join(" and ")); + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": message, + "warnings": warnings, + "discovery_results": { + "ips": ips, + "subnets": subnets, + }, + "invalid_entries": invalid, + "deduplication_applied": true + }))) +} + +// Context: Smart entry categorization for file uploads +// Operation: Separates entries into IPs/hostnames and subnets, returns invalid entries +fn categorize_and_validate_entries(entries: Vec) -> (Vec, Vec, Vec) { + let mut ips = vec![]; + let mut subnets = vec![]; + let mut invalid = vec![]; + + for entry in entries { + // Try to parse as subnet first (CIDR notation) + if entry.parse::().is_ok() { + subnets.push(entry); + } + // Then try as IP or hostname + else if entry.parse::().is_ok() || is_valid_hostname(&entry) { + ips.push(entry); + } + // Invalid entry + else { + invalid.push(entry); + } + } + + (ips, subnets, invalid) +} + +// Context: Deduplication system for uploaded entries +// Operation: Removes duplicate entries (case-insensitive for hostnames, preserving first occurrence) +fn deduplicate_entries(entries: Vec) -> Vec { + use std::collections::HashSet; + + let mut seen = HashSet::new(); + let mut deduplicated = vec![]; + + for entry in entries { + // Normalize to lowercase for comparison (handles hostnames case-insensitivity) + let normalized = entry.to_lowercase(); + + // Only add if not seen before + if seen.insert(normalized) { + deduplicated.push(entry); + } + } + + deduplicated +} + +fn is_valid_hostname(hostname: &str) -> bool { + // Basic hostname validation + !hostname.is_empty() && + hostname.len() <= 253 && + hostname.chars().all(|c| c.is_alphanumeric() || c == '.' || c == '-') && + !hostname.starts_with('-') && + !hostname.ends_with('-') +} \ No newline at end of file diff --git a/src/web/handlers/hosts.rs b/src/web/handlers/hosts.rs new file mode 100755 index 0000000..81b54ea --- /dev/null +++ b/src/web/handlers/hosts.rs @@ -0,0 +1,508 @@ +// src/web/handlers/hosts.rs - Host-related page handlers + +use askama::Template; +use axum::{ + extract::{Path, Query, State, Form}, + response::{Html, IntoResponse}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use sqlx::Row; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError, parse_host_summary, get_bool_from_row, xml_ports_to_port_info, calculate_host_progress, parse_action_outputs, find_host_screenshots, get_combined_tags_with_source, TagWithSource, HostSummary, PortInfo, ActionOutput, ScreenshotInfo, HostProgress}; + + +// Add to the existing HostDetail struct in handlers.rs +#[derive(Serialize)] +pub struct HostDetail { + pub ip: String, + pub hostname: String, + pub short_hostname: String, + pub tags: Vec, // Combined manual + refined with source info + pub automated_tags: Vec, // Raw automated tags for bottom section + pub ad_domain: String, + pub ports: Vec, + pub progress: HostProgress, + + // Raw nmap scan data - simplified to avoid filter issues + pub nmap_scan_raw: String, + + // Action outputs split by whether they produced tags + pub action_findings: Vec, + pub action_other: Vec, + + // HTTP services and screenshots + pub http_services: HashMap, + pub screenshots: Vec, + + // Nuclei findings + pub nuclei_results: Vec, + pub nuclei_findings_count: usize, + pub nuclei_scanned_ports: Vec, +} + +#[derive(Deserialize)] +pub struct HostsListParams { + pub sort: Option, + pub dir: Option, + pub filter: Option, +} + +#[derive(Template)] +#[template(path = "hosts.html")] +pub struct HostsListTemplate { + pub hosts: Vec, + pub sort: String, + pub dir: String, + pub filter: String, + // Precomputed filter URLs (preserve current sort) + pub url_filter_clear: String, + pub url_filter_interesting: String, + pub url_filter_not_interesting: String, + pub url_filter_analyzed_ready: String, + pub url_filter_analyzed_done: String, +} + +#[derive(Template)] +#[template(path = "hosts_rows.html")] +pub struct HostsRowsTemplate { + pub hosts: Vec, +} + +fn make_filter_url(filter_val: &str, cur_sort: &str, cur_dir: &str) -> String { + if filter_val.is_empty() { + format!("/hosts?sort={}&dir={}", cur_sort, cur_dir) + } else { + format!("/hosts?sort={}&dir={}&filter={}", cur_sort, cur_dir, filter_val) + } +} + +#[derive(Template)] +#[template(path = "host_detail.html")] +pub struct HostDetailTemplate { + pub host: HostDetail, +} + +#[derive(Deserialize)] +pub struct InterestingForm { + pub ip: String, + pub interesting: bool, +} + +// Add new form struct for analyzed status +#[derive(Deserialize)] +pub struct AnalyzedForm { + pub ip: String, + pub analyzed: bool, +} + +#[derive(Deserialize)] +pub struct NucleiReviewedForm { + pub ip: String, + pub nuclei_reviewed: bool, +} + +// Shared helper: whitelists params, builds SQL, returns sorted host list +async fn fetch_hosts_sorted( + ctx: &ProjectContext, + params: &HostsListParams, +) -> Result<(Vec, &'static str, &'static str, &'static str), WebError> { + let sort: &'static str = match params.sort.as_deref() { + Some("hostname") => "hostname", + Some("ports") => "ports", + Some("http") => "http", + Some("analyzed") => "analyzed", + Some("interesting") => "interesting", + Some("nuclei_reviewed") => "nuclei_reviewed", + _ => "ip", + }; + let dir: &'static str = match params.dir.as_deref() { + Some("desc") => "desc", + _ => "asc", + }; + let filter: &'static str = match params.filter.as_deref() { + Some("interesting") => "interesting", + Some("not_interesting") => "not_interesting", + Some("analyzed_ready") => "analyzed_ready", + Some("analyzed_done") => "analyzed_done", + _ => "", + }; + + let filter_clause = match filter { + "interesting" => " AND interesting = 1", + "not_interesting" => " AND (interesting = 0 OR interesting IS NULL)", + "analyzed_ready" => " AND full_scan_complete = 1 AND service_scan_complete = 1 \ + AND http_discovery_complete = 1 AND screenshots_complete = 1 \ + AND COALESCE(user_workflow_complete, 0) = 1 \ + AND (analyzed = 0 OR analyzed IS NULL)", + "analyzed_done" => " AND analyzed = 1", + _ => "", + }; + + // All values in order_clause are static strings — no user input reaches the query + let order_clause: &str = match (sort, dir) { + ("hostname", "asc") => "CASE WHEN hostname IS NULL OR hostname = '' THEN 1 ELSE 0 END ASC, LOWER(COALESCE(hostname,'')) ASC", + ("hostname", _) => "CASE WHEN hostname IS NULL OR hostname = '' THEN 1 ELSE 0 END ASC, LOWER(COALESCE(hostname,'')) DESC", + ("ports", "asc") => "CASE WHEN ports IS NULL OR ports = '' THEN 0 ELSE LENGTH(ports) - LENGTH(REPLACE(ports, ',', '')) + 1 END ASC", + ("ports", _) => "CASE WHEN ports IS NULL OR ports = '' THEN 0 ELSE LENGTH(ports) - LENGTH(REPLACE(ports, ',', '')) + 1 END DESC", + ("http", "asc") => "(SELECT COUNT(*) FROM http_services WHERE run_id = targets.run_id AND ip = targets.ip) ASC", + ("http", _) => "(SELECT COUNT(*) FROM http_services WHERE run_id = targets.run_id AND ip = targets.ip) DESC", + ("analyzed", "asc") => "COALESCE(analyzed, 0) ASC, ip ASC", + ("analyzed", _) => "COALESCE(analyzed, 0) DESC, ip ASC", + ("interesting", "asc") => "COALESCE(interesting, 0) ASC, ip ASC", + ("interesting", _) => "COALESCE(interesting, 0) DESC, ip ASC", + ("nuclei_reviewed", "asc") => "COALESCE(nuclei_reviewed, 0) ASC, ip ASC", + ("nuclei_reviewed", _) => "COALESCE(nuclei_reviewed, 0) DESC, ip ASC", + (_, "desc") => "ip DESC", + _ => "ip ASC", + }; + + let query_str = format!( + "SELECT ip, \ + COALESCE(hostname,'') as hostname, \ + COALESCE(ports,'') as ports, \ + COALESCE(ad_domain,'') as ad_domain, \ + COALESCE(full_scan_complete, 0) as full_scan_complete, \ + COALESCE(service_scan_complete, 0) as service_scan_complete, \ + COALESCE(http_discovery_complete, 0) as http_discovery_complete, \ + COALESCE(screenshots_complete, 0) as screenshots_complete, \ + (SELECT COUNT(*) FROM action_outputs WHERE run_id = targets.run_id AND ip = targets.ip) as action_count, \ + COALESCE(analyzed, 0) as analyzed, \ + COALESCE(interesting, 0) as interesting, \ + COALESCE(nuclei_reviewed, 0) as nuclei_reviewed, \ + CASE WHEN EXISTS (SELECT 1 FROM action_outputs WHERE run_id = targets.run_id AND ip = targets.ip AND action_name LIKE 'nuclei_%') THEN 1 ELSE 0 END as has_nuclei_findings \ + FROM targets WHERE run_id = ?{} ORDER BY {}", + filter_clause, order_clause + ); + + let rows = sqlx::query(&query_str) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let mut hosts = Vec::new(); + for row in rows { + hosts.push(parse_host_summary(row, ctx).await); + } + + Ok((hosts, sort, dir, filter)) +} + +// Context: Provides list view of all discovered hosts with server-side sort and filter +// Operation: Builds SQL from whitelisted query params, renders hosts listing template +pub async fn hosts_list( + Query(params): Query, + State(ctx): State, +) -> WebResult { + let (hosts, sort, dir, filter) = fetch_hosts_sorted(&ctx, ¶ms).await?; + + let template = HostsListTemplate { + hosts, + sort: sort.to_string(), + dir: dir.to_string(), + filter: filter.to_string(), + url_filter_clear: make_filter_url("", sort, dir), + url_filter_interesting: make_filter_url("interesting", sort, dir), + url_filter_not_interesting: make_filter_url("not_interesting", sort, dir), + url_filter_analyzed_ready: make_filter_url("analyzed_ready", sort, dir), + url_filter_analyzed_done: make_filter_url("analyzed_done", sort, dir), + }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Returns only the row HTML for AJAX table refresh (no full page reload) +// Operation: Runs same SQL as hosts_list, renders fragment template for client-side tbody replacement +pub async fn hosts_rows( + Query(params): Query, + State(ctx): State, +) -> WebResult { + let (hosts, _sort, _dir, _filter) = fetch_hosts_sorted(&ctx, ¶ms).await?; + let template = HostsRowsTemplate { hosts }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Builds port list for host detail using XML as sole structured source +// Operation: Parses nmap_xml JSON map, extracts service_scan XML, converts to PortInfo. +fn parse_ports_from_xml(nmap_xml: &str) -> Vec { + if !nmap_xml.is_empty() && nmap_xml != "{}" { + if let Ok(map) = serde_json::from_str::>(nmap_xml) { + if let Some(xml) = map.get("service_scan") { + if let Some(result) = crate::tag::parse_nmap_xml(xml) { + let mut ports = xml_ports_to_port_info(result.ports); + ports.sort_by_key(|p| p.port.parse::().unwrap_or(0)); + return ports; + } + } + } + } + Vec::new() +} + +// Context: Displays comprehensive information for a specific host including scan results +// Operation: Aggregates host data, ports, services, screenshots and renders detail template +pub async fn host_detail(Path(ip): Path, State(ctx): State) -> WebResult { + let row = sqlx::query("SELECT * FROM targets WHERE run_id = ? AND ip = ?") + .bind(&ctx.run_id) + .bind(&ip) + .fetch_optional(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))? + .ok_or(WebError::NotFound(format!("Host {} not found", ip)))?; + + // Extract basic data + let hostname: Option = row.get("hostname"); + + // Get combined tags with source info + let tags = get_combined_tags_with_source(&ctx, &ip).await.unwrap_or_default(); + + // Get automated tags separately + let automated_tags_set = ctx.get_automated_tags(&ip).await.unwrap_or_default(); + let mut automated_tags: Vec = automated_tags_set.into_iter().collect(); + automated_tags.sort(); + + let ad_domain: Option = row.get("ad_domain"); + + // Raw nmap text — kept solely for the human-readable display block + let nmap_scan_str: String = row.try_get("nmap_scan").unwrap_or_else(|_| "{}".to_string()); + let nmap_display_text = if let Ok(nmap_json) = serde_json::from_str::>(&nmap_scan_str) { + if let Some(service_scan) = nmap_json.get("service_scan") { + service_scan.clone() + } else { + let mut parts: Vec<_> = nmap_json.into_values().collect(); + parts.sort(); + parts.join("\n\n") + } + } else { + nmap_scan_str.clone() + }; + + // XML is the sole structured source for port/service data + let nmap_xml_str: String = row.try_get("nmap_xml").unwrap_or_else(|_| "{}".to_string()); + let ports = parse_ports_from_xml(&nmap_xml_str); + + // Fetch action outputs from normalized table + let all_action_outputs = ctx.get_action_outputs_for_host(&ip).await + .map(parse_action_outputs) + .unwrap_or_default(); + let action_count = all_action_outputs.len(); + let (action_findings, action_other): (Vec<_>, Vec<_>) = all_action_outputs.into_iter().partition(|a| a.tagged); + + // Calculate progress with corrected logic + let progress = calculate_host_progress( + get_bool_from_row(&row, "full_scan_complete"), + get_bool_from_row(&row, "service_scan_complete"), + get_bool_from_row(&row, "http_discovery_complete"), + get_bool_from_row(&row, "screenshots_complete"), + action_count, + ); + + // Use the dedicated table + let http_services_raw = ctx.get_http_services(&ip).await.unwrap_or_default(); + let mut http_services = HashMap::new(); + for (port, service_json) in http_services_raw { + if let Ok(service_info) = serde_json::from_str::(&service_json) { + http_services.insert(port, service_info); + } + } + + // Find screenshots + let screenshots_dir = crate::config::paths::get_screenshots_directory(&ctx.db_path, &ctx.run_id); + let screenshots = find_host_screenshots(&ip, &http_services, &screenshots_dir); + + // Nuclei findings + let mut nuclei_results = ctx.get_nuclei_results_for_host(&ip).await.unwrap_or_default(); + for result in &mut nuclei_results { + if let Some(svc) = http_services.get(&result.port) { + result.url = format!("{}://{}:{}", svc.protocol, ip, result.port); + } + } + let nuclei_findings_count: usize = nuclei_results.iter().map(|r| r.findings.len()).sum(); + let nuclei_scanned_ports: Vec = nuclei_results.iter().map(|r| r.port.clone()).collect(); + + let hostname_str = hostname.unwrap_or_default(); + let ad_domain_str = ad_domain.unwrap_or_default(); + let short_hostname = crate::web::handlers::utils::strip_domain_suffix(&hostname_str, &ad_domain_str); + + let host_detail = HostDetail { + ip: ip.clone(), + hostname: hostname_str, + short_hostname, + tags, + automated_tags, + ad_domain: ad_domain_str, + ports, + progress, + nmap_scan_raw: nmap_display_text, + action_findings, + action_other, + http_services, + screenshots, + nuclei_results, + nuclei_findings_count, + nuclei_scanned_ports, + }; + + let template = HostDetailTemplate { host: host_detail }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Handler for marking hosts as analyzed +// Context: Allows marking hosts as analyzed for workflow progress tracking +// Operation: Updates host analysis status with retry logic for database consistency +pub async fn mark_host_analyzed( + State(ctx): State, + Form(form): Form +) -> WebResult { + // Add timeout and retry logic + let mut attempts = 0; + let max_attempts = 3; + + while attempts < max_attempts { + match ctx.mark_host_analyzed(&form.ip, form.analyzed).await { + Ok(_) => { + return Ok(axum::Json(serde_json::json!({ + "success": true, + "ip": form.ip, + "analyzed": form.analyzed, + "attempts": attempts + 1 + }))); + } + Err(e) if attempts < max_attempts - 1 => { + attempts += 1; + eprintln!("Attempt {} failed for host {}: {}", attempts, form.ip, e); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + Err(e) => { + eprintln!("Final attempt failed for host {}: {}", form.ip, e); + return Err(WebError::DatabaseError(format!("Failed to update analysis status after {} attempts: {}", max_attempts, e))); + } + } + } + + unreachable!() +} + +// Context: Provides current analysis status for host progress tracking +// Operation: Queries database and returns JSON with host analysis boolean status +pub async fn get_host_analysis_status( + Path(ip): Path, + State(ctx): State +) -> WebResult { + let analyzed = ctx.is_host_analyzed(&ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "ip": ip, + "analyzed": analyzed + }))) +} + +// Context: Enables marking hosts as interesting for prioritized analysis tracking +// Operation: Updates host interesting flag in database and returns confirmation +pub async fn mark_host_interesting( + State(ctx): State, + Form(form): Form +) -> WebResult { + ctx.mark_host_interesting(&form.ip, form.interesting).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "ip": form.ip, + "interesting": form.interesting + }))) +} + +// Context: Retrieves current interesting status for host prioritization display +// Operation: Queries database and returns JSON with host interesting boolean flag +pub async fn get_host_interesting_status( + Path(ip): Path, + State(ctx): State +) -> WebResult { + let interesting = ctx.is_host_interesting(&ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "ip": ip, + "interesting": interesting + }))) +} + +pub async fn mark_host_nuclei_reviewed( + State(ctx): State, + Form(form): Form, +) -> WebResult { + ctx.mark_host_nuclei_reviewed(&form.ip, form.nuclei_reviewed).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + Ok(axum::Json(serde_json::json!({ + "success": true, "ip": form.ip, "nuclei_reviewed": form.nuclei_reviewed + }))) +} + +pub async fn get_host_nuclei_reviewed_status( + Path(ip): Path, + State(ctx): State, +) -> WebResult { + let reviewed = ctx.is_host_nuclei_reviewed(&ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + Ok(axum::Json(serde_json::json!({ "ip": ip, "nuclei_reviewed": reviewed }))) +} + +// Context: Serves screenshot images with security validation for web interface display +// Operation: Validates filename format and serves PNG files from project-specific screenshots directory +pub async fn screenshot(Path(filename): Path, State(ctx): State) -> WebResult { + // Enhanced filename validation for security + if filename.contains("..") || filename.contains("/") || filename.contains("\\") { + return Err(WebError::InvalidData("Invalid filename".to_string())); + } + + // Must be PNG with expected format + if !filename.ends_with(".png") { + return Err(WebError::InvalidData("Invalid file type".to_string())); + } + + // Validate filename format: IP_PORT_PROTOCOL_TIMESTAMP.png + let name_without_ext = filename.strip_suffix(".png").unwrap(); + let parts: Vec<&str> = name_without_ext.split('_').collect(); + + if parts.len() < 7 { + return Err(WebError::InvalidData("Invalid filename format".to_string())); + } + + // Validate IP format (4 numeric parts) + for i in 0..4 { + if parts[i].parse::().is_err() { + return Err(WebError::InvalidData("Invalid IP format in filename".to_string())); + } + } + + // Validate port + if parts[4].parse::().is_err() { + return Err(WebError::InvalidData("Invalid port in filename".to_string())); + } + + // Validate protocol + if !["http", "https"].contains(&parts[5]) { + return Err(WebError::InvalidData("Invalid protocol in filename".to_string())); + } + + // Validate access within current project and get file path + if !crate::config::paths::validate_screenshot_access(&ctx.db_path, &ctx.run_id, &filename) { + return Err(WebError::NotFound("Screenshot not found in current project".to_string())); + } + + let file_path = crate::config::paths::get_screenshot_file_path(&ctx.db_path, &ctx.run_id, &filename); + + if std::path::Path::new(&file_path).exists() { + let contents = tokio::fs::read(&file_path) + .await + .map_err(|e| WebError::NotFound(format!("File read error: {}", e)))?; + + let content_type = "image/png"; + + Ok(([(axum::http::header::CONTENT_TYPE, content_type)], contents)) + } else { + Err(WebError::NotFound("Screenshot not found".to_string())) + } +} \ No newline at end of file diff --git a/src/web/handlers/logs.rs b/src/web/handlers/logs.rs new file mode 100755 index 0000000..deac599 --- /dev/null +++ b/src/web/handlers/logs.rs @@ -0,0 +1,112 @@ +// src/web/handlers/logs.rs - Logs page handlers + +use askama::Template; +use axum::{ + extract::{Query, State, Path}, + response::{Html, IntoResponse}, +}; +use serde::Deserialize; +use crate::sql::{ProjectContext, LogEntry}; +use super::utils::{WebResult, WebError}; + + +#[derive(Template)] +#[template(path = "logs.html")] +pub struct LogsTemplate { + pub logs: Vec, + pub selected_level: String, +} + +#[derive(Deserialize)] +pub struct LogParams { + pub level: Option, + pub since: Option, +} + +// Context: Provides web interface for viewing system logs using core filtering logic +// Operation: Uses core log filtering and renders logs template +pub async fn logs(Query(params): Query, State(ctx): State) -> WebResult { + let log_entries = ctx.get_logs_filtered( + params.level.as_deref(), + None // No timestamp filter for basic logs view + ).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let template = LogsTemplate { + logs: log_entries, + selected_level: params.level.unwrap_or_default() + }; + + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Provides JSON API for logs with incremental updates using core filtering logic +// Operation: Returns filtered logs as JSON with full timestamp and level filtering support +pub async fn api_logs(Query(params): Query, State(ctx): State) -> WebResult { + let log_entries = ctx.get_logs_filtered( + params.level.as_deref(), + params.since.as_deref() + ).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "logs": log_entries, + "count": log_entries.len(), + "timestamp": chrono::Utc::now().to_rfc3339() + }))) +} + +#[derive(Deserialize)] +pub struct TaskHistoryParams { + pub search: Option, + pub limit: Option, +} + +pub async fn api_task_history( + Query(params): Query, + State(ctx): State, +) -> WebResult { + let run_id = { + let g = crate::workflow::constants::CURRENT_RUN_ID.read().unwrap(); + g.clone().unwrap_or_else(|| ctx.run_id.clone()) + }; + let search = params.search.unwrap_or_default(); + let limit = params.limit.unwrap_or(200); + let tasks = ctx.list_task_history(&run_id, &search, limit).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + Ok(axum::Json(serde_json::json!({ "tasks": tasks }))) +} + +#[derive(Template)] +#[template(path = "task_output.html")] +pub struct TaskOutputTemplate { + pub task_name: String, + pub target: String, + pub status: String, + pub started_at: i64, + pub ended_at: i64, + pub output: String, +} + +pub async fn task_output_page( + Path(id): Path, + State(ctx): State, +) -> WebResult { + let row = ctx.get_task_output(&id).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + match row { + None => Err(WebError::NotFound("Task not found".to_string())), + Some((task_name, target, status, output, started_at, ended_at)) => { + let template = TaskOutputTemplate { + task_name, + target, + status, + started_at, + ended_at, + output, + }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) + } + } +} \ No newline at end of file diff --git a/src/web/handlers/mod.rs b/src/web/handlers/mod.rs new file mode 100755 index 0000000..66b1f44 --- /dev/null +++ b/src/web/handlers/mod.rs @@ -0,0 +1,66 @@ +// src/web/handlers/mod.rs - Main handlers module with all re-exports + +pub mod utils; +pub mod dashboard; +pub mod hosts; +pub mod tags; +pub mod settings; +pub mod setup; +pub mod logs; +pub mod queue; +pub mod actions; +pub mod api; +pub mod forms; +pub mod scanners; +pub mod styles; + +// Re-export auth handlers from parent auth module +pub use super::auth::{login_page, login_submit}; + +// Re-export all handler functions from each module +pub use dashboard::dashboard; + +pub use hosts::{ + hosts_list, hosts_rows, host_detail, mark_host_analyzed, get_host_analysis_status, + mark_host_interesting, get_host_interesting_status, screenshot, + mark_host_nuclei_reviewed, get_host_nuclei_reviewed_status +}; + +pub use tags::{ + get_available_tags, add_manual_tag, remove_manual_tag, + get_manual_tags, api_all_tags +}; + +pub use settings::{settings, create_player_user, manage_player_user, user_role_check, update_concurrency_limits}; + +pub use logs::{logs, api_logs, api_task_history, task_output_page}; + +pub use queue::{ + queued_tasks_page, get_task_detail, get_task_output_stream, api_task_output +}; + +pub use actions::{ + action_builder_page, action_preview, action_execute +}; + +pub use api::{ + api_task_status, api_hosts_ready_analysis, api_concurrency_status, api_discovery_stats, + kill_task, api_active_tasks, api_queue_tasks, api_queued_tasks, retry_http_probe, + api_nuclei_findings +}; + +pub use forms::{ + add_individual_target, add_subnet_with_masscan, run_custom_command, + export_hosts_by_tags, reset_host_state, force_retag_host, + retake_screenshot, run_nuclei_scan, run_nuclei_bulk, nuclei_bulk_preview, add_blacklist_entry, get_blacklist_entries, + remove_blacklist_entry, parse_targets_file +}; + +pub use scanners::{scanners_page, launch_discovery_tool}; + +pub use styles::serve_css; + +pub use setup::{setup_page, setup_submit}; + +// Re-export only what's actually used externally +pub use utils::{parse_host_summary, HostSummary}; \ No newline at end of file diff --git a/src/web/handlers/queue.rs b/src/web/handlers/queue.rs new file mode 100755 index 0000000..6bed0fd --- /dev/null +++ b/src/web/handlers/queue.rs @@ -0,0 +1,171 @@ +// src/web/handlers/queue.rs - Task queue and management handlers + +use askama::Template; +use axum::{ + extract::Path, + response::{Html, IntoResponse, Response}, +}; +use axum::extract::ws::{WebSocket, WebSocketUpgrade}; +use tokio::time::Duration; +use super::utils::{WebResult, WebError, convert_task_info, WebTaskInfo}; +use crate::workflow::{ACTIVE_TASKS, QUEUED_TASK_INFOS}; + +#[derive(Template)] +#[template(path = "task_detail.html")] +pub struct TaskDetailTemplate { + pub task: crate::workflow::TaskInfo, +} + +#[derive(Template)] +#[template(path = "queue.html")] +pub struct QueueTemplate { + pub active_tasks: Vec, + pub queued_tasks: Vec, + pub concurrency_stats: crate::workflow::concurrency::ConcurrencyStats, +} + +// Context: Provides task queue management interface with active and queued task lists +// Operation: Aggregates task data and concurrency stats into queue management template +pub async fn queued_tasks_page() -> WebResult { + // Get active tasks + let active_tasks_raw: Vec<_> = crate::workflow::ACTIVE_TASKS.lock().unwrap().values().cloned().collect(); + let active_tasks: Vec = active_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + // Get queued tasks + let queued_tasks_raw: Vec<_> = crate::workflow::QUEUED_TASK_INFOS.lock().unwrap().values().cloned().collect(); + let queued_tasks: Vec = queued_tasks_raw.iter() + .map(convert_task_info) + .collect(); + + // Get concurrency stats + let concurrency_stats = crate::workflow::concurrency::get_concurrency_stats(); + + let template = QueueTemplate { + active_tasks, + queued_tasks, + concurrency_stats, + }; + + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Displays detailed information for specific task including execution status +// Operation: Finds task by ID in active or queued lists and renders detail template +pub async fn get_task_detail(Path(id): Path) -> WebResult { + // Find task in active or queued lists + let task_info = { + let active_tasks = ACTIVE_TASKS.lock().unwrap(); + if let Some(task) = active_tasks.get(&id) { + Some(task.clone()) + } else { + let queued_tasks = QUEUED_TASK_INFOS.lock().unwrap(); + queued_tasks.get(&id).cloned() + } + }; + + match task_info { + Some(task) => { + let template = TaskDetailTemplate { task }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) + } + None => Err(WebError::NotFound(format!("Task {} not found", id))) + } +} + +// Context: Provides real-time task output streaming via WebSocket for monitoring +// Operation: Upgrades HTTP connection to WebSocket for live task output updates +pub async fn get_task_output_stream( + Path(id): Path, + ws: WebSocketUpgrade, +) -> Response { + ws.on_upgrade(move |socket| handle_task_stream(socket, id)) +} + +// Context: Manages WebSocket connection for streaming task output to client +// Operation: Continuously polls task buffer and sends new lines via WebSocket +async fn handle_task_stream(mut socket: WebSocket, task_id: String) { + let mut last_line_count = 0; + + loop { + // Find the task and clone the buffer content (use explicit scopes to ensure locks are dropped) + let buffer_content = { + // First check active tasks + { + let active_tasks = ACTIVE_TASKS.lock().unwrap(); + if let Some(task) = active_tasks.get(&task_id) { + let lines = task.raw_buffer.lock().unwrap(); + Some(lines.clone()) + } else { + None + } + } // active_tasks lock is definitely dropped here + }; + + let final_buffer = match buffer_content { + Some(content) => content, + None => { + // Check queued tasks in a separate scope + let queued_result = { + let queued_tasks = QUEUED_TASK_INFOS.lock().unwrap(); + queued_tasks.get(&task_id).map(|task| { + let lines = task.raw_buffer.lock().unwrap(); + lines.clone() + }) + }; // queued_tasks lock is definitely dropped here + + match queued_result { + Some(content) => content, + None => { + // Task not found - no locks held at this point + let _ = socket.send(axum::extract::ws::Message::Text("TASK_NOT_FOUND".to_string())).await; + return; + } + } + } + }; + + let current_line_count = final_buffer.len(); + + // Send new lines if any - no locks held at this point + if current_line_count > last_line_count { + for line in final_buffer.iter().skip(last_line_count) { + if socket.send(axum::extract::ws::Message::Text(line.clone())).await.is_err() { + return; // Client disconnected + } + } + last_line_count = current_line_count; + } + + // Wait before checking again - no locks held + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +// Context: Provides JSON API for retrieving complete task output buffer +// Operation: Returns task output as JSON with line count for client applications +pub async fn api_task_output(Path(id): Path) -> WebResult { + let buffer_content = { + let active_tasks = ACTIVE_TASKS.lock().unwrap(); + if let Some(task) = active_tasks.get(&id) { + let lines = task.raw_buffer.lock().unwrap(); + lines.clone() + } else { + let queued_tasks = QUEUED_TASK_INFOS.lock().unwrap(); + if let Some(task) = queued_tasks.get(&id) { + let lines = task.raw_buffer.lock().unwrap(); + lines.clone() + } else { + return Err(WebError::NotFound(format!("Task {} not found", id))); + } + } + }; + + let output = buffer_content.join("\n"); + Ok(axum::Json(serde_json::json!({ + "task_id": id, + "output": output, + "line_count": buffer_content.len() + }))) +} \ No newline at end of file diff --git a/src/web/handlers/scanners.rs b/src/web/handlers/scanners.rs new file mode 100755 index 0000000..6797736 --- /dev/null +++ b/src/web/handlers/scanners.rs @@ -0,0 +1,165 @@ +// src/web/handlers/scanners.rs - Scanner tools handlers + +use askama::Template; +use axum::{ + extract::{State, Form}, + response::{Html, IntoResponse}, +}; +use tokio::time::Duration; +use tokio_util::sync::CancellationToken; +use serde::Deserialize; +use crate::sql::ProjectContext; +use crate::workflow::tasks::{register_task, finalize_task_output}; +use super::utils::{WebResult, WebError}; + +#[derive(Template)] +#[template(path = "scanners.html")] +pub struct ScannersTemplate { + pub available_tags: Vec, +} + +#[derive(Deserialize)] +pub struct LaunchDiscoveryToolForm { + pub tool: String, +} + +// Context: Provides interface for discovery and scanning tool selection +// Operation: Renders scanners template with available tool options for network discovery +pub async fn scanners_page(State(ctx): State) -> WebResult { + let available_tags = ctx.get_distinct_tags().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + let template = ScannersTemplate { available_tags }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Launches various network discovery tools based on user selection +// Operation: Executes discovery tools and returns results or task IDs for monitoring +pub async fn launch_discovery_tool( + State(ctx): State, + Form(form): Form +) -> WebResult { + let tool_type = form.tool.trim().to_string(); + + if tool_type.is_empty() { + return Err(WebError::InvalidData("Tool type cannot be empty".to_string())); + } + + let (task_name, target) = match tool_type.as_str() { + "arp-scan" => ("enumerate_arp_scan", "local_network"), + "dns-search" => ("local_dns_search", "local_network"), + "interface-explorer" => ("interface_explorer", "local_network"), + "passive-listener" => ("passive_listener", "local_network"), + _ => return Err(WebError::InvalidData("Invalid tool type".to_string())), + }; + + match tool_type.as_str() { + // Discovery tools - return results for modal; registered in ACTIVE_TASKS for + // visibility and task history, but bypass the semaphore (user-initiated, must be immediate). + "dns-search" => { + let (task_info, _, collector_handle) = register_task("local_dns_search", "local_network", CancellationToken::new()); + let raw_tx = Some(task_info.raw_tx.clone()); + let result = crate::enumeration::dns::local_dns_search_enhanced(&ctx, raw_tx).await; + let status = if result.is_ok() { "completed" } else { "cancelled" }; + finalize_task_output(task_info, collector_handle, status).await; + match result { + Ok((ips, subnets)) => { + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "DNS search completed", + "discovery_results": { "ips": ips, "subnets": subnets } + }))) + } + Err(e) => Err(WebError::DatabaseError(format!("DNS search failed: {}", e))), + } + }, + + "interface-explorer" => { + let (task_info, _, collector_handle) = register_task("interface_explorer", "local_network", CancellationToken::new()); + let raw_tx = Some(task_info.raw_tx.clone()); + let result = crate::enumeration::network::interface_explorer(&ctx, raw_tx).await; + let status = if result.is_ok() { "completed" } else { "cancelled" }; + finalize_task_output(task_info, collector_handle, status).await; + match result { + Ok((ips, subnets)) => { + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Interface exploration completed", + "discovery_results": { "ips": ips, "subnets": subnets } + }))) + } + Err(e) => Err(WebError::DatabaseError(format!("Interface exploration failed: {}", e))), + } + }, + + // Persistent task tools - use default_spawn + "arp-scan" | "passive-listener" => { + // Check for existing passive listener + if tool_type == "passive-listener" { + let active_tasks = crate::workflow::ACTIVE_TASKS.lock().unwrap(); + if let Some(existing_task) = active_tasks.values().find(|t| t.name == "passive_listener") { + let task_id = existing_task.id.clone(); + return Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Passive listener already running - opening existing task", + "task_id": task_id, + "existing": true + }))); + } + } + + let ctx_clone = ctx.clone(); + let tool_name = match tool_type.as_str() { + "arp-scan" => "ARP Scan", + "passive-listener" => "Passive Listener", + _ => "Tool" + }; + + let _handle = crate::workflow::default_spawn( + task_name, + target.to_string(), + crate::workflow::concurrency::OperationType::Global, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx_clone.clone(); + let tool_type_inner = tool_type.clone(); + async move { + let result = match tool_type_inner.as_str() { + "arp-scan" => { + crate::enumeration::network::enumerate_arp_scan(&ctx2, tx_opt).await + }, + "passive-listener" => { + crate::enumeration::passive::passive_listener_enhanced(&ctx2, tx_opt).await + }, + _ => Ok(()) + }; + + if let Err(e) = result { + crate::utilities::outputs::tag_info(&format!("{} failed: {}", tool_type_inner, e)); + } + } + }, + None, + crate::workflow::GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + + tokio::time::sleep(Duration::from_millis(200)).await; + + // Get the task ID from active tasks + let task_id = { + let active_tasks = crate::workflow::ACTIVE_TASKS.lock().unwrap(); + active_tasks.values() + .find(|t| t.name == task_name) + .map(|t| t.id.clone()) + .unwrap_or_else(|| "unknown".to_string()) + }; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": format!("{} launched successfully", tool_name), + "task_id": task_id + }))) + }, + + _ => Err(WebError::InvalidData("Invalid tool type".to_string())) + } +} \ No newline at end of file diff --git a/src/web/handlers/settings.rs b/src/web/handlers/settings.rs new file mode 100755 index 0000000..58de46b --- /dev/null +++ b/src/web/handlers/settings.rs @@ -0,0 +1,318 @@ +// src/web/handlers/settings.rs - Settings page handlers + +use askama::Template; +use axum::{ + extract::State, + response::{Html, IntoResponse, Json}, + Form, +}; +use crate::workflow::concurrency::{set_concurrency_limit, set_launch_stagger_ms, get_concurrency_config, OperationType, ConcurrencyConfig}; +use serde::{Deserialize, Serialize}; +use crate::sql::ProjectContext; +use crate::web::auth::{UserRole, generate_secure_password, get_session_user}; +use super::utils::{WebResult, WebError, get_session_token_from_headers}; +use axum::http::HeaderMap; + +#[derive(Template)] +#[template(path = "settings.html")] +pub struct SettingsTemplate { + pub current_user_role: UserRole, + pub player_users: Vec, + pub concurrency_config: ConcurrencyConfig, + pub ephemeral_ports: usize, +} + +#[derive(Deserialize)] +pub struct ConcurrencyLimitsForm { + pub global: usize, + pub nmap_full_tcp: usize, + pub nmap_service: usize, + pub subnet_phase1: usize, + pub subnet_phase2: usize, + pub http_discovery: usize, + pub launch_stagger_ms: u64, +} + +#[derive(Deserialize)] +pub struct CreatePlayerForm { + pub username: String, + pub auto_generate_password: Option, + pub custom_password: Option, +} + +#[derive(Serialize)] +pub struct CreatePlayerResponse { + pub success: bool, + pub message: String, + pub username: Option, + pub password: Option, +} + +#[derive(Deserialize)] +pub struct UserActionForm { + pub username: String, + pub action: String, // "enable", "disable", "reset_password", "delete" +} + +// Context: Displays current system configuration and project settings with user management +// Operation: Retrieves configuration data and user information, renders enhanced settings template +pub async fn settings(State(ctx): State, headers: HeaderMap) -> WebResult { + // Get current user role from session + let current_user_role = if let Some(token) = get_session_token_from_headers(&headers) { + if let Some(user) = get_session_user(&token) { + user.role + } else { + return Err(WebError::NotFound("Invalid session".to_string())); + } + } else { + return Err(WebError::NotFound("No session found".to_string())); + }; + + // Get player users (only for admin) + let player_users = if current_user_role == UserRole::Admin { + ctx.get_player_users().await.unwrap_or_else(|_| vec![]) + } else { + vec![] + }; + + let concurrency_config = get_concurrency_config().unwrap_or_else(ConcurrencyConfig::conservative); + let ephemeral_ports = std::fs::read_to_string("/proc/sys/net/ipv4/ip_local_port_range") + .ok() + .and_then(|s| { + let mut parts = s.split_whitespace(); + let lo: usize = parts.next()?.parse().ok()?; + let hi: usize = parts.next()?.parse().ok()?; + Some(hi.saturating_sub(lo)) + }) + .unwrap_or(28231); // 32768–60999 kernel default + + let template = SettingsTemplate { + current_user_role, + player_users, + concurrency_config, + ephemeral_ports, + }; + + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + + +// Context: Creates new player user account (admin only) +// Operation: Validates input, generates password if needed, creates user in database +pub async fn create_player_user( + State(ctx): State, + headers: HeaderMap, + Form(form): Form +) -> impl IntoResponse { + // Verify admin privileges + if let Some(token) = get_session_token_from_headers(&headers) { + if let Some(user) = get_session_user(&token) { + if user.role != UserRole::Admin { + return Json(CreatePlayerResponse { + success: false, + message: "Admin privileges required".to_string(), + username: None, + password: None, + }); + } + } else { + return Json(CreatePlayerResponse { + success: false, + message: "Invalid session".to_string(), + username: None, + password: None, + }); + } + } else { + return Json(CreatePlayerResponse { + success: false, + message: "Authentication required".to_string(), + username: None, + password: None, + }); + } + + let username = form.username.trim(); + + // Validate username + if username.is_empty() || username.len() < 3 { + return Json(CreatePlayerResponse { + success: false, + message: "Username must be at least 3 characters".to_string(), + username: None, + password: None, + }); + } + + // Check if username already exists + if ctx.username_exists(username).await.unwrap_or(true) { + return Json(CreatePlayerResponse { + success: false, + message: "Username already exists".to_string(), + username: None, + password: None, + }); + } + + // Generate or use provided password + // HTML checkboxes send "on" when checked, nothing when unchecked + let auto_generate = form.auto_generate_password.is_some(); + let password = if auto_generate { + generate_secure_password() + } else { + form.custom_password.unwrap_or_else(|| generate_secure_password()) + }; + + // Create user + match ctx.create_player_user(username, &password).await { + Ok(_) => Json(CreatePlayerResponse { + success: true, + message: format!("Player account '{}' created successfully", username), + username: Some(username.to_string()), + password: Some(password), + }), + Err(e) => Json(CreatePlayerResponse { + success: false, + message: format!("Failed to create user: {}", e), + username: None, + password: None, + }) + } +} + +// Context: Manages player user actions (enable, disable, reset password, delete) +// Operation: Performs requested action on specified user account (admin only) +pub async fn manage_player_user( + State(ctx): State, + headers: HeaderMap, + Form(form): Form +) -> impl IntoResponse { + // Verify admin privileges + if let Some(token) = get_session_token_from_headers(&headers) { + if let Some(user) = get_session_user(&token) { + if user.role != UserRole::Admin { + return Json(serde_json::json!({ + "success": false, + "message": "Admin privileges required" + })); + } + } else { + return Json(serde_json::json!({ + "success": false, + "message": "Invalid session" + })); + } + } else { + return Json(serde_json::json!({ + "success": false, + "message": "Authentication required" + })); + } + + let username = form.username.trim(); + + match form.action.as_str() { + "enable" => { + match ctx.set_user_enabled(username, true).await { + Ok(_) => Json(serde_json::json!({ + "success": true, + "message": format!("User '{}' enabled", username) + })), + Err(e) => Json(serde_json::json!({ + "success": false, + "message": format!("Failed to enable user: {}", e) + })) + } + }, + "disable" => { + match ctx.set_user_enabled(username, false).await { + Ok(_) => Json(serde_json::json!({ + "success": true, + "message": format!("User '{}' disabled", username) + })), + Err(e) => Json(serde_json::json!({ + "success": false, + "message": format!("Failed to disable user: {}", e) + })) + } + }, + "reset_password" => { + let new_password = generate_secure_password(); + match ctx.reset_user_password(username, &new_password).await { + Ok(_) => Json(serde_json::json!({ + "success": true, + "message": format!("Password reset for user '{}'", username), + "new_password": new_password + })), + Err(e) => Json(serde_json::json!({ + "success": false, + "message": format!("Failed to reset password: {}", e) + })) + } + }, + "delete" => { + match ctx.delete_user(username).await { + Ok(_) => Json(serde_json::json!({ + "success": true, + "message": format!("User '{}' deleted", username) + })), + Err(e) => Json(serde_json::json!({ + "success": false, + "message": format!("Failed to delete user: {}", e) + })) + } + }, + _ => Json(serde_json::json!({ + "success": false, + "message": "Invalid action" + })) + } +} + +// Simple endpoint to check if user has admin privileges +pub async fn user_role_check() -> impl IntoResponse { + Json(serde_json::json!({ + "success": true, + "message": "Admin access confirmed" + })) +} + +// Context: Real-time concurrency limit adjustment from the settings UI +// Operation: Validates admin session, applies new limits immediately via set_concurrency_limit, +// then persists them to the DB so they survive restart/resume. +pub async fn update_concurrency_limits( + State(ctx): State, + headers: HeaderMap, + Json(form): Json, +) -> impl IntoResponse { + // Verify admin privileges + if let Some(token) = get_session_token_from_headers(&headers) { + if let Some(user) = get_session_user(&token) { + if user.role != UserRole::Admin { + return Json(serde_json::json!({"success": false, "message": "Admin privileges required"})); + } + } else { + return Json(serde_json::json!({"success": false, "message": "Invalid session"})); + } + } else { + return Json(serde_json::json!({"success": false, "message": "Authentication required"})); + } + + // Apply immediately + set_concurrency_limit(OperationType::Global, form.global); + set_concurrency_limit(OperationType::NmapFullTcp, form.nmap_full_tcp); + set_concurrency_limit(OperationType::NmapService, form.nmap_service); + set_concurrency_limit(OperationType::SubnetPhase1, form.subnet_phase1); + set_concurrency_limit(OperationType::SubnetPhase2, form.subnet_phase2); + set_concurrency_limit(OperationType::HttpDiscovery, form.http_discovery); + set_launch_stagger_ms(form.launch_stagger_ms); + + // Persist to DB + if let Some(config) = get_concurrency_config() { + if let Err(e) = ctx.save_concurrency_config(&config).await { + return Json(serde_json::json!({"success": false, "message": format!("Applied but failed to persist: {}", e)})); + } + } + + Json(serde_json::json!({"success": true, "message": "Concurrency limits updated"})) +} \ No newline at end of file diff --git a/src/web/handlers/setup.rs b/src/web/handlers/setup.rs new file mode 100644 index 0000000..49258a9 --- /dev/null +++ b/src/web/handlers/setup.rs @@ -0,0 +1,64 @@ +// src/web/handlers/setup.rs - First-run admin password setup + +use askama::Template; +use axum::{ + extract::State, + response::{Html, IntoResponse, Redirect}, + Form, +}; +use serde::Deserialize; +use crate::sql::ProjectContext; + +#[derive(Template)] +#[template(path = "setup.html")] +pub struct SetupTemplate { + pub error: String, +} + +#[derive(Deserialize)] +pub struct SetupForm { + pub password: String, + pub password_confirm: String, +} + +// Context: Renders the first-run admin password setup page +// Operation: Shows setup form if no admin exists; redirects to login if already configured +pub async fn setup_page(State(ctx): State) -> impl IntoResponse { + if ctx.admin_exists().await.unwrap_or(false) { + return Redirect::to("/login").into_response(); + } + let template = SetupTemplate { error: String::new() }; + Html(template.render().unwrap_or_else(|_| "Template error".to_string())).into_response() +} + +// Context: Processes first-run admin password form submission +// Operation: Validates password, creates admin account, redirects to login +pub async fn setup_submit( + State(ctx): State, + Form(form): Form, +) -> impl IntoResponse { + // Already configured — just redirect + if ctx.admin_exists().await.unwrap_or(false) { + return Redirect::to("/login").into_response(); + } + + let render_error = |msg: &str| { + let template = SetupTemplate { error: msg.to_string() }; + Html(template.render().unwrap_or_else(|_| "Template error".to_string())).into_response() + }; + + let password = form.password.trim(); + let confirm = form.password_confirm.trim(); + + if password.len() < 8 { + return render_error("Password must be at least 8 characters."); + } + if password != confirm { + return render_error("Passwords do not match."); + } + + match ctx.create_admin_user(password).await { + Ok(_) => Redirect::to("/login").into_response(), + Err(e) => render_error(&format!("Failed to create admin account: {}", e)), + } +} diff --git a/src/web/handlers/styles.rs b/src/web/handlers/styles.rs new file mode 100755 index 0000000..84b3fac --- /dev/null +++ b/src/web/handlers/styles.rs @@ -0,0 +1,23 @@ +// src/web/handlers/styles.rs - CSS serving handlers + +use askama::Template; +use axum::response::IntoResponse; + +#[derive(Template)] +#[template(path = "styles.html")] +pub struct StylesTemplate { +} + +// Context: Serves dynamically generated CSS styles for web interface theming +// Operation: Renders CSS template with proper content-type header for browser styling +pub async fn serve_css() -> impl IntoResponse { + let template = StylesTemplate { + }; + + let css_content = template.render().unwrap_or_default(); + + ( + [(axum::http::header::CONTENT_TYPE, "text/css")], + css_content + ) +} \ No newline at end of file diff --git a/src/web/handlers/tags.rs b/src/web/handlers/tags.rs new file mode 100755 index 0000000..966a83e --- /dev/null +++ b/src/web/handlers/tags.rs @@ -0,0 +1,258 @@ +// src/web/handlers/tags.rs - Tag management handlers + +use askama::Template; +use axum::{ + extract::{Path, Query, State, Form}, + response::{Html, IntoResponse}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use sqlx::Row; +use crate::sql::ProjectContext; +use super::utils::{WebResult, WebError, HostSummary}; + +/// Tag information for interactive tag browser +#[derive(Serialize)] +pub struct TagInfo { + pub name: String, + pub count: usize, +} + +/// Category group for template +#[derive(Serialize)] +pub struct CategoryGroup { + pub name: String, + pub tags: Vec, +} + +#[derive(Deserialize)] +pub struct TagSearchParams { + pub tag: Option, +} + +#[derive(Template)] +#[template(path = "available_tags.html")] +pub struct AvailableTagsTemplate { + pub categories: Vec, + pub search_query: String, + pub search_results: Vec, +} + +#[derive(Deserialize)] +pub struct ManualTagForm { + pub ip: String, + pub tag: String, +} + +// Context: Aggregates and categorizes all host tags for tag browser interface +// Operation: Queries tag data and groups by category with counts and sorting +async fn collect_all_tags(ctx: &ProjectContext) -> Result, WebError> { + let rows = sqlx::query( + "SELECT manual_tags, refined_tags, action_tags FROM targets WHERE run_id = ?" + ) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let mut combined_tag_counts: HashMap = HashMap::new(); + let mut manual_tag_counts: HashMap = HashMap::new(); + + for row in rows { + let manual_tags_str: Option = row.get("manual_tags"); + let refined_tags_str: Option = row.get("refined_tags"); + let action_tags_str: Option = row.get("action_tags"); + + let mut combined_tags = std::collections::HashSet::new(); + + // Process manual tags separately (they get their own top-level category) + if let Some(tags_str) = manual_tags_str { + for tag in tags_str.split(',').map(str::trim).filter(|s| !s.is_empty()) { + let tag = tag.to_string(); + *manual_tag_counts.entry(tag.clone()).or_insert(0) += 1; + combined_tags.insert(tag); + } + } + + // Refined tags and action tags (from YAML workflow) flow into categorized view + for tags_str in [refined_tags_str, action_tags_str].into_iter().flatten() { + for tag in tags_str.split(',').map(str::trim).filter(|s| !s.is_empty()) { + combined_tags.insert(tag.to_string()); + } + } + + for tag in combined_tags { + *combined_tag_counts.entry(tag).or_insert(0) += 1; + } + } + + let mut categories = Vec::new(); + + // Add Manual Tags category first (alphabetical order) + if !manual_tag_counts.is_empty() { + let mut manual_tags: Vec = manual_tag_counts.iter() + .map(|(name, count)| TagInfo { name: name.clone(), count: *count }) + .collect(); + manual_tags.sort_by(|a, b| a.name.cmp(&b.name)); // Alphabetical order + + categories.push(CategoryGroup { + name: "Manual Tags".to_string(), + tags: manual_tags, + }); + } + + // Group remaining tags by category (existing categorize_tag logic) + let mut other_categories: HashMap> = HashMap::new(); + + for (name, count) in combined_tag_counts { + // CRITICAL FIX: Skip manual tags to prevent duplicates + if manual_tag_counts.contains_key(&name) { + continue; + } + + let category = categorize_tag(&name).unwrap_or_else(|| "Other".to_string()); + other_categories.entry(category).or_insert_with(Vec::new).push(TagInfo { name, count }); + } + + // Add other categories with special character sorting + for (name, mut tags) in other_categories { + tags.sort_by(|a, b| { + let a_starts_special = a.name.chars().next() + .map(|c| !c.is_alphanumeric()) + .unwrap_or(false); + let b_starts_special = b.name.chars().next() + .map(|c| !c.is_alphanumeric()) + .unwrap_or(false); + + match (a_starts_special, b_starts_special) { + (true, false) => std::cmp::Ordering::Greater, // Special chars AFTER alphanumeric + (false, true) => std::cmp::Ordering::Less, // Alphanumeric BEFORE special chars + _ => b.count.cmp(&a.count).then_with(|| a.name.cmp(&b.name)) + } + }); + categories.push(CategoryGroup { name, tags }); + } + + // Sort categories (Manual Tags first, then alphabetically) + categories.sort_by(|a, b| { + if a.name == "Manual Tags" { + std::cmp::Ordering::Less + } else if b.name == "Manual Tags" { + std::cmp::Ordering::Greater + } else { + a.name.cmp(&b.name) + } + }); + + Ok(categories) +} + +// Context: Provides tag categorization logic for organizing tag browser display +// Operation: Matches tag names against predefined patterns to assign categories +fn categorize_tag(tag: &str) -> Option { + let tag_lower = tag.to_lowercase(); + + if ["windows", "linux", "freebsd", "macos", "unix", "centos", "ubuntu", "debian", "rhel"].iter().any(|&os| tag_lower.contains(os)) { + Some("OS".to_string()) + } else if ["apache", "nginx", "iis", "tomcat", "jenkins", "gitlab", "wordpress", "drupal", "joomla", "phpmyadmin"].iter().any(|&web| tag_lower.contains(web)) { + Some("Web".to_string()) + } else if ["mysql", "postgresql", "mongodb", "redis", "oracle", "mssql", "sqlite", "mariadb"].iter().any(|&db| tag_lower.contains(db)) { + Some("Database".to_string()) + } else if ["exchange", "postfix", "sendmail", "dovecot", "zimbra", "outlook"].iter().any(|&mail| tag_lower.contains(mail)) { + Some("Mail".to_string()) + } else if ["printer", "hp-", "canon", "epson", "xerox", "brother", "lexmark", "ricoh", "kyocera", "samsung", "dell", "oki", "konica", "minolta", "sharp", "toshiba", "panasonic", "fuji"].iter().any(|&printer| tag_lower.contains(printer)) { + Some("Printer".to_string()) + } else { + Some("Other".to_string()) + } +} + +// Context: Provides tag browser interface with categorized tag listings and inline host search +// Operation: Collects and renders all tags by category; runs host search when ?tag= param present +pub async fn get_available_tags( + Query(params): Query, + State(ctx): State, +) -> WebResult { + let categories = collect_all_tags(&ctx).await?; + + let (search_query, search_results) = if let Some(ref tag) = params.tag { + let results = ctx.search_hosts_by_tag(tag).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + (tag.clone(), results) + } else { + (String::new(), Vec::new()) + }; + + let template = AvailableTagsTemplate { categories, search_query, search_results }; + Ok(Html(template.render().map_err(|e| WebError::ParseError(e.to_string()))?)) +} + +// Context: Enables adding custom manual tags to hosts for classification +// Operation: Updates host manual tags set and saves to database +pub async fn add_manual_tag( + State(ctx): State, + Form(form): Form +) -> WebResult { + let mut manual_tags = ctx.get_manual_tags(&form.ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + manual_tags.insert(form.tag.clone()); + + ctx.set_manual_tags(&form.ip, manual_tags).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Tag added successfully" + }))) +} + +// Context: Allows removal of manual tags from hosts for tag management +// Operation: Removes specified tag from host manual tags set and updates database +pub async fn remove_manual_tag( + State(ctx): State, + Form(form): Form +) -> WebResult { + let mut manual_tags = ctx.get_manual_tags(&form.ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + manual_tags.remove(&form.tag); + + ctx.set_manual_tags(&form.ip, manual_tags).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "success": true, + "message": "Tag removed successfully" + }))) +} + +// Context: Retrieves current manual tags for specific host display +// Operation: Returns sorted list of manual tags for host as JSON response +pub async fn get_manual_tags( + Path(ip): Path, + State(ctx): State +) -> WebResult { + let manual_tags = ctx.get_manual_tags(&ip).await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + let mut tags: Vec = manual_tags.into_iter().collect(); + tags.sort(); + + Ok(axum::Json(serde_json::json!({ + "tags": tags + }))) +} + +// Context: Provides comprehensive tag statistics API for all tag types +// Operation: Returns counts for manual, refined, and automated tags as JSON using core functions +pub async fn api_all_tags(State(ctx): State) -> WebResult { + let tag_counts = ctx.get_all_tags_with_counts().await + .map_err(|e| WebError::DatabaseError(e.to_string()))?; + + Ok(axum::Json(serde_json::json!({ + "manual_tags": tag_counts.manual_tags, + "refined_tags": tag_counts.refined_tags, + "automated_tags": tag_counts.automated_tags + }))) +} \ No newline at end of file diff --git a/src/web/handlers/utils.rs b/src/web/handlers/utils.rs new file mode 100755 index 0000000..85ba202 --- /dev/null +++ b/src/web/handlers/utils.rs @@ -0,0 +1,472 @@ +// src/web/handlers/utils.rs - Shared utilities and error handling + +use axum::{ + response::{IntoResponse, Response}, + http::StatusCode, +}; +use serde::Serialize; +use std::collections::HashMap; +use sqlx::Row; +use crate::sql::ProjectContext; +use std::time::{SystemTime, UNIX_EPOCH}; + +// Custom filters module for Askama templates +pub mod filters { +} + +#[derive(Debug)] +pub enum WebError { + InvalidData(String), + DatabaseError(String), + ParseError(String), + NotFound(String), + Database(String), +} + +impl IntoResponse for WebError { + fn into_response(self) -> Response { + let (status, message) = match self { + WebError::InvalidData(msg) => (StatusCode::BAD_REQUEST, msg), + WebError::DatabaseError(msg) => (StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", msg)), + WebError::Database(msg) => (StatusCode::INTERNAL_SERVER_ERROR, format!("Database error: {}", msg)), + WebError::ParseError(msg) => (StatusCode::INTERNAL_SERVER_ERROR, format!("Parse error: {}", msg)), + WebError::NotFound(msg) => (StatusCode::NOT_FOUND, format!("Not found: {}", msg)), + }; + + let json_response = serde_json::json!({ + "success": false, + "message": message + }); + + (status, axum::Json(json_response)).into_response() + } +} + +impl std::fmt::Display for WebError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WebError::InvalidData(msg) => write!(f, "Invalid data: {}", msg), + WebError::DatabaseError(msg) => write!(f, "Database error: {}", msg), + WebError::Database(msg) => write!(f, "Database error: {}", msg), + WebError::NotFound(msg) => write!(f, "Not found: {}", msg), + WebError::ParseError(msg) => write!(f, "Parse error: {}", msg), + } + } +} + +impl std::error::Error for WebError {} + +pub type WebResult = Result; + +// Context: Provides safe boolean conversion from SQLite integer values +// Operation: Extracts integer column value and converts to boolean for web display +pub fn get_bool_from_row(row: &sqlx::sqlite::SqliteRow, column: &str) -> bool { + row.try_get::, _>(column) + .unwrap_or(Some(0)) + .unwrap_or(0) != 0 +} + +// Context: Converts action output rows into structured data for web display +// Operation: Parses (action_name, output_json) rows with deterministic alphabetical ordering +pub fn parse_action_outputs(rows: Vec<(String, String, bool)>) -> Vec { + let mut outputs: Vec = rows.into_iter().filter_map(|(action_name, output_json, tagged)| { + let v = serde_json::from_str::(&output_json).ok()?; + let commands = v.get("commands")?.as_array()?; + let first = commands.first()?; + let command = first.get("command").and_then(|c| c.as_str()).unwrap_or("").to_string(); + let output = first.get("output").and_then(|o| o.as_str()).unwrap_or("").to_string(); + let exit_code = first.get("exit_code").and_then(|e| e.as_i64()).unwrap_or(-1); + let duration_ms = first.get("duration_ms").and_then(|d| d.as_u64()).unwrap_or(0); + Some(ActionOutput { name: action_name, command, output, exit_code, duration_ms, tagged }) + }).collect(); + outputs.sort_by(|a, b| a.name.cmp(&b.name)); + outputs +} + +// Context: Locates screenshot files for specific host based on filename patterns +// Operation: Scans project-specific screenshot directory and matches files by IP with validation +pub fn find_host_screenshots(ip: &str, _http_services: &HashMap, screenshots_dir: &str) -> Vec { + let mut screenshots = Vec::new(); + + if let Ok(entries) = std::fs::read_dir(&screenshots_dir) { + for entry in entries.flatten() { + if let Some(filename) = entry.file_name().to_str() { + if filename.ends_with(".png") { + let name_without_ext = filename.strip_suffix(".png").unwrap_or(filename); + let parts: Vec<&str> = name_without_ext.split('_').collect(); + + // ROBUST VALIDATION: Exactly 4 IP octets + port + protocol + timestamp + if parts.len() >= 7 { + // Reconstruct IP from first 4 parts + let reconstructed_ip = format!("{}.{}.{}.{}", parts[0], parts[1], parts[2], parts[3]); + + // EXACT IP MATCH required + if reconstructed_ip == ip { + let port = parts[4].to_string(); + let protocol = parts[5].to_string(); + + // Validate port is numeric + if port.parse::().is_ok() { + let url = format!("{}://{}:{}", protocol, ip, port); + + screenshots.push(ScreenshotInfo { + filename: filename.to_string(), + port, + protocol, + url, + }); + } + } + } + } + } + } + } + + // Sort by port number + screenshots.sort_by(|a, b| { + a.port.parse::().unwrap_or(0) + .cmp(&b.port.parse::().unwrap_or(0)) + }); + + screenshots +} + +// Context: Determines host scan completion percentage for progress tracking +// Operation: Evaluates completion flags and calculates overall progress percentage +pub fn calculate_host_progress( + full_scan: bool, + service_scan: bool, + http_discovery: bool, + screenshots: bool, + action_count: usize, +) -> HostProgress { + // SIMPLIFIED: 4 core phases only (no web enumeration bonus) + let total_phases = 4.0_f32; + let mut completed = 0.0_f32; + + if full_scan { completed += 1.0; } + if service_scan { completed += 1.0; } + if http_discovery { completed += 1.0; } + if screenshots { completed += 1.0; } + + HostProgress { + full_scan, + service_scan, + http_discovery, + web_enumeration: action_count > 0, // Keep for internal tracking but don't count toward percentage + screenshots, + completion_percentage: ((completed / total_phases) * 100.0).round(), + } +} + +pub(crate) fn strip_domain_suffix(hostname: &str, domain: &str) -> String { + if domain.is_empty() || hostname.is_empty() { + return hostname.to_string(); + } + // PTR returned just the domain — no short hostname to extract + if hostname.to_lowercase() == domain.to_lowercase() { + return String::new(); + } + let suffix = format!(".{}", domain.to_lowercase()); + if hostname.to_lowercase().ends_with(&suffix) { + hostname[..hostname.len() - suffix.len()].to_string() + } else { + hostname.to_string() + } +} + +// Context: Converts database row into structured host summary for web display +// Operation: Aggregates host data from multiple sources into unified summary object +pub async fn parse_host_summary(row: sqlx::sqlite::SqliteRow, ctx: &ProjectContext) -> HostSummary { + let ip: String = row.get("ip"); + let hostname: Option = row.get("hostname"); + let ports_str: String = row.get("ports"); + let ad_domain: Option = row.get("ad_domain"); + + // Parse ports count + let ports_count = if ports_str.is_empty() { + 0 + } else { + ports_str.split(',').filter(|s| !s.trim().is_empty()).count() + }; + + // Get combined tags as simple strings for JavaScript + let tags = get_combined_tags(ctx, &ip).await.unwrap_or_default(); + + // Get tags with source info for template rendering + let tags_with_source = get_combined_tags_for_display(ctx, &ip).await.unwrap_or_default(); + + // Get HTTP services count from dedicated table + let http_services_count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM http_services WHERE run_id = ? AND ip = ?" + ) + .bind(&ctx.run_id) + .bind(&ip) + .fetch_one(&ctx.pool) + .await + .unwrap_or(0) as usize; + + // Count from normalized action_outputs table — provided as correlated subquery in all callers + let action_outputs_count: i64 = row.try_get("action_count").unwrap_or(0); + let action_outputs_count = action_outputs_count as usize; + + let hostname_str = hostname.unwrap_or_default(); + let ad_domain_str = ad_domain.unwrap_or_default(); + let short_hostname = strip_domain_suffix(&hostname_str, &ad_domain_str); + + HostSummary { + ip, + hostname: hostname_str, + short_hostname, + ports_count, + tags, + tags_with_source, + ad_domain: ad_domain_str, + full_scan_complete: get_bool_from_row(&row, "full_scan_complete"), + service_scan_complete: get_bool_from_row(&row, "service_scan_complete"), + http_discovery_complete: get_bool_from_row(&row, "http_discovery_complete"), + screenshots_complete: get_bool_from_row(&row, "screenshots_complete"), + http_services_count, + action_outputs_count, + analyzed: get_bool_from_row(&row, "analyzed"), + interesting: get_bool_from_row(&row, "interesting"), + nuclei_reviewed: get_bool_from_row(&row, "nuclei_reviewed"), + has_nuclei_findings: get_bool_from_row(&row, "has_nuclei_findings"), + nuclei_severity_counts: crate::sql::nuclei::NucleiSeverityCounts::default(), + } +} + +// Context: Adapts workflow task information for web interface display +// Operation: Formats task data with duration calculation for template rendering +pub fn convert_task_info(task: &crate::workflow::TaskInfo) -> WebTaskInfo { + let current_time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + let duration_seconds = current_time.saturating_sub(task.start_epoch as u64); + + let duration_str = if duration_seconds < 60 { + format!("{}s", duration_seconds) + } else if duration_seconds < 3600 { + let minutes = duration_seconds / 60; + let seconds = duration_seconds % 60; + format!("{}m {}s", minutes, seconds) + } else { + let hours = duration_seconds / 3600; + let minutes = (duration_seconds % 3600) / 60; + format!("{}h {}m", hours, minutes) + }; + + WebTaskInfo { + id: task.id.clone(), + name: task.name.clone(), + target: task.target.clone(), + duration: duration_str, + } +} + +// Context: Merges action/manual/refined tags for unified host classification +// Operation: Combines all tag sets with deduplication and alphabetical sorting +pub async fn get_combined_tags(ctx: &ProjectContext, ip: &str) -> Result, Box> { + let action_tags = ctx.get_action_tags(ip).await.unwrap_or_default(); + let manual_tags = ctx.get_manual_tags(ip).await?; + let refined_tags = ctx.get_refined_tags(ip).await?; + + let mut combined: std::collections::HashSet = action_tags; + combined.extend(manual_tags); + combined.extend(refined_tags); + + let mut result: Vec = combined.into_iter().collect(); + result.sort(); + Ok(result) +} + +#[derive(Serialize)] +pub struct TagWithSource { + pub name: String, + pub source: String, // "manual" or "refined" +} + +impl std::fmt::Display for TagWithSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name) + } +} + +// Context: Provides combined tags with source attribution for detailed display +// Operation: Merges action/manual/refined tags; action tags always sort first (distinct color in UI) +pub async fn get_combined_tags_with_source(ctx: &ProjectContext, ip: &str) -> Result, Box> { + let action_tags = ctx.get_action_tags(ip).await.unwrap_or_default(); + let manual_tags = ctx.get_manual_tags(ip).await?; + let refined_tags = ctx.get_refined_tags(ip).await?; + + // Action tags first — sorted alpha within group + let mut action_sorted: Vec = action_tags.into_iter().collect(); + action_sorted.sort(); + let mut result: Vec = action_sorted.into_iter() + .map(|name| TagWithSource { name, source: "action".to_string() }) + .collect(); + + // Manual tags next — deduplicated against action tags, sorted alpha within group + let seen: std::collections::HashSet = result.iter().map(|t| t.name.clone()).collect(); + let mut manual_sorted: Vec = manual_tags.into_iter().filter(|t| !seen.contains(t)).collect(); + manual_sorted.sort(); + for name in manual_sorted { + result.push(TagWithSource { name, source: "manual".to_string() }); + } + + // Refined tags last — deduplicated against action+manual, sorted alpha within group + let seen: std::collections::HashSet = result.iter().map(|t| t.name.clone()).collect(); + let mut refined_sorted: Vec = refined_tags.into_iter().filter(|t| !seen.contains(t)).collect(); + refined_sorted.sort(); + for name in refined_sorted { + result.push(TagWithSource { name, source: "refined".to_string() }); + } + + Ok(result) +} + +// Context: Retrieves tags with source metadata optimized for template display +// Operation: Action tags first, then manual (normalized), then refined; action tags have distinct color in UI +pub async fn get_combined_tags_for_display(ctx: &ProjectContext, ip: &str) -> Result, Box> { + let action_tags = ctx.get_action_tags(ip).await.unwrap_or_default(); + let manual_tags_raw = ctx.get_manual_tags(ip).await?; + let refined_tags = ctx.get_refined_tags(ip).await?; + + // Action tags first — sorted alpha within group + let mut action_sorted: Vec = action_tags.into_iter().collect(); + action_sorted.sort(); + let mut result: Vec = action_sorted.into_iter() + .map(|name| TagWithSource { name, source: "action".to_string() }) + .collect(); + + // Normalize manual tags and add next — deduplicated against action tags + let manual_tags_raw_vec: Vec = manual_tags_raw.into_iter().collect(); + let manual_canonical = crate::tag::refinement::apply_hardcoded_rules(&manual_tags_raw_vec); + let seen: std::collections::HashSet = result.iter().map(|t| t.name.clone()).collect(); + let mut manual_sorted: Vec = manual_canonical.into_iter().filter(|t| !seen.contains(t)).collect(); + manual_sorted.sort(); + for name in manual_sorted { + result.push(TagWithSource { name, source: "manual".to_string() }); + } + + // Refined tags last — deduplicated against action+manual + let seen: std::collections::HashSet = result.iter().map(|t| t.name.clone()).collect(); + let mut refined_sorted: Vec = refined_tags.into_iter().filter(|t| !seen.contains(t)).collect(); + refined_sorted.sort(); + for name in refined_sorted { + result.push(TagWithSource { name, source: "refined".to_string() }); + } + + Ok(result) +} + +#[derive(Serialize)] +pub struct HostSummary { + pub ip: String, + pub hostname: String, + pub short_hostname: String, + pub ports_count: usize, + pub tags: Vec, + pub tags_with_source: Vec, + pub ad_domain: String, + pub full_scan_complete: bool, + pub service_scan_complete: bool, + pub http_discovery_complete: bool, + pub screenshots_complete: bool, + pub http_services_count: usize, + pub action_outputs_count: usize, + pub analyzed: bool, + pub interesting: bool, + pub nuclei_reviewed: bool, + pub has_nuclei_findings: bool, + pub nuclei_severity_counts: crate::sql::nuclei::NucleiSeverityCounts, +} + +#[derive(Serialize)] +pub struct PortInfo { + pub port: String, + pub protocol: String, + pub state: String, + pub service: String, + pub version: String, + pub extrainfo: String, + pub cpe: String, +} + +// Context: Converts XML-parsed port structs into the web layer PortInfo type +// Operation: Combines product + version strings and picks first CPE for display +pub fn xml_ports_to_port_info(ports: Vec) -> Vec { + ports.into_iter().map(|p| { + let version = match (p.product.is_empty(), p.version.is_empty()) { + (false, false) => format!("{} {}", p.product, p.version), + (false, true) => p.product, + (true, false) => p.version, + (true, true) => String::new(), + }; + let service = crate::tag::nmap_parsing::normalize_service_name(&p.service_name); + PortInfo { + port: p.port, + protocol: p.protocol, + state: p.state, + service, + version, + extrainfo: p.extrainfo, + cpe: p.cpe.into_iter().next().unwrap_or_default(), + } + }).collect() +} + +#[derive(Serialize)] +pub struct ActionOutput { + pub name: String, + pub command: String, + pub output: String, + pub exit_code: i64, + pub duration_ms: u64, + pub tagged: bool, +} + +#[derive(Serialize)] +pub struct ScreenshotInfo { + pub filename: String, + pub port: String, + pub protocol: String, + pub url: String, +} + +#[derive(Serialize)] +pub struct HostProgress { + pub full_scan: bool, + pub service_scan: bool, + pub http_discovery: bool, + pub web_enumeration: bool, + pub screenshots: bool, + pub completion_percentage: f32, +} + +#[derive(Serialize)] +pub struct WebTaskInfo { + pub id: String, + pub name: String, + pub target: String, + pub duration: String, +} + +// Context: Extracts session token from HTTP headers for authentication +// Operation: Parses cookie header and returns session token if found +pub fn get_session_token_from_headers(headers: &axum::http::HeaderMap) -> Option { + if let Some(cookie_header) = headers.get(axum::http::header::COOKIE) { + if let Ok(cookie_str) = cookie_header.to_str() { + for cookie in cookie_str.split(';') { + let cookie = cookie.trim(); + if let Some(value) = cookie.strip_prefix("rofmad_session=") { + return Some(value.to_string()); + } + } + } + } + None +} \ No newline at end of file diff --git a/src/web/mod.rs b/src/web/mod.rs new file mode 100755 index 0000000..0c5cb2b --- /dev/null +++ b/src/web/mod.rs @@ -0,0 +1,135 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// src/web/mod.rs - Updated with interface binding support + +pub mod auth; +pub mod handlers; +use handlers::{reset_host_state, force_retag_host, retake_screenshot, run_nuclei_scan}; + +use axum::{ + Router, + routing::{get, post}, + middleware, +}; +use std::error::Error; +use crate::sql::ProjectContext; +use crate::utilities::outputs::tag_info; +use tokio::sync::watch; + +// Context: Provides web interface for ROFMAD system management and control +// Operation: Starts Axum web server with all routes and middleware configured +pub async fn run_server_on_interface( + ctx: ProjectContext, + interface_ip: &str, + port: u16, + mut shutdown_rx: watch::Receiver +) -> Result<(), Box> { + // Create separate routers for different security levels + let admin_router = Router::new() + .route("/run-custom-command", post(handlers::run_custom_command)) + .route("/actions/execute", post(handlers::action_execute)) + .route("/launch-discovery-tool", post(handlers::launch_discovery_tool)) + .route("/api/kill-task/:id", post(handlers::kill_task)) + .route("/add-target", post(handlers::add_individual_target)) + .route("/add-subnet", post(handlers::add_subnet_with_masscan)) + .route("/parse-targets-file", post(handlers::parse_targets_file)) + .route("/actions/builder", get(handlers::action_builder_page)) + .route("/add-blacklist", post(handlers::add_blacklist_entry)) + .route("/remove-blacklist", post(handlers::remove_blacklist_entry)) + .route("/host/reset", post(reset_host_state)) + .route("/host/retag", post(force_retag_host)) + .route("/host/screenshot", post(retake_screenshot)) + .route("/host/nuclei", post(run_nuclei_scan)) + .route("/create-player", post(handlers::create_player_user)) + .route("/manage-player", post(handlers::manage_player_user)) + .route("/api/user-role-check", post(handlers::user_role_check)) + .route("/api/concurrency/limits", post(handlers::update_concurrency_limits)) + .layer(middleware::from_fn_with_state(ctx.clone(), auth::admin_required_middleware)); + + let player_router = Router::new() + .route("/static/style.css", get(handlers::serve_css)) + .route("/", get(handlers::dashboard)) + .route("/scanners", get(handlers::scanners_page)) + .route("/hosts", get(handlers::hosts_list)) + .route("/hosts/rows", get(handlers::hosts_rows)) + .route("/hosts/:ip", get(handlers::host_detail)) + .route("/tags", get(handlers::get_available_tags)) + .route("/settings", get(handlers::settings)) + .route("/logs", get(handlers::logs)) + .route("/screenshots/:filename", get(handlers::screenshot)) + .route("/mark-analyzed", post(handlers::mark_host_analyzed)) + .route("/host/:ip/analysis-status", get(handlers::get_host_analysis_status)) + .route("/mark-interesting", post(handlers::mark_host_interesting)) + .route("/host/:ip/interesting-status", get(handlers::get_host_interesting_status)) + .route("/queue", get(handlers::queued_tasks_page)) + .route("/api/task-status", get(handlers::api_task_status)) + .route("/api/hosts-ready-analysis", get(handlers::api_hosts_ready_analysis)) + .route("/api/nuclei-findings", get(handlers::api_nuclei_findings)) + .route("/export-hosts-by-tags", post(handlers::export_hosts_by_tags)) + .route("/run-nuclei-bulk", post(handlers::run_nuclei_bulk)) + .route("/api/nuclei-bulk-preview", get(handlers::nuclei_bulk_preview)) + .route("/api/concurrency-status", get(handlers::api_concurrency_status)) + .route("/api/discovery-stats", get(handlers::api_discovery_stats)) + .route("/host/add-manual-tag", post(handlers::add_manual_tag)) + .route("/host/remove-manual-tag", post(handlers::remove_manual_tag)) + .route("/host/:ip/manual-tags", get(handlers::get_manual_tags)) + .route("/api/all-tags", get(handlers::api_all_tags)) + .route("/actions/preview", post(handlers::action_preview)) + .route("/api/blacklist", get(handlers::get_blacklist_entries)) + .route("/api/active-tasks", get(handlers::api_active_tasks)) + .route("/api/queue-tasks", get(handlers::api_queue_tasks)) + .route("/task/:id", get(handlers::get_task_detail)) + .route("/task/:id/stream", get(handlers::get_task_output_stream)) + .route("/api/task/:id/output", get(handlers::api_task_output)) + .route("/api/logs", get(handlers::api_logs)) + .route("/api/task-history", get(handlers::api_task_history)) + .route("/task-output/:id", get(handlers::task_output_page)) + .route("/api/queued-tasks", get(handlers::api_queued_tasks)) + .route("/api/hosts/:ip/ports/:port/retry-probe", post(handlers::retry_http_probe)) + .route("/mark-nuclei-reviewed", post(handlers::mark_host_nuclei_reviewed)) + .route("/host/:ip/nuclei-reviewed-status", get(handlers::get_host_nuclei_reviewed_status)) + .layer(middleware::from_fn_with_state(ctx.clone(), auth::authenticated_middleware)); + + let app = Router::new() + // Public routes (no authentication required) - these must come first + .route("/login", get(handlers::login_page).post(handlers::login_submit)) + .route("/setup", get(handlers::setup_page).post(handlers::setup_submit)) + // Merge protected routers + .merge(admin_router) + .merge(player_router) + .with_state(ctx); + + let bind_address = format!("{}:{}", interface_ip, port); + let listener = tokio::net::TcpListener::bind(&bind_address).await + .map_err(|e| format!("Failed to bind to {}: {}", bind_address, e))?; + + tag_info(&format!("Web interface available at http://{} (admin username: admin)", bind_address)); + + // Create a future that completes when shutdown is signaled + let server = axum::serve(listener, app); + + tokio::select! { + result = server => { + if let Err(e) = result { + tag_info(&format!("Web server error: {}", e)); + } + } + _ = shutdown_rx.changed() => { + tag_info("Web interface shutting down..."); + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/workflow/concurrency.rs b/src/workflow/concurrency.rs new file mode 100755 index 0000000..ca494d5 --- /dev/null +++ b/src/workflow/concurrency.rs @@ -0,0 +1,472 @@ +// src/concurrency.rs - Queue-based concurrency management for ROFMAD + +use tokio::sync::{Semaphore, OwnedSemaphorePermit}; +use tokio::time::{Duration, Instant}; +use once_cell::sync::Lazy; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::collections::HashMap; +use serde::{Deserialize, Serialize}; + +/// Types of operations that require semaphore management +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum OperationType { + Global, + NmapFullTcp, + NmapService, + SubnetPhase1, + SubnetPhase2, + HttpDiscovery, +} + +impl OperationType { + // Context: Network resource estimation for different operation types + // Operation: Returns estimated TCP connection usage for capacity planning + pub fn estimated_tcp_connections(&self) -> usize { + match self { + OperationType::Global => 10, + OperationType::NmapFullTcp => 1500, + OperationType::NmapService => 200, + OperationType::SubnetPhase1 => 400, + OperationType::SubnetPhase2 => 1200, + OperationType::HttpDiscovery => 75, + } + } + + pub fn all() -> &'static [OperationType] { + &[ + OperationType::Global, + OperationType::NmapFullTcp, + OperationType::NmapService, + OperationType::SubnetPhase1, + OperationType::SubnetPhase2, + OperationType::HttpDiscovery, + ] + } +} + +impl std::fmt::Display for OperationType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OperationType::Global => write!(f, "Global"), + OperationType::NmapFullTcp => write!(f, "Nmap Full TCP"), + OperationType::NmapService => write!(f, "Nmap Service"), + OperationType::SubnetPhase1 => write!(f, "Subnet Phase 1"), + OperationType::SubnetPhase2 => write!(f, "Subnet Phase 2"), + OperationType::HttpDiscovery => write!(f, "HTTP Discovery"), + } + } +} + +/// Configuration for semaphore limits +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConcurrencyConfig { + pub global: usize, + pub nmap_full_tcp: usize, + pub nmap_service: usize, + pub subnet_phase1: usize, + pub subnet_phase2: usize, + pub http_discovery: usize, + /// Minimum delay (ms) enforced between successive task launches, to spread out + /// bursts (e.g. when masscan discovery fires hooks for many hosts at once). + #[serde(default = "default_launch_stagger_ms")] + pub launch_stagger_ms: u64, +} + +fn default_launch_stagger_ms() -> u64 { + 2000 +} + +impl ConcurrencyConfig { + pub fn conservative() -> Self { + Self { + global: 50, + nmap_full_tcp: 5, + nmap_service: 5, + subnet_phase1: 10, + subnet_phase2: 3, + http_discovery: 10, + launch_stagger_ms: 2000, + } + } + + pub fn aggressive() -> Self { + Self { + global: 100, + nmap_full_tcp: 10, + nmap_service: 10, + subnet_phase1: 1, + subnet_phase2: 1, + http_discovery: 10, + launch_stagger_ms: 0, + } + } + + pub fn get_limit(&self, op_type: OperationType) -> usize { + match op_type { + OperationType::Global => self.global, + OperationType::NmapFullTcp => self.nmap_full_tcp, + OperationType::NmapService => self.nmap_service, + OperationType::SubnetPhase1 => self.subnet_phase1, + OperationType::SubnetPhase2 => self.subnet_phase2, + OperationType::HttpDiscovery => self.http_discovery, + } + } + + pub fn set_limit(&mut self, op_type: OperationType, value: usize) { + match op_type { + OperationType::Global => self.global = value, + OperationType::NmapFullTcp => self.nmap_full_tcp = value, + OperationType::NmapService => self.nmap_service = value, + OperationType::SubnetPhase1 => self.subnet_phase1 = value, + OperationType::SubnetPhase2 => self.subnet_phase2 = value, + OperationType::HttpDiscovery => self.http_discovery = value, + } + } +} + +/// Global semaphores for each operation type. +/// Uses Arc so we can clone the Arc out of the lock before awaiting, +/// and OwnedSemaphorePermit (no lifetime) so ConcurrencyPermit needs no lifetime param. +pub struct ConcurrencyManager { + semaphores: HashMap>, + config: Mutex, + usage_counters: HashMap, + // Permits held to enforce a reduced limit. When the limit is raised these are released. + held_reduction_permits: Mutex>>, +} + +impl ConcurrencyManager { + fn new(config: ConcurrencyConfig) -> Self { + let mut semaphores = HashMap::new(); + let mut usage_counters = HashMap::new(); + + for &op_type in OperationType::all() { + let limit = config.get_limit(op_type); + semaphores.insert(op_type, Arc::new(Semaphore::new(limit))); + usage_counters.insert(op_type, AtomicUsize::new(0)); + } + + Self { + semaphores, + config: Mutex::new(config), + usage_counters, + held_reduction_permits: Mutex::new(HashMap::new()), + } + } + + // Context: Controlling concurrent task execution through permits + // Operation: Acquires owned permit for operation type, blocking if limit reached. + // Clones the Arc under a brief lock so no lock is held across the await. + pub async fn acquire(&self, op_type: OperationType) -> Result { + let arc = self.semaphores + .get(&op_type) + .ok_or_else(|| format!("No semaphore for operation type: {:?}", op_type))? + .clone(); + + let permit = arc + .acquire_owned() + .await + .map_err(|_| "Failed to acquire semaphore permit".to_string())?; + + self.usage_counters + .get(&op_type) + .unwrap() + .fetch_add(1, Ordering::SeqCst); + + Ok(ConcurrencyPermit { + op_type, + _permit: permit, + }) + } + + // Context: Runtime concurrency limit adjustment + // Operation: Increases or decreases the effective limit for an operation type immediately. + // Increases add permits directly. Decreases absorb available permits so that + // in-flight work at the old limit completes naturally while no new slots open. + pub fn set_limit(&self, op_type: OperationType, new_limit: usize) { + let mut config = self.config.lock().unwrap(); + let current = config.get_limit(op_type); + let semaphore = self.semaphores.get(&op_type).unwrap(); + + if new_limit > current { + let delta = new_limit - current; + // Release held reduction permits first, then add fresh ones for the remainder. + let mut held = self.held_reduction_permits.lock().unwrap(); + let entry = held.entry(op_type).or_default(); + let release_from_held = delta.min(entry.len()); + for _ in 0..release_from_held { + entry.pop(); // drop permit → semaphore capacity restored + } + let remaining = delta - release_from_held; + if remaining > 0 { + semaphore.add_permits(remaining); + } + } else if new_limit < current { + let to_absorb = current - new_limit; + let mut held = self.held_reduction_permits.lock().unwrap(); + let entry = held.entry(op_type).or_default(); + for _ in 0..to_absorb { + if let Ok(p) = semaphore.clone().try_acquire_owned() { + entry.push(p); + } + // If try_acquire fails (all permits in use), we skip — in-flight work + // will return permits eventually, but we can't block here. The stored + // config value is updated regardless so future acquisitions see the new limit. + } + } + + config.set_limit(op_type, new_limit); + } + + pub fn get_config(&self) -> ConcurrencyConfig { + self.config.lock().unwrap().clone() + } + + pub fn get_launch_stagger_ms(&self) -> u64 { + self.config.lock().unwrap().launch_stagger_ms + } + + pub fn set_launch_stagger_ms(&self, ms: u64) { + self.config.lock().unwrap().launch_stagger_ms = ms; + } + + pub fn get_usage_stats(&self) -> ConcurrencyStats { + let config = self.config.lock().unwrap(); + let mut stats = HashMap::new(); + + for &op_type in OperationType::all() { + let total_limit = config.get_limit(op_type); + let available = self.semaphores + .get(&op_type) + .map(|s| s.available_permits()) + .unwrap_or(0); + // available_permits counts permits that are free AND not held for reduction + let in_use = self.usage_counters + .get(&op_type) + .map(|c| c.load(Ordering::SeqCst)) + .unwrap_or(0); + + stats.insert(op_type, OperationStats { + total_limit, + in_use, + available, + estimated_tcp_connections: in_use * op_type.estimated_tcp_connections(), + }); + } + + let total_tcp_estimate = stats.values() + .map(|s| s.estimated_tcp_connections) + .sum(); + + let display_groups = build_display_groups(&stats); + + drop(config); + ConcurrencyStats { + operation_stats: stats, + total_estimated_tcp_connections: total_tcp_estimate, + display_groups, + } + } +} + +// Context: Builds the grouped boxes shown in the Dashboard/Queue concurrency widgets — +// Nmap (NmapFullTcp + NmapService) and Subnet (SubnetPhase1 + SubnetPhase2) are merged +// into single boxes, and Nuclei/Screenshot reflect their dedicated global semaphores +fn build_display_groups(stats: &HashMap) -> Vec { + let get = |op: OperationType| stats.get(&op).cloned().unwrap_or(OperationStats { + total_limit: 0, in_use: 0, available: 0, estimated_tcp_connections: 0, + }); + + let global = get(OperationType::Global); + let nmap_full = get(OperationType::NmapFullTcp); + let nmap_service = get(OperationType::NmapService); + let subnet1 = get(OperationType::SubnetPhase1); + let subnet2 = get(OperationType::SubnetPhase2); + let http_discovery = get(OperationType::HttpDiscovery); + + let nuclei_available = super::constants::NUCLEI_SEMAPHORE.available_permits(); + let screenshot_available = super::constants::SCREENSHOT_SEMAPHORE.available_permits(); + + vec![ + ConcurrencyDisplayGroup { + name: "Global".to_string(), + in_use: global.in_use, + total_limit: global.total_limit, + }, + ConcurrencyDisplayGroup { + name: "Nmap".to_string(), + in_use: nmap_full.in_use + nmap_service.in_use, + total_limit: nmap_full.total_limit + nmap_service.total_limit, + }, + ConcurrencyDisplayGroup { + name: "Subnet".to_string(), + in_use: subnet1.in_use + subnet2.in_use, + total_limit: subnet1.total_limit + subnet2.total_limit, + }, + ConcurrencyDisplayGroup { + name: "HTTP Discovery".to_string(), + in_use: http_discovery.in_use, + total_limit: http_discovery.total_limit, + }, + ConcurrencyDisplayGroup { + name: "Nuclei".to_string(), + in_use: super::constants::NUCLEI_SEMAPHORE_LIMIT.saturating_sub(nuclei_available), + total_limit: super::constants::NUCLEI_SEMAPHORE_LIMIT, + }, + ConcurrencyDisplayGroup { + name: "Screenshot".to_string(), + in_use: super::constants::SCREENSHOT_SEMAPHORE_LIMIT.saturating_sub(screenshot_available), + total_limit: super::constants::SCREENSHOT_SEMAPHORE_LIMIT, + }, + ] +} + +/// RAII permit — no lifetime parameter because it owns the permit via OwnedSemaphorePermit. +pub struct ConcurrencyPermit { + op_type: OperationType, + _permit: OwnedSemaphorePermit, +} + +impl Drop for ConcurrencyPermit { + fn drop(&mut self) { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + if let Some(manager) = guard.as_ref() { + if let Some(counter) = manager.usage_counters.get(&self.op_type) { + counter.fetch_sub(1, Ordering::SeqCst); + } + } + } +} + +/// Statistics for a single operation type +#[derive(Debug, Clone, Serialize)] +pub struct OperationStats { + pub total_limit: usize, + pub in_use: usize, + pub available: usize, + pub estimated_tcp_connections: usize, +} + +/// A grouped box for the Dashboard/Queue concurrency widgets — combines related +/// operation types (e.g. Nmap, Subnet) or dedicated semaphores (Nuclei, Screenshot) +/// into a single named entry with one consistent key for both template rendering +/// and live JSON updates. +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencyDisplayGroup { + pub name: String, + pub in_use: usize, + pub total_limit: usize, +} + +/// Overall concurrency statistics +#[derive(Debug, Clone, Serialize)] +pub struct ConcurrencyStats { + pub operation_stats: HashMap, + pub total_estimated_tcp_connections: usize, + pub display_groups: Vec, +} + +/// Global concurrency manager instance +pub static CONCURRENCY_MANAGER: Lazy>>> = + Lazy::new(|| std::sync::RwLock::new(None)); + +/// Timestamp of the last task launch, used to enforce a minimum spacing between +/// launches across all task types (see `stagger_launch`). +static LAST_LAUNCH: Lazy>> = + Lazy::new(|| tokio::sync::Mutex::new(None)); + +// Context: Global concurrency system initialization +// Operation: Sets up concurrency manager with specified configuration +pub fn initialize_concurrency(config: ConcurrencyConfig) { + let manager = ConcurrencyManager::new(config.clone()); + *CONCURRENCY_MANAGER.write().unwrap() = Some(Arc::new(manager)); + crate::utilities::tag_info(&format!( + "Concurrency initialized: Global:{} Full_TCP:{} Service:{} Sub_P1:{} Sub_P2:{} HTTP:{}", + config.global, config.nmap_full_tcp, config.nmap_service, + config.subnet_phase1, config.subnet_phase2, config.http_discovery + )); +} + +// Context: Global permit acquisition — Arc clone releases the lock before the async acquire +// Operation: Acquires permit from global manager +pub async fn acquire_permit(op_type: OperationType) -> Result { + let manager = { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + guard.as_ref() + .ok_or("Concurrency manager not initialized")? + .clone() + }; + manager.acquire(op_type).await +} + +// Context: Runtime limit adjustment from settings UI +// Operation: Changes the effective limit for one operation type and applies it immediately +pub fn set_concurrency_limit(op_type: OperationType, new_limit: usize) { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + if let Some(manager) = guard.as_ref() { + manager.set_limit(op_type, new_limit); + } +} + +// Context: Returns current configured limits for display / persistence +pub fn get_concurrency_config() -> Option { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + guard.as_ref().map(|m| m.get_config()) +} + +// Context: Runtime adjustment of the launch-stagger delay from the settings UI +pub fn set_launch_stagger_ms(ms: u64) { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + if let Some(manager) = guard.as_ref() { + manager.set_launch_stagger_ms(ms); + } +} + +// Context: Burst smoothing — called by default_spawn after permits are acquired +// Operation: Blocks until at least `launch_stagger_ms` has elapsed since the previous +// launch, so that bursts of simultaneously-ready tasks (e.g. after masscan +// discovery fires hooks for many hosts at once) start with spacing between +// them rather than all at the same instant. Once started, tasks still run +// fully concurrently for their own duration. +pub async fn stagger_launch() { + let stagger_ms = { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + guard.as_ref().map(|m| m.get_launch_stagger_ms()).unwrap_or(0) + }; + if stagger_ms == 0 { + return; + } + let stagger = Duration::from_millis(stagger_ms); + + let mut last = LAST_LAUNCH.lock().await; + let now = Instant::now(); + if let Some(prev) = *last { + let elapsed = now.duration_since(prev); + if elapsed < stagger { + tokio::time::sleep(stagger - elapsed).await; + } + } + *last = Some(Instant::now()); +} + +// Context: Public interface for concurrency monitoring +// Operation: Returns current usage statistics from global manager +pub fn get_concurrency_stats() -> ConcurrencyStats { + let guard = CONCURRENCY_MANAGER.read().unwrap(); + match guard.as_ref() { + Some(manager) => { + let stats = manager.get_usage_stats(); + drop(guard); + stats + } + None => { + drop(guard); + ConcurrencyStats { + operation_stats: HashMap::new(), + total_estimated_tcp_connections: 0, + display_groups: Vec::new(), + } + } + } +} diff --git a/src/workflow/conditions.rs b/src/workflow/conditions.rs new file mode 100755 index 0000000..10572f7 --- /dev/null +++ b/src/workflow/conditions.rs @@ -0,0 +1,147 @@ +// src/workflow/conditions.rs +use std::sync::atomic::Ordering; +use crate::sql::hosts::HostSnapshot; +use crate::utilities::tag_debug; +use crate::VERBOSE; +use super::types::Condition; + +// Context: Port-based workflow condition evaluation with logical operators +// Operation: Parses port conditions with AND/OR logic and checks against available ports +pub fn parse_port_condition(condition_str: &str, ports: &[String]) -> bool { + let condition_lower = condition_str.to_lowercase(); + + if condition_lower.contains(" and ") { + // Split by AND and check all ports exist + let required_ports: Vec<&str> = condition_lower.split(" and ").map(str::trim).collect(); + required_ports.iter().all(|&required_port| { + ports.iter().any(|p| p == required_port) + }) + } else if condition_lower.contains(" or ") { + // Split by OR and check any port exists + let any_ports: Vec<&str> = condition_lower.split(" or ").map(str::trim).collect(); + any_ports.iter().any(|&any_port| { + ports.iter().any(|p| p == any_port) + }) + } else { + // Single port condition (backward compatibility) + ports.iter().any(|p| p == condition_str) + } +} + +// Context: Tag-based workflow condition evaluation with logical operators +// Operation: Parses tag conditions with AND/OR logic using exact case-insensitive matching +pub fn parse_tag_condition(condition_str: &str, tags: &[String]) -> bool { + let condition_lower = condition_str.to_lowercase(); + + if condition_lower.contains(" and ") { + let required_tags: Vec<&str> = condition_lower.split(" and ").map(str::trim).collect(); + required_tags.iter().all(|&required_tag| { + tags.iter().any(|t| t.to_lowercase() == required_tag) + }) + } else if condition_lower.contains(" or ") { + let any_tags: Vec<&str> = condition_lower.split(" or ").map(str::trim).collect(); + any_tags.iter().any(|&any_tag| { + tags.iter().any(|t| t.to_lowercase() == any_tag) + }) + } else { + tags.iter().any(|t| t.to_lowercase() == condition_lower) + } +} + +// Context: Complete workflow condition evaluation system +// Operation: Evaluates all condition types using pre-fetched HostSnapshot — no DB calls +pub fn evaluate_condition( + condition: &Condition, + ports: &[String], + tags: &[String], + snapshot: &HostSnapshot, +) -> bool { + // Reconnaissance completion gating — derived from snapshot, no DB call + if let Some(needs_recon_complete) = condition.reconnaissance_complete { + let is_complete = snapshot.full_scan_complete + && snapshot.service_scan_complete + && snapshot.http_discovery_complete; + if needs_recon_complete && !is_complete { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: reconnaissance not complete for {}", snapshot.ip)); + } + return false; + } + } + + // Enhanced port-based logic with AND/OR support + if let Some(port_condition) = &condition.contains { + if !parse_port_condition(port_condition, ports) { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: port condition '{}' not satisfied in {:?}", port_condition, ports)); + } + return false; + } + } + + if let Some(port) = &condition.equals { + if ports.len() != 1 || ports[0] != *port { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: ports don't equal {} (found: {:?})", port, ports)); + } + return false; + } + } + + // Enhanced tag-based evaluation with AND/OR support + if let Some(tag_condition) = &condition.has_tag { + if !parse_tag_condition(tag_condition, tags) { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: tag condition '{}' not satisfied in {:?}", tag_condition, tags)); + } + return false; + } + } + + // Negative tag check — fires only when the tag condition is NOT present + if let Some(tag_condition) = &condition.has_not_tag { + if parse_tag_condition(tag_condition, tags) { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: has_not_tag '{}' is present in {:?}", tag_condition, tags)); + } + return false; + } + } + + // Service-specific conditions — derived from snapshot, no DB call + if let Some(needs_http) = condition.has_http_service { + if needs_http && !snapshot.has_http_services { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: HTTP service required but not found for {}", snapshot.ip)); + } + return false; + } + if !needs_http && snapshot.has_http_services { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition failed: HTTP service forbidden but found for {}", snapshot.ip)); + } + return false; + } + } + + // Dynamic service check — matches against port_services values (supports AND/OR) + if let Some(svc_condition) = &condition.has_service { + let port_services = crate::sql::hosts::parse_port_services(&snapshot.port_services); + let service_values: Vec = port_services.values().cloned().collect(); + if !parse_tag_condition(svc_condition, &service_values) { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!( + "Condition failed: service condition '{}' not satisfied; found services: {:?}", + svc_condition, service_values + )); + } + return false; + } + } + + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Condition passed for {}", snapshot.ip)); + } + + true +} diff --git a/src/workflow/constants.rs b/src/workflow/constants.rs new file mode 100755 index 0000000..f0cd2db --- /dev/null +++ b/src/workflow/constants.rs @@ -0,0 +1,62 @@ +// src/workflow/constants.rs +use std::collections::{HashSet, HashMap}; +use std::sync::{Arc, Mutex, RwLock, atomic::{AtomicBool, AtomicUsize}}; +use once_cell::sync::Lazy; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; +use super::types::{Hook, TaskInfo, SpecificActionStep, CompletedTaskRecord}; + +pub static FIRED_HOOKS: Lazy>> = Lazy::new(|| Mutex::new(HashSet::new())); + +pub static RUNTIME_ACTIONS: Lazy>> = + Lazy::new(|| Mutex::new(Vec::new())); + +/// When true, no semaphore‐driven tasks may actually start. +pub static INIT_IN_PROGRESS: Lazy = Lazy::new(|| AtomicBool::new(true)); + +pub static RESUME_MODE: Lazy = Lazy::new(|| AtomicBool::new(false)); + +/// Block hooks during masscan +pub static MASSCAN_BLOCK_HOOKS: Lazy = Lazy::new(|| AtomicBool::new(false)); + +/// Prevent duplicate shutdown calls +pub static SHUTDOWN_IN_PROGRESS: Lazy = Lazy::new(|| AtomicBool::new(false)); + +/// Masscan status for UI +pub static MASSCAN_STATUS: Lazy>> = Lazy::new(|| RwLock::new(None)); + +/// Concurrency limiter +pub const MAX_CONCURRENT_TASKS: usize = 200; +pub static TASK_SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(MAX_CONCURRENT_TASKS)); + +/// Global Chrome slot limiter — at most 5 Chromium processes alive simultaneously +pub const SCREENSHOT_SEMAPHORE_LIMIT: usize = 5; +pub static SCREENSHOT_SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(SCREENSHOT_SEMAPHORE_LIMIT)); + +/// Global nuclei slot limiter — nuclei is as resource-demanding as masscan, so only +/// one instance is allowed to run at a time across the whole project. +pub const NUCLEI_SEMAPHORE_LIMIT: usize = 1; +pub static NUCLEI_SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(NUCLEI_SEMAPHORE_LIMIT)); + +/// Per-host nuclei semaphores — at most 2 concurrent nuclei per IP. +/// Arc is cloned before any await so the Mutex is never held across an await point. +/// All tasks for the same IP share the same Arc → FIFO order via Tokio's fair semaphore. +pub static NUCLEI_HOST_SEMAPHORES: Lazy>>> = + Lazy::new(|| Mutex::new(HashMap::new())); + +/// Count of queued tasks +pub static QUEUED_TASKS: Lazy = Lazy::new(|| AtomicUsize::new(0)); + +pub static WEB_ONLY_MODE: Lazy = Lazy::new(|| AtomicBool::new(false)); + +pub static GLOBAL_CANCEL_TOKEN: Lazy>> = + Lazy::new(|| RwLock::new(None)); + +pub static ACTIVE_TASKS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); +pub static QUEUED_TASK_INFOS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); + +pub static CURRENT_RUN_ID: Lazy>> = + Lazy::new(|| RwLock::new(None)); + +pub static TASK_HISTORY_TX: Lazy>>> = + Lazy::new(|| RwLock::new(None)); \ No newline at end of file diff --git a/src/workflow/dispatch.rs b/src/workflow/dispatch.rs new file mode 100755 index 0000000..0c038d0 --- /dev/null +++ b/src/workflow/dispatch.rs @@ -0,0 +1,328 @@ +// src/workflow/dispatch.rs +use std::sync::atomic::Ordering; +use crate::enumeration::{nmap_full_port_detection_scan, nmap_service_scan, subnet_phase1_scan, subnet_phase2_scan}; +use super::concurrency::OperationType; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::types::Hook; +use super::constants::{FIRED_HOOKS, ACTIVE_TASKS, QUEUED_TASK_INFOS, GLOBAL_CANCEL_TOKEN}; +use super::spawning::default_spawn; +use super::screenshots::dispatch_individual_screenshots; + +// Context: Central event dispatching system for workflow automation +// Operation: Routes workflow events to appropriate scan tasks with deduplication +pub fn dispatch_event(event: Hook, data: &str, ctx: &ProjectContext) { + // Deduplication check + let key = (event, data.to_string()); + { + let mut fired = FIRED_HOOKS.lock().unwrap(); + if fired.contains(&key) { + return; // Already fired + } + fired.insert(key); + } + { + let name = match event { + Hook::Ip => "nmap_full_port_detection_scan", + Hook::ServiceDiscoveryReady => "nmap_service_scan", + Hook::Subnet => "subnet_phase1_scan", + Hook::ServiceScanComplete => "", // Will trigger HTTP Phase 2 + Hook::HttpAutoTagComplete => "", // Will trigger screenshots + Hook::ReconnaissanceComplete => "", // Will enable specific_actions + _ => "", + }; + if !name.is_empty() { + let active = ACTIVE_TASKS.lock().unwrap(); + if active.values().any(|t| t.name == name && t.target == data) { + return; + } + let queued = QUEUED_TASK_INFOS.lock().unwrap(); + if queued.values().any(|t| t.name == name && t.target == data) { + return; + } + } + } + + match event { + + Hook::Ip => { + let ip = data.to_string(); + let ctx2 = ctx.clone(); + + tokio::spawn(async move { + // Check if already complete + if let Ok(complete) = ctx2.is_full_scan_complete(&ip).await { + if complete { + tag_debug(&format!("Full port scan already complete for {}, skipping", ip)); + return; + } + } + + // Check if host is marked as slow + if let Ok(slow) = ctx2.is_host_slow(&ip).await { + if slow { + tag_debug(&format!("Host {} marked as slow, skipping full port scan", ip)); + return; + } + } + + default_spawn( + "nmap_full_port_detection_scan", + ip.clone(), + OperationType::NmapFullTcp, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx2.clone(); + let ip_clone = ip.clone(); + async move { + // nmap_full_port_detection_scan now handles timeout internally + match nmap_full_port_detection_scan(&ctx2, &ip_clone, tx_opt).await { + Ok(_) => { + // Success or timeout handled internally + } + Err(e) => { + tag_info(&format!("nmap_full_port_detection_scan failed on {}: {}", ip_clone, e)); + } + } + } + }, + None, // Remove timeout parameter - handled internally now + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + }); + } + + Hook::ServiceDiscoveryReady => { + let ip = data.to_string(); + let ctx2 = ctx.clone(); + + tokio::spawn(async move { + if let Ok(complete) = ctx2.is_service_scan_complete(&ip).await { + if complete { + tag_debug(&format!("Service scan already complete for {}, skipping", ip)); + return; + } + } + + default_spawn( + "nmap_service_scan", + ip.clone(), + OperationType::NmapService, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx2.clone(); + let ip_clone = ip.clone(); + async move { + match nmap_service_scan(&ctx2, &ip_clone, tx_opt).await { + Ok(_) => {}, + Err(e) => tag_info(&format!("nmap_service_scan failed on {}: {}", ip_clone, e)), + } + } + }, + None, + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + }); + } + + Hook::SubnetPhase1Complete => { // New hook + let subnet = data.to_string(); + let ctx2 = ctx.clone(); + // Direct call (no outer tokio::spawn) so register_queued_task runs synchronously + // and phase2 appears in QUEUED_TASK_INFOS immediately. + default_spawn( + "subnet_phase2_scan", + subnet.clone(), + OperationType::SubnetPhase2, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx2.clone(); + async move { + match subnet_phase2_scan(&ctx2, subnet.clone(), tx_opt).await { + Ok(_) => {}, + Err(e) => tag_info(&format!("subnet_phase2_scan failed on {}: {}", subnet, e)), + } + } + }, + None, + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + } + + // Update the Hook::HttpAutoTagComplete case: + Hook::HttpAutoTagComplete => { + let ip = data.to_string(); + let ctx2 = ctx.clone(); + + tokio::spawn(async move { + // Trigger refinement now that ALL automated tags are collected + if let Err(e) = crate::tag::refine_automated_tags(&ctx2, &ip).await { + tag_debug(&format!("Tag refinement failed for {}: {}", ip, e)); + } + + // Check screenshots + let screenshots_complete = sqlx::query_scalar("SELECT screenshots_complete FROM targets WHERE run_id = ? AND ip = ?") + .bind(&ctx2.run_id) + .bind(&ip) + .fetch_optional(&ctx2.pool) + .await + .unwrap_or(None) + .unwrap_or(false); + + if !screenshots_complete { + tag_info(&format!("HTTP discovery complete for {} - triggering screenshots", ip)); + dispatch_individual_screenshots(&ip, &ctx2); + } + + check_and_dispatch_reconnaissance_complete(&ip, &ctx2).await; + }); + } + + Hook::Subnet => { + let subnet = data.to_string(); + let ctx2 = ctx.clone(); + // Direct call (no outer tokio::spawn) so register_queued_task runs synchronously + // while MASSCAN_BLOCK_HOOKS is still true — task is immediately visible in QUEUED. + default_spawn( + "subnet_phase1_scan", + subnet.clone(), + OperationType::SubnetPhase1, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx2.clone(); + async move { + match subnet_phase1_scan(&ctx2, subnet.clone(), tx_opt).await { + Ok(_) => {}, + Err(e) => tag_info(&format!("subnet_phase1_scan failed on {}: {}", subnet, e)), + } + } + }, + None, + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); + } + + Hook::Ports => { + } + + Hook::ServiceScanComplete => { + let ip = data.to_string(); + let ctx2 = ctx.clone(); + + tokio::spawn(async move { + if let Ok(complete) = ctx2.is_http_discovery_complete(&ip).await { + if complete { + tag_debug(&format!("HTTP discovery already complete for {}, skipping", ip)); + return; + } + } + + tag_info(&format!("Service scan complete for {} - triggering HTTP discovery", ip)); + dispatch_http_discovery(&ip, &ctx2); + }); + } + + Hook::ReconnaissanceComplete => { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Reconnaissance complete for: {}", data)); + } + tag_info(&format!("Reconnaissance complete for {} - ready for targeted enumeration", data)); + } + + _ => {} + } +} + +// Context: HTTP service discovery coordination +// Operation: Spawns unified HTTP discovery task for all open ports on a host +pub fn dispatch_http_discovery(ip: &str, ctx: &ProjectContext) { + let ip_clone = ip.to_string(); + let ctx2 = ctx.clone(); + + default_spawn( + "http_discovery", + ip_clone.clone(), + OperationType::HttpDiscovery, + false, + move |tx_opt, _cancel_token| { + let ctx2 = ctx2.clone(); + let ip_clone = ip_clone.clone(); + async move { + match crate::http::discover_http_services(ctx2, ip_clone.clone(), tx_opt).await { + Ok(_) => {}, + Err(e) => tag_info(&format!("http_discovery failed on {}: {}", ip_clone, e)), + } + } + }, + None, + GLOBAL_CANCEL_TOKEN.read().unwrap().clone() + ); +} + +// Context: Reconnaissance completion verification and signaling +// Operation: Checks if reconnaissance is complete and fires completion event +pub async fn check_and_dispatch_reconnaissance_complete(ip: &str, ctx: &ProjectContext) { + if let Ok(recon_complete) = ctx.is_host_reconnaissance_complete(ip).await { + if recon_complete { + // Check if we've already fired this hook + let key = (Hook::ReconnaissanceComplete, ip.to_string()); + { + let mut fired = FIRED_HOOKS.lock().unwrap(); + if fired.contains(&key) { + return; // Already fired + } + fired.insert(key); + } + + tag_info(&format!("All reconnaissance phases complete for {} - enabling targeted actions", ip)); + // Don't call dispatch_event to avoid recursion, just log readiness + } + } +} + +// Context: Timeout handling for slow or unresponsive hosts +// Operation: Marks host as slow and applies appropriate tags after timeout +pub async fn handle_nmap_timeout(ctx: &ProjectContext, ip: &str) { + // GUARD: Don't double-process timeouts + if ctx.is_full_scan_complete(ip).await.unwrap_or(false) { + return; // Already handled + } + + tag_info(&format!("Host {} timed out during full port scan - marking as slow", ip)); + + // Mark as slow + let _ = ctx.mark_host_slow(ip).await; + + // Add slow-nmap tag + if let Ok(mut existing_tags) = ctx.get_manual_tags(ip).await { + existing_tags.insert("slow-nmap".to_string()); + let _ = ctx.set_manual_tags(ip, existing_tags).await; + } + + // NOTE: Mark complete and workflow dispatch now handled by shared function +} + +// Context: Post-scan workflow coordination for both success and timeout cases +// Operation: Marks scan complete and continues workflow based on discovered ports +pub async fn complete_port_scan_workflow( + ctx: &ProjectContext, + ip: &str +) -> Result<(), Box> { + use crate::enumeration::handle_no_ports_host; // Import the function + + // Mark scan complete (idempotent - safe to call multiple times) + ctx.mark_full_scan_complete(ip).await?; + + // Get discovered ports + let ports = ctx.get_open_ports(ip).await?; + + if ports.is_empty() { + handle_no_ports_host(ctx, ip).await?; + } else { + // Continue normal workflow + dispatch_event(Hook::ServiceDiscoveryReady, ip, ctx); + } + + Ok(()) +} \ No newline at end of file diff --git a/src/workflow/execution.rs b/src/workflow/execution.rs new file mode 100755 index 0000000..fd4171a --- /dev/null +++ b/src/workflow/execution.rs @@ -0,0 +1,170 @@ +// src/workflow/execution.rs +use tokio::time::Duration; +use tokio_util::sync::CancellationToken; +use tokio::process::Command; +use tokio::io::{AsyncBufReadExt, BufReader}; +use std::time::Instant; +use crate::http::HttpServiceInfo; +use crate::sql::ProjectContext; +use crate::utilities::{print_action, print_cmd, tag_info}; +use super::types::SpecificActionStep; +use super::placeholders::replace_placeholders; + +// Context: Action execution system with timeout and cancellation support +// Operation: Executes workflow action commands with placeholder substitution and result capture +pub async fn execute_action_commands( + action_def: SpecificActionStep, + ip_clone: String, + ports: Vec, + port_services_json: String, + foreach_instance: Option<(String, Option)>, + ctx: &ProjectContext, + cancel_token: CancellationToken, + tx: tokio::sync::mpsc::UnboundedSender, +) -> String { + let start_time = Instant::now(); + + print_action(&format!("{} on {}", action_def.name, ip_clone)); + + // NEW: Get HTTP services for URL placeholder + let http_services = ctx.get_http_services(&ip_clone).await.unwrap_or_default(); + let (protocol, _port) = if let Some((first_port, service_json)) = http_services.iter().next() { + if let Ok(service_info) = serde_json::from_str::(service_json) { + (Some(service_info.protocol), Some(first_port.as_str())) + } else { + (None, ports.first().map(|p| p.as_str())) + } + } else { + (None, ports.first().map(|p| p.as_str())) + }; + + let port_services = crate::sql::hosts::parse_port_services(&port_services_json); + + let mut command_results = Vec::new(); + + // Check for cancellation + if cancel_token.is_cancelled() { + let msg = format!("Action '{}' cancelled before execution", action_def.name); + tag_info(&msg); + let _ = tx.send(msg); + return serde_json::json!({"commands": command_results}).to_string(); + } + + // Process single command + let available_port = ports.first().map(|p| p.as_str()); + + let full_cmd = replace_placeholders(&action_def.command, &ip_clone, available_port, protocol.as_deref(), &port_services, foreach_instance.as_ref()); + let timeout_duration = Duration::from_secs(action_def.timeout_s.unwrap_or(3000)); + + print_cmd(&full_cmd); + let log_msg = format!("[CMD] {}", full_cmd); + let _ = tx.send(log_msg); + + let cmd_start = Instant::now(); + + // Execute command with timeout + let (exit_code, output) = match execute_simple_command(&full_cmd, timeout_duration, cancel_token.clone(), tx.clone()).await { + Ok((code, out)) => (code, out), + Err(e) => { + let error_msg = format!("[CMD] Error: {}", e); + tag_info(&error_msg); + let _ = tx.send(error_msg.clone()); + (-1, error_msg) + } + }; + + let duration_ms = cmd_start.elapsed().as_millis() as u64; + + // Store command result + let cmd_result = serde_json::json!({ + "command": full_cmd, + "timeout_s": action_def.timeout_s.unwrap_or(3000), + "exit_code": exit_code, + "duration_ms": duration_ms, + "output": output + }); + + command_results.push(cmd_result); + + let success_msg = format!("[CMD] Completed (exit: {}, duration: {}ms)", exit_code, duration_ms); + let _ = tx.send(success_msg); + + let total_duration = start_time.elapsed(); + let completion_msg = format!("Action '{}' completed in {:?}", action_def.name, total_duration); + tag_info(&completion_msg); + let _ = tx.send(completion_msg); + + // Return structured JSON with commands array + serde_json::json!({ + "commands": command_results + }).to_string() +} + +// Context: Low-level command execution with comprehensive timeout handling +// Operation: Executes shell command with timeout, cancellation, and real-time output capture +pub async fn execute_simple_command( + command: &str, + timeout: Duration, + cancel_token: CancellationToken, + tx: tokio::sync::mpsc::UnboundedSender, +) -> Result<(i32, String), Box> { + let mut child = Command::new("sh") + .arg("-c") + .arg(command) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn()?; + + let stdout = child.stdout.take().expect("stdout piped"); + let stderr = child.stderr.take().expect("stderr piped"); + + let tx_out = tx.clone(); + let tx_err = tx.clone(); + + // Drain stdout and stderr as independent tasks — prevents pipe deadlock when one pipe fills + let stdout_task = tokio::spawn(async move { + let mut buf = String::new(); + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + let _ = tx_out.send(line.clone()); + buf.push_str(&line); + buf.push('\n'); + } + buf + }); + + let stderr_task = tokio::spawn(async move { + let mut buf = String::new(); + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + let _ = tx_err.send(line.clone()); + buf.push_str(&line); + buf.push('\n'); + } + buf + }); + + // Timeout now covers the entire streaming + wait phase, not just child.wait() + let drain = async move { + let o1 = stdout_task.await.unwrap_or_default(); + let o2 = stderr_task.await.unwrap_or_default(); + o1 + &o2 + }; + + tokio::select! { + output = drain => { + let status = child.wait().await?; + Ok((status.code().unwrap_or(-1), output)) + } + _ = tokio::time::sleep(timeout) => { + let _ = child.kill().await; + let _ = child.wait().await; + Err(format!("Command timed out after {:?}", timeout).into()) + } + _ = cancel_token.cancelled() => { + let _ = child.kill().await; + let _ = child.wait().await; + Err("Command cancelled".into()) + } + } +} \ No newline at end of file diff --git a/src/workflow/host_queue.rs b/src/workflow/host_queue.rs new file mode 100644 index 0000000..84f8c2a --- /dev/null +++ b/src/workflow/host_queue.rs @@ -0,0 +1,319 @@ +// src/workflow/host_queue.rs +// +// Per-host action serialization — additive layer above default_spawn. +// Ensures only one workflow action runs at a time per host. When the queue +// drains, immediately re-evaluates that host's conditions against the full +// action list without waiting for the next 2-second global tick. +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::{Arc, Mutex}; +use once_cell::sync::Lazy; +use tokio_util::sync::CancellationToken; + +use super::conditions::evaluate_condition; +use super::concurrency::OperationType; +use super::execution::execute_action_commands; +use super::spawning::default_spawn; +use super::types::SpecificActionStep; +use crate::sql::ProjectContext; +use crate::utilities::tag_info; + +// All data needed to execute one action dispatch, carried through the queue. +#[derive(Clone)] +pub struct PendingAction { + pub composite_key: String, + pub action_def: SpecificActionStep, + // (port, proto) for per-service/per-http fan-out; None for single-spawn path + pub foreach_instance: Option<(String, Option)>, + pub ports: Vec, + pub port_services_json: String, + // Shared with workflow.rs so fired-key inserts from re-evaluation are visible globally + pub all_actions: Arc>, + pub fired: Arc>>, +} + +pub(crate) struct HostActionState { + busy: bool, + queue: VecDeque, +} + +pub(crate) static HOST_ACTION_STATES: Lazy>> = + Lazy::new(|| Mutex::new(HashMap::new())); + +// Returns true when the host has no running action and an empty queue. +pub(crate) fn is_host_idle(ip: &str) -> bool { + let states = HOST_ACTION_STATES.lock().unwrap(); + states.get(ip).map(|s| !s.busy && s.queue.is_empty()).unwrap_or(true) +} + +// Entry point called by workflow.rs instead of default_spawn. +// If host is free: mark busy and dispatch immediately. +// If host is busy: push to per-host queue. +pub fn try_dispatch_or_queue( + ip: &str, + pending: PendingAction, + ctx: &ProjectContext, + cancel_token: CancellationToken, +) { + let mut states = HOST_ACTION_STATES.lock().unwrap(); + let state = states.entry(ip.to_string()).or_insert_with(|| HostActionState { + busy: false, + queue: VecDeque::new(), + }); + if state.busy { + state.queue.push_back(pending); + } else { + state.busy = true; + drop(states); + dispatch_pending_action(ip.to_string(), pending, ctx.clone(), cancel_token); + } +} + +// Spawn the action via default_spawn. The closure calls on_action_complete when done. +fn dispatch_pending_action( + ip: String, + pending: PendingAction, + ctx: ProjectContext, + cancel_token: CancellationToken, +) { + let PendingAction { + composite_key, + action_def, + foreach_instance, + ports, + port_services_json, + all_actions, + fired, + } = pending; + + let ip_c = ip.clone(); + let key_c = composite_key.clone(); + let def_c = action_def.clone(); + let ctx2 = ctx.clone(); + let cancel_for_spawn = cancel_token.clone(); + + default_spawn( + composite_key, + ip, + OperationType::Global, + false, + move |tx_opt, ct| { + let tx = tx_opt.expect("raw_tx must be present for action tasks"); + async move { + if !ct.is_cancelled() { + let combined = execute_action_commands( + def_c.clone(), + ip_c.clone(), + ports, + port_services_json, + foreach_instance, + &ctx2, + ct.clone(), + tx, + ).await; + + ctx2.insert_action_output(&ip_c, &key_c, &combined).await.ok(); + + if let Some(tag_patterns) = &def_c.tags { + match crate::tag::apply_action_tags( + &ctx2, &ip_c, &key_c, &combined, tag_patterns, + ).await { + Ok(true) => { ctx2.mark_action_tagged(&ip_c, &key_c).await.ok(); } + Ok(false) => {} + Err(e) => { tag_info(&format!("Failed to apply action tags for {}: {}", ip_c, e)); } + } + } + + ctx2.mark_action_complete(&ip_c, &key_c).await.ok(); + } + + on_action_complete(ip_c, ctx2, all_actions, fired, ct).await; + } + }, + None, + Some(cancel_for_spawn), + ); +} + +// Called at the end of every dispatched action. +// If the per-host queue has pending actions, dispatch the next one. +// If the queue is empty, immediately re-evaluate conditions for this host. +async fn on_action_complete( + ip: String, + ctx: ProjectContext, + all_actions: Arc>, + fired: Arc>>, + cancel_token: CancellationToken, +) { + let next = { + let mut states = HOST_ACTION_STATES.lock().unwrap(); + let state = states.entry(ip.clone()).or_insert_with(|| HostActionState { + busy: false, + queue: VecDeque::new(), + }); + state.queue.pop_front() + }; + + if let Some(pending) = next { + dispatch_pending_action(ip, pending, ctx, cancel_token); + } else { + { + let mut states = HOST_ACTION_STATES.lock().unwrap(); + if let Some(state) = states.get_mut(&ip) { + state.busy = false; + } + } + re_evaluate_single_host(ip, ctx, all_actions, fired, cancel_token).await; + } +} + +// Fresh snapshot → evaluate all actions → try_dispatch_or_queue for any newly triggerable ones. +// Calls existing evaluate_condition() from conditions.rs — no logic duplication. +async fn re_evaluate_single_host( + ip: String, + ctx: ProjectContext, + all_actions: Arc>, + fired: Arc>>, + cancel_token: CancellationToken, +) { + if cancel_token.is_cancelled() { + return; + } + + let snapshot = match ctx.get_host_snapshot(&ip).await { + Ok(Some(s)) => s, + _ => return, + }; + + let recon_complete = snapshot.full_scan_complete + && snapshot.service_scan_complete + && snapshot.http_discovery_complete; + if !recon_complete { + return; + } + + let ports: Vec = snapshot.ports + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + + let mut tag_vec: Vec = Vec::new(); + for t in snapshot.action_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.manual_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.refined_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.automated_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + + let mut dispatched = 0usize; + + for action in all_actions.iter() { + let trigger = if let Some(condition) = &action.condition { + evaluate_condition(condition, &ports, &tag_vec, &snapshot) + } else { + true + }; + if !trigger { + continue; + } + + // Mirror the same fan-out logic as the global loop + let svc_name_for_iter = action.condition.as_ref() + .and_then(|c| c.has_service.as_deref()) + .filter(|s: &&str| !s.contains(" AND ") && !s.contains(" OR ")); + + let http_iter = action.condition.as_ref() + .and_then(|c| c.has_http_service) + .unwrap_or(false); + + if let Some(svc_name) = svc_name_for_iter { + let port_map = crate::sql::hosts::parse_port_services(&snapshot.port_services); + let mut matching: Vec = port_map.iter() + .filter(|(_, v)| v.eq_ignore_ascii_case(svc_name)) + .map(|(k, _)| k.clone()) + .collect(); + matching.sort(); + + for svc_port in matching { + let composite_key = format!("{}__{}", action.name, svc_port); + let fired_key = (composite_key.clone(), ip.clone()); + if fired.lock().unwrap().contains(&fired_key) { continue; } + if ctx.is_action_complete(&ip, &composite_key).await.unwrap_or(false) { continue; } + if !ctx.mark_action_started(&ip, &composite_key).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(fired_key); + + tag_info(&format!("Re-eval: triggering '{}' on {}:{}", action.name, ip, svc_port)); + + let pending = PendingAction { + composite_key, + action_def: action.clone(), + foreach_instance: Some((svc_port, None)), + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(&ip, pending, &ctx, cancel_token.clone()); + dispatched += 1; + } + + } else if http_iter { + for (svc_port, svc_proto) in &snapshot.http_service_list { + let composite_key = format!("{}__{}", action.name, svc_port); + let fired_key = (composite_key.clone(), ip.clone()); + if fired.lock().unwrap().contains(&fired_key) { continue; } + if ctx.is_action_complete(&ip, &composite_key).await.unwrap_or(false) { continue; } + if !ctx.mark_action_started(&ip, &composite_key).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(fired_key); + + tag_info(&format!("Re-eval: triggering '{}' on {} ({}://{}:{})", + action.name, ip, svc_proto, ip, svc_port)); + + let pending = PendingAction { + composite_key, + action_def: action.clone(), + foreach_instance: Some((svc_port.clone(), Some(svc_proto.clone()))), + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(&ip, pending, &ctx, cancel_token.clone()); + dispatched += 1; + } + + } else { + let fired_key = (action.name.clone(), ip.clone()); + if fired.lock().unwrap().contains(&fired_key) { continue; } + if ctx.is_action_complete(&ip, &action.name).await.unwrap_or(false) { continue; } + if !ctx.mark_action_started(&ip, &action.name).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(fired_key); + + tag_info(&format!("Re-eval: triggering '{}' on {}", action.name, ip)); + + let pending = PendingAction { + composite_key: action.name.clone(), + action_def: action.clone(), + foreach_instance: None, + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(&ip, pending, &ctx, cancel_token.clone()); + dispatched += 1; + } + } + + // Queue drained and re-evaluation found nothing new — all workflow actions complete + if dispatched == 0 { + ctx.mark_workflow_complete(&ip).await.ok(); + } +} diff --git a/src/workflow/mod.rs b/src/workflow/mod.rs new file mode 100755 index 0000000..e3626a3 --- /dev/null +++ b/src/workflow/mod.rs @@ -0,0 +1,40 @@ +// Copyright 2024 P-Aimon-Pen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// src/workflow/mod.rs + +pub mod concurrency; +pub mod constants; +pub mod conditions; +pub mod dispatch; +pub mod execution; +pub mod host_queue; +pub mod nuclei; +pub mod placeholders; +pub mod resume; +pub mod screenshots; +pub mod shutdown; +pub mod spawning; +pub mod tasks; +pub mod types; +pub mod workflow; + +// Re-export commonly used items to maintain compatibility +pub use concurrency::{ConcurrencyConfig, initialize_concurrency}; +pub use conditions::evaluate_condition; +pub use constants::{FIRED_HOOKS, ACTIVE_TASKS, QUEUED_TASK_INFOS, RUNTIME_ACTIONS, GLOBAL_CANCEL_TOKEN, QUEUED_TASKS, RESUME_MODE, MASSCAN_BLOCK_HOOKS, MASSCAN_STATUS, WEB_ONLY_MODE, SCREENSHOT_SEMAPHORE}; +pub use dispatch::{dispatch_event, handle_nmap_timeout, complete_port_scan_workflow}; +pub use spawning::default_spawn; +pub use types::{Hook, TaskInfo, SpecificActionStep, Condition, TagPattern}; +pub use workflow::{run_workflow, save_action_to_workflow}; \ No newline at end of file diff --git a/src/workflow/nuclei.rs b/src/workflow/nuclei.rs new file mode 100644 index 0000000..3e62181 --- /dev/null +++ b/src/workflow/nuclei.rs @@ -0,0 +1,323 @@ +// src/workflow/nuclei.rs +use std::process::Stdio; +use serde::Deserialize; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio_util::sync::CancellationToken; +use tokio::sync::mpsc::UnboundedSender; +use super::concurrency::OperationType; +use super::constants::{NUCLEI_SEMAPHORE, NUCLEI_HOST_SEMAPHORES, GLOBAL_CANCEL_TOKEN}; +use super::spawning::default_spawn; +use crate::sql::ProjectContext; +use crate::sql::nuclei::NucleiFinding; +use crate::utilities::{tag_info, tag_critical}; + +// Temporary deserialization struct matching nuclei's JSONL output format +#[derive(Deserialize)] +struct NucleiRawFinding { + #[serde(rename = "template-id")] + template_id: Option, + info: Option, + #[serde(rename = "matched-at")] + matched_at: Option, + // WHICH matcher fired (e.g., "ms-httpapi", "nginx", "waf-cloudflare") + #[serde(rename = "matcher-name")] + matcher_name: Option, + // Values extracted by extractor-based templates (version strings, emails, etc.) + #[serde(rename = "extracted-results")] + extracted_results: Option>, + host: Option, +} + +#[derive(Deserialize)] +struct NucleiRawInfo { + name: Option, + severity: Option, + description: Option, + // nuclei emits reference as either a JSON array or a single string + reference: Option, + // nuclei emits tags as either a JSON array or a comma-separated string + tags: Option, +} + +/// Runs `nuclei -update-templates` and reports whether it completed successfully. +/// Does not decide whether templates are usable afterward — see `nuclei_templates_present()` +/// and `ensure_nuclei_templates_ready()` for the caller-facing readiness check. +pub async fn run_nuclei_template_update() -> Result<(), String> { + use std::process::Stdio; + tag_info("Updating nuclei templates..."); + + let child_result = tokio::process::Command::new("nuclei") + .args(["-update-templates", "-silent"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + + let mut child = match child_result { + Ok(c) => c, + Err(e) => { + let msg = format!("nuclei not available ({})", e); + tag_info(&msg); + return Err(msg); + } + }; + + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + let out_task = tokio::spawn(async move { + let mut lines = BufReader::new(stdout).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if !line.is_empty() { tag_info(&format!("[nuclei-update] {}", line)); } + } + }); + let err_task = tokio::spawn(async move { + let mut lines = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = lines.next_line().await { + if !line.is_empty() { tag_info(&format!("[nuclei-update] {}", line)); } + } + }); + + let result = match tokio::time::timeout(tokio::time::Duration::from_secs(300), child.wait()).await { + Ok(Ok(status)) if status.success() => { + tag_info("Nuclei template update complete"); + Ok(()) + } + Ok(Ok(status)) => Err(format!("nuclei -update-templates exited with {:?}", status)), + Ok(Err(e)) => Err(format!("nuclei -update-templates error: {}", e)), + Err(_) => { + child.kill().await.ok(); + Err("nuclei -update-templates timed out after 5 minutes".to_string()) + } + }; + out_task.abort(); + err_task.abort(); + result +} + +/// Checks whether nuclei templates already exist under the current user's home directory, +/// covering both the v3+ default path and the older pre-v3 default path. +pub fn nuclei_templates_present() -> bool { + let home = match std::env::var("HOME") { + Ok(h) => h, + Err(_) => return false, + }; + + for candidate in [".local/nuclei-templates", "nuclei-templates"] { + let dir = std::path::Path::new(&home).join(candidate); + let has_templates = std::fs::read_dir(&dir) + .map(|mut entries| entries.next().is_some()) + .unwrap_or(false); + if has_templates { + return true; + } + } + false +} + +/// Blocking readiness check for project creation/resume: runs the template update and, +/// regardless of whether the update itself succeeded, only fails if no templates are +/// usable afterward (an update failure with templates already present from a prior run +/// is just a stale-template warning, not a hard blocker). +pub async fn ensure_nuclei_templates_ready() -> Result<(), String> { + let update_result = run_nuclei_template_update().await; + + if nuclei_templates_present() { + if let Err(e) = update_result { + tag_critical(&format!( + "nuclei template update failed ({}), continuing with existing templates (may be stale)", + e + )); + } else { + tag_critical("nuclei templates ready"); + } + Ok(()) + } else { + let reason = update_result.err().unwrap_or_else(|| "templates directory is still empty".to_string()); + Err(format!( + "nuclei has no templates installed and the template update failed: {}. \ + Run 'nuclei -update-templates' manually, or check network/proxy access for the user ROFMAD runs as.", + reason + )) + } +} + +pub fn dispatch_nuclei_scan(ip: &str, port: &str, protocol: &str, ctx: &ProjectContext) { + let ip_c = ip.to_string(); + let port_c = port.to_string(); + let protocol_c = protocol.to_string(); + let ctx2 = ctx.clone(); + + default_spawn( + "nuclei_scan", + ip_c.clone(), + OperationType::Global, + true, + move |raw_tx, cancel_token| { + let ctx2 = ctx2.clone(); + async move { + // Clone Arc before any await — Mutex released immediately + let host_sem = { + let mut map = NUCLEI_HOST_SEMAPHORES.lock().unwrap(); + map.entry(ip_c.clone()) + .or_insert_with(|| std::sync::Arc::new(tokio::sync::Semaphore::new(2))) + .clone() + }; + let _host_permit = host_sem.acquire().await.ok(); + let _global_permit = NUCLEI_SEMAPHORE.acquire().await.ok(); + run_nuclei_scan(&ctx2, &ip_c, &port_c, &protocol_c, raw_tx, cancel_token).await; + } + }, + Some(300), + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); +} + +async fn run_nuclei_scan( + ctx: &ProjectContext, + ip: &str, + port: &str, + protocol: &str, + raw_tx: Option>, + cancel_token: CancellationToken, +) { + let target = format!("{}://{}:{}/", protocol, ip, port); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("Starting nuclei scan: {}", target)); + let _ = tx.send(format!("$ nuclei -target {} -j -silent -duc -as -pt http -tlsi -timeout 10", target)); + } + + let child_result = tokio::process::Command::new("nuclei") + .args(["-target", &target, "-j", "-silent", "-duc", "-as", "-pt", "http", "-tlsi", "-timeout", "10"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + + let mut child = match child_result { + Ok(c) => c, + Err(e) => { + let msg = format!("Failed to launch nuclei: {} (is nuclei installed?)", e); + tag_info(&msg); + if let Some(tx) = &raw_tx { let _ = tx.send(msg); } + return; + } + }; + + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + // Drain stderr in a background task + let stderr_tx = raw_tx.clone(); + let stderr_task = tokio::spawn(async move { + let mut reader = BufReader::new(stderr).lines(); + while let Ok(Some(line)) = reader.next_line().await { + if !line.is_empty() { + if let Some(tx) = &stderr_tx { + let _ = tx.send(format!("nuclei: {}", line)); + } + } + } + }); + + let mut reader = BufReader::new(stdout).lines(); + let mut findings: Vec = Vec::new(); + + loop { + tokio::select! { + line_result = reader.next_line() => { + match line_result { + Ok(Some(line)) => { + if !line.is_empty() { + if let Ok(raw) = serde_json::from_str::(&line) { + let sev = raw.info.as_ref().and_then(|i| i.severity.as_deref()).unwrap_or("info"); + let name = raw.info.as_ref().and_then(|i| i.name.as_deref()).unwrap_or("unknown"); + let tmpl = raw.template_id.as_deref().unwrap_or("?"); + let matched = raw.matched_at.as_deref().unwrap_or("?"); + let matcher = raw.matcher_name.as_deref().unwrap_or(""); + let extracted = raw.extracted_results.as_deref().unwrap_or(&[]); + let detail = match (!matcher.is_empty(), !extracted.is_empty()) { + (true, true) => format!(" [{}] {} → {}", matcher, matched, extracted.join(", ")), + (true, false) => format!(" [{}] {}", matcher, matched), + (false, true) => format!(" {} → {}", matched, extracted.join(", ")), + (false, false) => format!(" {}", matched), + }; + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("[{}] {} [{}]{}", sev.to_uppercase(), name, tmpl, detail)); + } + + let references = raw.info.as_ref() + .and_then(|i| match &i.reference { + Some(serde_json::Value::Array(arr)) => Some( + arr.iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect(), + ), + Some(serde_json::Value::String(s)) => Some(vec![s.clone()]), + _ => None, + }) + .unwrap_or_default(); + let nuclei_tags: Vec = raw.info.as_ref() + .and_then(|i| match &i.tags { + Some(serde_json::Value::Array(arr)) => Some( + arr.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect(), + ), + Some(serde_json::Value::String(s)) => Some( + s.split(',').map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()).collect(), + ), + _ => None, + }) + .unwrap_or_default(); + findings.push(NucleiFinding { + template_id: raw.template_id.unwrap_or_default(), + name: raw.info.as_ref() + .and_then(|i| i.name.clone()) + .unwrap_or_default(), + severity: raw.info.as_ref() + .and_then(|i| i.severity.clone()) + .unwrap_or_else(|| "unknown".to_string()), + matched_at: raw.matched_at.unwrap_or_default(), + matcher_name: raw.matcher_name.unwrap_or_default(), + extracted_results: raw.extracted_results.unwrap_or_default(), + host: raw.host.unwrap_or_default(), + description: raw.info.as_ref() + .and_then(|i| i.description.clone()) + .unwrap_or_default(), + references, + nuclei_tags, + }); + } else { + // non-JSON line (progress text, warnings) — pass through as-is + if let Some(tx) = &raw_tx { let _ = tx.send(line); } + } + } + } + Ok(None) => break, + Err(_) => break, + } + } + _ = cancel_token.cancelled() => { + child.kill().await.ok(); + stderr_task.abort(); + if let Some(tx) = &raw_tx { + let _ = tx.send("Nuclei scan cancelled — saving partial results".to_string()); + } + ctx.insert_nuclei_output(ip, port, findings, &target).await.ok(); + return; + } + } + } + + stderr_task.await.ok(); + child.wait().await.ok(); + + let count = findings.len(); + tag_info(&format!("Nuclei scan complete: {}:{} — {} finding(s)", ip, port, count)); + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("Nuclei complete: {} finding(s) for {}", count, target)); + } + + ctx.insert_nuclei_output(ip, port, findings, &target).await.ok(); +} diff --git a/src/workflow/placeholders.rs b/src/workflow/placeholders.rs new file mode 100755 index 0000000..6b023f0 --- /dev/null +++ b/src/workflow/placeholders.rs @@ -0,0 +1,97 @@ +// src/workflow/placeholders.rs +use std::collections::HashMap; + +// Context: Template placeholder replacement system for dynamic command generation +// Operation: Replaces template placeholders with actual values for IP, port, protocol, URL, +// and service-specific port resolution using the port_services map from nmap -sV. +// +// Supported placeholders: +// {{IP}} / {{ip}} / __IP__ / → target IP address +// {{PORT}} / {{port}} → first open port (legacy, order-dependent) +// {{URL}} / {{BASE_URL}} → http(s)://IP:PORT for first HTTP service +// {{PROTOCOL}} → http or https +// {{PORT:service}} → first port where service_name == "service" +// {{PORTS:service}} → all ports where service_name == "service", comma-joined +// {{SERVICE_PORT}} → port for current per-instance iteration +// {{SERVICE_PROTOCOL}} → protocol for current per-instance iteration (HTTP only) +// {{HTTP_URL}} → http(s)://IP:PORT (HTTP per-instance alias) +// {{HTTP_PORT}} → port (HTTP per-instance alias) +// {{HTTP_PROTOCOL}} → http or https (HTTP per-instance alias) +pub fn replace_placeholders( + s: &str, + ip: &str, + port: Option<&str>, + protocol: Option<&str>, + port_services: &HashMap, + foreach_instance: Option<&(String, Option)>, +) -> String { + let mut result = s.replace("{{IP}}", ip) + .replace("{{ip}}", ip) + .replace("__IP__", ip) + .replace("", ip) + .replace("", ip); + + // Legacy port / URL placeholders + if let Some(port_value) = port { + result = result.replace("{{PORT}}", port_value) + .replace("{{port}}", port_value); + + if let Some(protocol_value) = protocol { + let url = format!("{}://{}:{}", protocol_value, ip, port_value); + result = result.replace("{{URL}}", &url) + .replace("{{BASE_URL}}", &url); + result = result.replace("{{PROTOCOL}}", protocol_value); + } + } + + // {{PORT:service}} — first port whose service matches (case-insensitive) + // Walk the string manually to avoid pulling in regex here + while let Some(start) = result.find("{{PORT:") { + let rest = &result[start + 7..]; + if let Some(end) = rest.find("}}") { + let svc = rest[..end].trim().to_lowercase(); + let replacement = port_services.iter() + .find(|(_, v)| v.to_lowercase() == svc) + .map(|(k, _)| k.as_str()) + .unwrap_or(""); + result = format!("{}{}{}", &result[..start], replacement, &result[start + 7 + end + 2..]); + } else { + break; // malformed — stop to avoid infinite loop + } + } + + // {{PORTS:service}} — all ports whose service matches, comma-joined + while let Some(start) = result.find("{{PORTS:") { + let rest = &result[start + 8..]; + if let Some(end) = rest.find("}}") { + let svc = rest[..end].trim().to_lowercase(); + let mut matched: Vec<&str> = port_services.iter() + .filter(|(_, v)| v.to_lowercase() == svc) + .map(|(k, _)| k.as_str()) + .collect(); + matched.sort(); // deterministic order + let replacement = matched.join(","); + result = format!("{}{}{}", &result[..start], replacement, &result[start + 8 + end + 2..]); + } else { + break; + } + } + + // Per-instance placeholders — resolve when action is driven by has_service or has_http_service + if let Some((svc_port, svc_proto)) = foreach_instance { + result = result + .replace("{{SERVICE_PORT}}", svc_port) + .replace("{{SERVICE_PROTOCOL}}", svc_proto.as_deref().unwrap_or("")); + + // HTTP convenience aliases — only when protocol is available (HTTP service iteration) + if let Some(proto) = svc_proto { + let url = format!("{}://{}:{}", proto, ip, svc_port); + result = result + .replace("{{HTTP_URL}}", &url) + .replace("{{HTTP_PORT}}", svc_port) + .replace("{{HTTP_PROTOCOL}}", proto); + } + } + + result +} diff --git a/src/workflow/resume.rs b/src/workflow/resume.rs new file mode 100755 index 0000000..3df8b4d --- /dev/null +++ b/src/workflow/resume.rs @@ -0,0 +1,83 @@ +// src/workflow/resume.rs +use sqlx::Row; +use std::sync::atomic::Ordering; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::constants::{INIT_IN_PROGRESS, RESUME_MODE}; + +// Context: Resume functionality for interrupted ROFMAD sessions +// Operation: Processes resume mode by resetting flags and restarting incomplete workflows +pub async fn process_resume_mode(ctx: &ProjectContext) -> Result<(), Box> { + // Check if resume mode is enabled + if RESUME_MODE.load(Ordering::SeqCst) { + tag_info("Resuming previous run with clean restart strategy..."); + + // STEP 1: Reset all started flags to allow clean restart + ctx.reset_all_started_flags().await?; + + // STEP 2: Process interrupted tasks + tag_info("Processing interrupted tasks..."); + } + + // Resume mode: check for incomplete tasks and restart workflows + let rows = sqlx::query( + "SELECT ip FROM targets + WHERE run_id = ? + AND (full_scan_complete = 0 OR service_scan_complete = 0 OR http_discovery_complete = 0)" + ) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await?; + + if !rows.is_empty() { + tag_info(&format!("Resume mode: Found {} hosts with incomplete workflows", rows.len())); + + for row in rows { + let ip: String = row.get("ip"); + + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Resuming workflow for {}", ip)); + } + + // Let the dispatcher determine what phases need completion + if let Err(e) = ctx.restart_host_workflow(&ip).await { + tag_info(&format!("Failed to restart workflow for {}: {}", ip, e)); + } + } + + INIT_IN_PROGRESS.store(true, Ordering::SeqCst); + + // Give resumed tasks time to start before beginning polling + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + } + + // Re-trigger screenshots for hosts where recon is complete but screenshots were interrupted. + // A full reset would be overkill here — just reset the started flag and re-dispatch. + let screenshot_rows = sqlx::query( + "SELECT ip FROM targets + WHERE run_id = ? + AND full_scan_complete = 1 + AND service_scan_complete = 1 + AND http_discovery_complete = 1 + AND screenshots_complete = 0" + ) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await?; + + if !screenshot_rows.is_empty() { + tag_info(&format!("Resume: re-dispatching screenshots for {} host(s)", screenshot_rows.len())); + for row in screenshot_rows { + let ip: String = row.get("ip"); + sqlx::query("UPDATE targets SET screenshots_started = FALSE WHERE run_id = ? AND ip = ?") + .bind(&ctx.run_id) + .bind(&ip) + .execute(&ctx.pool) + .await?; + super::screenshots::dispatch_individual_screenshots(&ip, ctx); + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/workflow/screenshots.rs b/src/workflow/screenshots.rs new file mode 100755 index 0000000..84d5d79 --- /dev/null +++ b/src/workflow/screenshots.rs @@ -0,0 +1,199 @@ +// src/workflow/screenshots.rs +use tokio::sync::mpsc::UnboundedSender; +use super::concurrency::OperationType; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::constants::GLOBAL_CANCEL_TOKEN; +use super::spawning::default_spawn; + +// Context: Screenshot task coordination for HTTP services +// Operation: Dispatches browser screenshot task for all HTTP services on a host +pub fn dispatch_individual_screenshots(ip: &str, ctx: &ProjectContext) { + let ip_clone = ip.to_string(); + let ctx2 = ctx.clone(); + + default_spawn( + "host_screenshots", + ip_clone.clone(), + OperationType::Global, + false, + move |raw_tx, _cancel_token| { + let ctx2 = ctx2.clone(); + let ip_clone = ip_clone.clone(); + async move { + // Per-port timeouts inside screenshot_http_services bound each capture. + // Always mark screenshots_complete so the host reaches the analysis queue + // regardless of how many screenshots succeeded or timed out. + let result = create_host_screenshot_session(&ctx2, &ip_clone, raw_tx).await; + if let Err(e) = result { + tag_info(&format!("Screenshot session failed for {}: {}", ip_clone, e)); + } + ctx2.mark_screenshots_complete(&ip_clone).await.ok(); + } + }, + None, // no global timeout — per-port timeout (90s) in screenshot_http_services + GLOBAL_CANCEL_TOKEN.read().unwrap().clone() + ); +} + +// Context: Shared browser screenshot session management +// Operation: Creates screenshot session using shared browser for all HTTP services on host +pub async fn create_host_screenshot_session( + ctx: &ProjectContext, + ip: &str, + raw_tx: Option>, +) -> Result<(), Box> { + + // Mark screenshots as started + if !ctx.mark_screenshots_started(ip).await? { + tag_debug(&format!("Screenshots already started for {}, skipping", ip)); + return Ok(()); + } + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("Starting screenshot session for {}", ip)); + } + + // Get confirmed HTTP services from database + let http_services = ctx.get_http_services(ip).await?; + + if http_services.is_empty() { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("No HTTP services found for screenshots on {}", ip)); + } + return Ok(()); + } + + if VERBOSE.load(std::sync::atomic::Ordering::SeqCst) { + tag_debug(&format!("Found {} HTTP services for screenshot session on {}", + http_services.len(), ip)); + } + + // Acquire one global Chrome slot — held for the entire host session (sequential per host) + let _permit = crate::workflow::SCREENSHOT_SEMAPHORE.acquire().await.ok(); + + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("Chrome slot acquired for {}, starting captures", ip)); + } + + let results = crate::http::capture_multiple_screenshots(ctx, ip, raw_tx).await?; + + // Update database with screenshot paths + for result in &results { + if result.success && !result.file_path.is_empty() { + // Extract port from URL for database update + if let Some(port) = extract_port_from_url(&result.url) { + if let Ok(mut service_data) = ctx.get_http_services(ip).await { + if let Some(service_json) = service_data.get_mut(&port) { + if let Ok(mut service_info) = serde_json::from_str::(service_json) { + service_info.screenshot_path = Some(result.file_path.clone()); + if let Ok(updated_json) = serde_json::to_string(&service_info) { + let _ = ctx.insert_http_service(ip, &port, &updated_json).await; + } + } + } + } + } + } + } + + for result in &results { + if !result.success { + if let Some(port) = extract_port_from_url(&result.url) { + tag_info(&format!("Screenshot failed for {}:{} — use Retry to re-attempt", ip, port)); + } + } + } + + ctx.mark_screenshots_complete(ip).await?; + + tag_info(&format!("Screenshot session complete for {}: {}/{} successful", + ip, results.iter().filter(|r| r.success).count(), results.len())); + + Ok(()) +} + +// Context: Retake screenshot for a single HTTP service, visible in task queue +// Operation: Dispatches via default_spawn so the retake appears in Current Tasks and logs +pub fn dispatch_retake_screenshot( + ip: &str, + url: String, + output_path: String, + port: String, + ctx: &ProjectContext, +) { + let ip_c = ip.to_string(); + let ctx2 = ctx.clone(); + + default_spawn( + "retake_screenshot", + ip_c.clone(), + OperationType::Global, + true, + move |raw_tx, _cancel_token| { + let ctx2 = ctx2.clone(); + let ip_c = ip_c.clone(); + let url = url.clone(); + let output_path = output_path.clone(); + let port = port.clone(); + async move { + if let Some(tx) = &raw_tx { + let _ = tx.send(format!("Retaking screenshot: {}", url)); + } + let _permit = crate::workflow::SCREENSHOT_SEMAPHORE.acquire().await.ok(); + let config = crate::http::config::ScreenshotConfig::default(); + let mut bm = crate::http::screenshot::BrowserManager::new(config); + match bm.take_screenshot(&url, &output_path, raw_tx.as_ref()).await { + Ok(result) if result.success => { + if let Ok(mut svcs) = ctx2.get_http_services(&ip_c).await { + if let Some(json) = svcs.get_mut(&port) { + if let Ok(mut info) = serde_json::from_str::(json) { + info.screenshot_path = Some(result.file_path.clone()); + if let Ok(updated) = serde_json::to_string(&info) { + ctx2.insert_http_service(&ip_c, &port, &updated).await.ok(); + } + } + } + } + tag_info(&format!("Retake done {}:{} -> {}", ip_c, port, result.file_path)); + } + Ok(result) => { + tag_info(&format!("Retake failed {}:{}: {}", + ip_c, port, result.error_message.unwrap_or_default())); + } + Err(e) => { + tag_info(&format!("Retake error {}:{}: {}", ip_c, port, e)); + } + } + } + }, + None, + GLOBAL_CANCEL_TOKEN.read().unwrap().clone(), + ); +} + +// Context: URL parsing utility for screenshot database updates +// Operation: Extracts port number from URL string with default port handling +pub fn extract_port_from_url(url: &str) -> Option { + // Parse URL to extract port: "https://192.168.1.1:8443/" -> "8443" + if let Ok(parsed) = url::Url::parse(url) { + if let Some(port) = parsed.port() { + return Some(port.to_string()); + } + // Default ports + match parsed.scheme() { + "https" => Some("443".to_string()), + "http" => Some("80".to_string()), + _ => None, + } + } else { + // Fallback regex parsing + if let Ok(re) = regex::Regex::new(r":(\d+)/") { + if let Some(caps) = re.captures(url) { + return caps.get(1).map(|m| m.as_str().to_string()); + } + } + None + } +} \ No newline at end of file diff --git a/src/workflow/shutdown.rs b/src/workflow/shutdown.rs new file mode 100755 index 0000000..a8f5145 --- /dev/null +++ b/src/workflow/shutdown.rs @@ -0,0 +1,58 @@ +// src/workflow/shutdown.rs +use tokio_util::sync::CancellationToken; +use crate::sql::ProjectContext; +use crate::utilities::tag_critical; +use super::constants::{TASK_SEMAPHORE, SHUTDOWN_IN_PROGRESS}; +use std::sync::atomic::Ordering; +use super::tasks::{dump_active_tasks, dump_pending_tasks}; + +// Context: Graceful shutdown system for ROFMAD with task state preservation +// Operation: Handles shutdown by saving task state, cancelling tasks, and providing summary +pub async fn handle_shutdown( + ctx: &ProjectContext, + cancel_token: CancellationToken +) -> Result<(), Box> { + // Prevent duplicate shutdown calls + if SHUTDOWN_IN_PROGRESS.swap(true, Ordering::SeqCst) { + tag_critical("Shutdown already in progress - ignoring duplicate request"); + return Ok(()); + } + + tag_critical("Shutdown requested - capturing task state..."); + + // Step 1: Immediately freeze new task spawning + super::constants::INIT_IN_PROGRESS.store(true, std::sync::atomic::Ordering::SeqCst); + + // Step 2: Capture current running state immediately + tag_critical("Saving pending tasks to database..."); + dump_pending_tasks(ctx).await?; + + // Step 3: Cancel all tasks + tag_critical("Terminating active tasks..."); + cancel_token.cancel(); + + // Brief wait for tasks to shut down gracefully + tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await; + + // Step 4: Show final task summary using dump_active_tasks + let active_tasks = dump_active_tasks(); + let active_count = active_tasks.len(); + + tag_critical("=== SHUTDOWN SUMMARY ==="); + if active_count > 0 { + tag_critical(&format!("{} tasks were still active during shutdown", active_count)); + for task in active_tasks.iter() { + tag_critical(&format!(" • {} on {}", task.name, task.target)); + } + } else { + tag_critical("No tasks were running at shutdown"); + } + + // Check semaphore state + let available = TASK_SEMAPHORE.available_permits(); + tag_critical(&format!("Shutdown complete (semaphore: {} permits available)", available)); + tag_critical("Shutdown complete. Exiting in 3s..."); + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + + Ok(()) +} \ No newline at end of file diff --git a/src/workflow/spawning.rs b/src/workflow/spawning.rs new file mode 100755 index 0000000..93252dd --- /dev/null +++ b/src/workflow/spawning.rs @@ -0,0 +1,265 @@ +// src/workflow/spawning.rs +use tokio::time::Duration; +use tokio_util::sync::CancellationToken; +use std::sync::atomic::Ordering; +use super::concurrency::OperationType; +use crate::utilities::tag_info; +use super::constants::{INIT_IN_PROGRESS, MASSCAN_BLOCK_HOOKS, QUEUED_TASKS, TASK_SEMAPHORE}; +use super::tasks::{register_task, unregister_task, register_queued_task, unregister_queued_task, finalize_task_output}; + +// Context: Universal task spawning system with concurrency management +// Operation: Spawns tasks with permits, timeouts, cancellation, and queueing support +pub fn default_spawn( + name: impl Into + Send + 'static, + target: String, + op_type: OperationType, + start_immediate: bool, + factory: F, + timeout_s: Option, + global_cancel_token: Option, +) -> Option> +where + F: FnOnce(Option>, CancellationToken) -> Fut + + Send + + 'static, + Fut: std::future::Future + Send + 'static, +{ + let name_str = name.into(); + + // Create cancellation token for this task + let cancel_token = CancellationToken::new(); + + // Chain with global cancellation token if provided + let child_token = if let Some(global_token) = global_cancel_token { + // Create a token that cancels when either the task-specific OR global token is cancelled + let combined_token = CancellationToken::new(); + let combined_clone = combined_token.clone(); + let task_token_clone = cancel_token.clone(); + + // Spawn a task to watch for cancellation from either source + tokio::spawn(async move { + tokio::select! { + _ = global_token.cancelled() => { + combined_clone.cancel(); + } + _ = task_token_clone.cancelled() => { + combined_clone.cancel(); + } + } + }); + + combined_token + } else { + cancel_token.child_token() + }; + + // ——— Immediate path ——— + if start_immediate { + let (active, _, collector_handle) = register_task(&name_str, &target, cancel_token.clone()); + let raw_tx = Some(active.raw_tx.clone()); + let work_clone = active.clone(); + // `active` is NOT moved into the spawn: it drops when default_spawn returns, + // closing one sender ref before the spawn even starts running. + + let handle = tokio::spawn(async move { + // Check for global cancellation before acquiring permits + if child_token.is_cancelled() { + tag_info(&format!("Task '{}' on '{}' cancelled before start", name_str, target)); + unregister_task(&work_clone); + return; + } + + // Acquire both semaphores: concurrency-specific and global task limit + let _concurrency_permit = match super::concurrency::acquire_permit(op_type).await { + Ok(permit) => permit, + Err(e) => { + tag_info(&format!("Skipping {} on {}: {}", name_str, target, e)); + unregister_task(&work_clone); + return; + } + }; + let _permit = TASK_SEMAPHORE.acquire().await.unwrap(); + + // Spread out bursts of simultaneously-ready launches (e.g. after masscan + // discovery fires hooks for many hosts at once). + super::concurrency::stagger_launch().await; + + // Check for cancellation after acquiring permits + if child_token.is_cancelled() { + tag_info(&format!("Task '{}' on '{}' cancelled after acquiring permits", name_str, target)); + unregister_task(&work_clone); + return; + } + + // Create the work future + let work_future = factory(raw_tx, child_token.clone()); + + // Apply timeout using tokio::select! + let completion_status = if let Some(timeout_duration) = timeout_s.map(Duration::from_secs) { + tokio::select! { + _ = work_future => { + tag_info(&format!("Task '{}' on '{}' completed successfully", + work_clone.name, work_clone.target)); + "completed" + } + _ = tokio::time::sleep(timeout_duration) => { + tag_info(&format!("Task '{}' on '{}' timed out after {}s", + work_clone.name, work_clone.target, timeout_duration.as_secs())); + cancel_token.cancel(); + tokio::time::sleep(Duration::from_secs(5)).await; + "timeout" + } + _ = child_token.cancelled() => { + tag_info(&format!("Task '{}' on '{}' cancelled during execution", + work_clone.name, work_clone.target)); + "cancelled" + } + } + } else { + tokio::select! { + _ = work_future => { + tag_info(&format!("Task '{}' on '{}' completed successfully", + work_clone.name, work_clone.target)); + "completed" + } + _ = child_token.cancelled() => { + tag_info(&format!("Task '{}' on '{}' cancelled during execution", + work_clone.name, work_clone.target)); + "cancelled" + } + } + }; + + // Drain the collector before reading raw_buffer to avoid the race where + // persist reads a partially-filled buffer (factory's raw_tx already dropped + // above; active's was dropped when default_spawn returned). + finalize_task_output(work_clone, collector_handle, completion_status).await; + }); + + return Some(handle); + } + + // ——— Queued path ——— + let queued = register_queued_task(&name_str, &target, cancel_token.clone()); + QUEUED_TASKS.fetch_add(1, Ordering::SeqCst); + + tokio::spawn(async move { + // Check for cancellation in queue + if child_token.is_cancelled() { + tag_info(&format!("Queued task '{}' on '{}' cancelled while in queue", name_str, target)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + + // Wait for init to finish + while INIT_IN_PROGRESS.load(Ordering::SeqCst) { + // Check for cancellation while waiting + tokio::select! { + _ = tokio::task::yield_now() => {}, + _ = child_token.cancelled() => { + tag_info(&format!("Queued task '{}' on '{}' cancelled during init wait", name_str, target)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + } + } + + // FIX: Wait for masscan to complete before starting execution + while MASSCAN_BLOCK_HOOKS.load(Ordering::SeqCst) { + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis(100)) => {}, + _ = child_token.cancelled() => { + tag_info(&format!("Queued task '{}' on '{}' cancelled during masscan wait", name_str, target)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + } + } + + // Check for cancellation before acquiring permits + if child_token.is_cancelled() { + tag_info(&format!("Queued task '{}' on '{}' cancelled before acquiring permits", name_str, target)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + + // Acquire both semaphores: concurrency-specific first, then global task limit + let _concurrency_permit = match super::concurrency::acquire_permit(op_type).await { + Ok(permit) => permit, + Err(e) => { + tag_info(&format!("Skipping queued {} on {}: {}", name_str, target, e)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + }; + let _permit = TASK_SEMAPHORE.acquire().await.unwrap(); + + // Spread out bursts of simultaneously-ready launches (e.g. after masscan + // discovery fires hooks for many hosts at once). + super::concurrency::stagger_launch().await; + + // Check for cancellation after acquiring permits + if child_token.is_cancelled() { + tag_info(&format!("Queued task '{}' on '{}' cancelled after acquiring permits", name_str, target)); + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + return; + } + + // Move from queued to active + QUEUED_TASKS.fetch_sub(1, Ordering::SeqCst); + unregister_queued_task(&queued); + let (active, _, collector_handle) = register_task(&name_str, &target, cancel_token.clone()); + let raw_tx = Some(active.raw_tx.clone()); + let work_clone = active.clone(); + + // Create the work future + let work_future = factory(raw_tx, child_token.clone()); + + // Apply timeout using tokio::select! + let completion_status = if let Some(timeout_duration) = timeout_s.map(Duration::from_secs) { + tokio::select! { + _ = work_future => { + tag_info(&format!("Task '{}' on '{}' completed successfully", + work_clone.name, work_clone.target)); + "completed" + } + _ = tokio::time::sleep(timeout_duration) => { + tag_info(&format!("Task '{}' on '{}' timed out after {}s", + work_clone.name, work_clone.target, timeout_duration.as_secs())); + cancel_token.cancel(); + tokio::time::sleep(Duration::from_secs(5)).await; + "timeout" + } + _ = child_token.cancelled() => { + tag_info(&format!("Task '{}' on '{}' cancelled during execution", + work_clone.name, work_clone.target)); + "cancelled" + } + } + } else { + tokio::select! { + _ = work_future => { + tag_info(&format!("Task '{}' on '{}' completed successfully", + work_clone.name, work_clone.target)); + "completed" + } + _ = child_token.cancelled() => { + tag_info(&format!("Task '{}' on '{}' cancelled during execution", + work_clone.name, work_clone.target)); + "cancelled" + } + } + }; + + drop(active); + finalize_task_output(work_clone, collector_handle, completion_status).await; + }); + + None +} \ No newline at end of file diff --git a/src/workflow/tasks.rs b/src/workflow/tasks.rs new file mode 100755 index 0000000..c23f1f1 --- /dev/null +++ b/src/workflow/tasks.rs @@ -0,0 +1,249 @@ +// src/workflow/tasks.rs +use sqlx::Row; +use chrono::Utc; +use uuid::Uuid; +use std::sync::{Arc, Mutex, atomic::Ordering}; +use tokio_util::sync::CancellationToken; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use super::types::{TaskInfo, CompletedTaskRecord}; +use super::constants::{ACTIVE_TASKS, QUEUED_TASK_INFOS, WEB_ONLY_MODE, MAX_CONCURRENT_TASKS, TASK_SEMAPHORE, CURRENT_RUN_ID, TASK_HISTORY_TX}; + +// Context: Task registration system for active running tasks +// Operation: Registers task with output channel and adds to active tasks list +pub fn register_task( + name: &str, + target: &str, + cancel_token: CancellationToken, +) -> (TaskInfo, CancellationToken, tokio::task::JoinHandle<()>) { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let raw_buffer = Arc::new(Mutex::new(Vec::new())); + let collector = raw_buffer.clone(); + let handle = tokio::spawn(async move { + while let Some(line) = rx.recv().await { + collector.lock().unwrap().push(line); + } + }); + let info = TaskInfo { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + target: target.to_string(), + start_epoch: Utc::now().timestamp(), + raw_buffer, + raw_tx: tx, + cancel_token: cancel_token.clone(), + }; + ACTIVE_TASKS.lock().unwrap().insert(info.id.clone(), info.clone()); + (info, cancel_token, handle) +} + +// Context: Task cleanup system for completed tasks +// Operation: Removes task from active tasks list when complete +pub fn unregister_task(info: &TaskInfo) { + ACTIVE_TASKS.lock().unwrap().remove(&info.id); +} + +pub fn emit_task_history(name: &str, target: &str, started_at: i64, status: &str, output: String) { + let run_id = { + let g = CURRENT_RUN_ID.read().unwrap(); + match g.as_deref() { Some(id) => id.to_string(), None => return } + }; + let tx = { + let g = TASK_HISTORY_TX.read().unwrap(); + match g.clone() { Some(tx) => tx, None => return } + }; + let _ = tx.send(CompletedTaskRecord { + id: Uuid::new_v4().to_string(), + run_id, + task_name: name.to_string(), + target: target.to_string(), + started_at, + ended_at: Utc::now().timestamp(), + status: status.to_string(), + output, + }); +} + +// Drops all remaining sender references, waits for the collector to drain, then persists output. +// Must be called after the work future has completed (factory's raw_tx already dropped). +// The caller is responsible for dropping any other raw_tx clones (e.g. `active`) beforehand. +pub async fn finalize_task_output( + info: TaskInfo, + collector: tokio::task::JoinHandle<()>, + status: &str, +) { + let name = info.name.clone(); + let target = info.target.clone(); + let start = info.start_epoch; + let buffer = info.raw_buffer.clone(); + unregister_task(&info); // removes ACTIVE_TASKS entry, drops that sender ref + drop(info); // drops this copy's raw_tx, closing the channel + let _ = collector.await; // wait for collector to drain all remaining items + let output = buffer.lock().unwrap().join("\n"); + emit_task_history(&name, &target, start, status, output); +} + +// Context: Task registration system for queued pending tasks +// Operation: Registers task with output channel and adds to queued tasks list +pub fn register_queued_task(name: &str, target: &str, cancel_token: CancellationToken) -> TaskInfo { + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + let raw_buffer = Arc::new(Mutex::new(Vec::new())); + let collector = raw_buffer.clone(); + tokio::spawn(async move { + while let Some(line) = rx.recv().await { + collector.lock().unwrap().push(line); + } + }); + let info = TaskInfo { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + target: target.to_string(), + start_epoch: Utc::now().timestamp(), + raw_buffer, + raw_tx: tx, + cancel_token: cancel_token.clone(), + }; + QUEUED_TASK_INFOS.lock().unwrap().insert(info.id.clone(), info.clone()); + info +} + +// Context: Queued task cleanup system +// Operation: Removes task from queued tasks list when moved to active or cancelled +pub fn unregister_queued_task(info: &TaskInfo) { + QUEUED_TASK_INFOS.lock().unwrap().remove(&info.id); +} + +// Context: Shutdown persistence system for task state preservation +// Operation: Saves all active and queued tasks to database for resume functionality +pub async fn dump_pending_tasks(ctx: &ProjectContext) -> Result<(), Box> { + // Only clear existing tasks if NOT in web-only mode + if !WEB_ONLY_MODE.load(Ordering::SeqCst) { + sqlx::query("DELETE FROM pending_tasks") + .execute(&ctx.pool) + .await?; + } + + let mut total_dumped = 0; + + // 1. Active tasks (currently running with permits) - FIXED: Clone data before awaiting + let active_tasks_data: Vec = { + let active_tasks = ACTIVE_TASKS.lock().unwrap(); + tag_info(&format!("Found {} active tasks", active_tasks.len())); + active_tasks.values().cloned().collect() + }; // Lock is dropped here + + for t in active_tasks_data.iter() { + sqlx::query("INSERT INTO pending_tasks (id,name,target,status) VALUES (?1,?2,?3,'active')") + .bind(&t.id) + .bind(&t.name) + .bind(&t.target) + .execute(&ctx.pool) + .await?; + total_dumped += 1; + tag_debug(&format!(" Active: {} on {}", t.name, t.target)); + } + + // 2. Queued tasks (waiting for permits) - FIXED: Clone data before awaiting + let queued_tasks_data: Vec = { + let queued_tasks = QUEUED_TASK_INFOS.lock().unwrap(); + tag_info(&format!("Found {} queued tasks", queued_tasks.len())); + queued_tasks.values().cloned().collect() + }; // Lock is dropped here + + for t in queued_tasks_data.iter() { + sqlx::query("INSERT INTO pending_tasks (id,name,target,status) VALUES (?1,?2,?3,'queued')") + .bind(&t.id) + .bind(&t.name) + .bind(&t.target) + .execute(&ctx.pool) + .await?; + total_dumped += 1; + tag_debug(&format!(" Queued: {} on {}", t.name, t.target)); + } + + // 3. Check semaphore usage to see if we missed any + let stats = super::concurrency::get_concurrency_stats(); + let total_in_use: usize = stats.operation_stats.values().map(|s| s.in_use).sum(); + let active_count = active_tasks_data.len(); // Use cloned data length + + if total_in_use > active_count { + tag_info(&format!(" Semaphore mismatch: {} permits in use but only {} active tasks tracked", + total_in_use, active_count)); + tag_info("Some tasks may not be properly tracked in ACTIVE_TASKS"); + } + + // 4. Check global task semaphore + let global_permits_used = MAX_CONCURRENT_TASKS - TASK_SEMAPHORE.available_permits(); + if global_permits_used > active_count { + tag_info(&format!(" Global task semaphore mismatch: {} permits used but {} active tasks", + global_permits_used, active_count)); + } + + // 5. Add any database-tracked incomplete work as fallback + // (This catches tasks that might have been missed from in-memory tracking) + let incomplete_work = sqlx::query(" + SELECT ip, 'nmap_full_port_detection_scan' as task_type FROM targets + WHERE run_id = ? AND full_scan_started = TRUE AND full_scan_complete = FALSE + UNION + SELECT ip, 'nmap_service_scan' as task_type FROM targets + WHERE run_id = ? AND service_scan_started = TRUE AND service_scan_complete = FALSE + UNION + SELECT ip, 'http_discovery' as task_type FROM targets + WHERE run_id = ? AND http_discovery_started = TRUE AND http_discovery_complete = FALSE + ") + .bind(&ctx.run_id) + .bind(&ctx.run_id) + .bind(&ctx.run_id) + .fetch_all(&ctx.pool) + .await?; + + let mut db_backup_count = 0; + for row in incomplete_work { + let ip: String = row.get("ip"); + let task_type: String = row.get("task_type"); + + // Check if we already captured this task in memory + let already_captured = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM pending_tasks WHERE target = ? AND name = ?") + .bind(&ip) + .bind(&task_type) + .fetch_one(&ctx.pool) + .await? > 0; + + if !already_captured { + sqlx::query("INSERT INTO pending_tasks (id,name,target,status) VALUES (?1,?2,?3,'db_incomplete')") + .bind(format!("db_{}", uuid::Uuid::new_v4())) + .bind(&task_type) + .bind(&ip) + .execute(&ctx.pool) + .await?; + db_backup_count += 1; + tag_debug(&format!(" DB incomplete: {} on {}", task_type, ip)); + } + } + + if db_backup_count > 0 { + tag_info(&format!("Added {} incomplete tasks from database state", db_backup_count)); + total_dumped += db_backup_count; + } + + tag_info(&format!("Total captured {} tasks for resume", total_dumped)); + + // Summary by type + let summary = sqlx::query("SELECT status, COUNT(*) as count FROM pending_tasks GROUP BY status") + .fetch_all(&ctx.pool) + .await?; + + for row in summary { + let status: String = row.get("status"); + let count: i64 = row.get("count"); + tag_info(&format!(" {} {} tasks", count, status)); + } + + Ok(()) +} + +// Context: Task inspection system for monitoring +// Operation: Returns snapshot of all currently active tasks +pub fn dump_active_tasks() -> Vec { + ACTIVE_TASKS.lock().unwrap().values().cloned().collect() +} \ No newline at end of file diff --git a/src/workflow/types.rs b/src/workflow/types.rs new file mode 100755 index 0000000..31f37b6 --- /dev/null +++ b/src/workflow/types.rs @@ -0,0 +1,88 @@ +// src/workflow/types.rs +use serde::Serialize; +use tokio::sync::mpsc::UnboundedSender; +use tokio_util::sync::CancellationToken; +use std::sync::{Arc, Mutex}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Hash)] +pub enum Hook { + Ip, + Subnet, + SubnetPhase1Complete, + ServiceDiscoveryReady, + Ports, + Hostname, + ActionOutput, + ServiceScanComplete, + HttpAutoTagComplete, + ReconnaissanceComplete, +} + +/// Task record +#[derive(Clone, Serialize)] +pub struct TaskInfo { + pub id: String, + pub name: String, + pub target: String, + pub start_epoch: i64, + #[serde(skip)] + pub raw_buffer: Arc>>, + #[serde(skip)] + pub raw_tx: UnboundedSender, + #[serde(skip)] + pub cancel_token: CancellationToken, +} + +/// Completed task record for history persistence +#[derive(Clone)] +pub struct CompletedTaskRecord { + pub id: String, + pub run_id: String, + pub task_name: String, + pub target: String, + pub started_at: i64, + pub ended_at: i64, + pub status: String, + pub output: String, +} + +/// Workflow config +#[derive(Debug, serde::Deserialize, Clone)] +pub struct WorkflowConfig { + pub specific_actions: Vec, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] +pub struct TagPattern { + pub pattern: String, + pub tag: String, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] +pub struct SpecificActionStep { + pub name: String, + pub condition: Option, + pub command: String, + pub timeout_s: Option, + pub tags: Option>, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize, Clone)] +pub struct Condition { + // Port-based conditions with AND/OR support + pub contains: Option, + pub equals: Option, + + // Tag-based conditions with AND/OR support + pub has_tag: Option, + pub has_not_tag: Option, + + // Service-specific conditions + pub has_http_service: Option, + pub reconnaissance_complete: Option, + + // Dynamic service-presence check against port_services map (e.g. "mssql", "rdp OR winrm") + // Fires even when the service runs on a non-standard port + pub has_service: Option, +} + diff --git a/src/workflow/workflow.rs b/src/workflow/workflow.rs new file mode 100755 index 0000000..6714e7c --- /dev/null +++ b/src/workflow/workflow.rs @@ -0,0 +1,349 @@ +// src/workflow/workflow.rs +use tokio::signal::ctrl_c; +use tokio::time; +use futures::FutureExt; +use std::{ + collections::HashSet, + error::Error, + fs, + sync::{Arc, Mutex, atomic::Ordering}, +}; +use tokio_util::sync::CancellationToken; +use crate::sql::ProjectContext; +use crate::utilities::{tag_info, tag_debug}; +use crate::VERBOSE; +use super::types::{WorkflowConfig, SpecificActionStep}; +use super::constants::{INIT_IN_PROGRESS, RUNTIME_ACTIONS, GLOBAL_CANCEL_TOKEN, ACTIVE_TASKS, QUEUED_TASK_INFOS, MAX_CONCURRENT_TASKS}; +use super::conditions::evaluate_condition; +use super::shutdown::handle_shutdown; +use super::resume::process_resume_mode; +use super::host_queue::{try_dispatch_or_queue, PendingAction}; + +// Context: Workflow configuration loading from YAML files +// Operation: Loads and parses workflow configuration from specified file path +pub fn load_workflow(path: &str) -> Result> { + let yaml = fs::read_to_string(path)?; + Ok(serde_yaml::from_str(&yaml)?) +} + +// Context: Main workflow execution engine and polling system +// Operation: Runs workflow polling loop for specific actions with shutdown coordination +pub async fn run_workflow( + path: &str, + ctx: &ProjectContext, + mut shutdown_rx: tokio::sync::watch::Receiver, +) -> Result<(), Box> { + + // Create global cancellation token for shutdown coordination + let global_cancel_token = CancellationToken::new(); + *GLOBAL_CANCEL_TOKEN.write().unwrap() = Some(global_cancel_token.clone()); + let shutdown_cancel_token = global_cancel_token.clone(); + + // Load only specific_actions + let WorkflowConfig { specific_actions } = load_workflow(path)?; + let mut ctrl_c = ctrl_c().boxed(); + + // Handle resume mode if needed + process_resume_mode(ctx).await?; + + // Nuclei template readiness is validated once, up front, in validate_required_tools() + // (src/main.rs) before the project even starts — no need to repeat it here. + + INIT_IN_PROGRESS.store(false, Ordering::SeqCst); + + // Polling loop for specific_actions + // Arc so host_queue.rs can insert fired keys from re-evaluation closures + let fired = Arc::new(Mutex::new(HashSet::<(String, String)>::new())); + + loop { + tokio::select! { + // 1) Shutdown requested by TUI (via watch channel) + res = shutdown_rx.changed() => { + // ignore error: it only happens if sender is dropped + let _ = res; + return handle_shutdown(ctx, shutdown_cancel_token).await; + } + + // 2) Direct Ctrl-C (outside TUI) + _ = &mut ctrl_c => { + return handle_shutdown(ctx, shutdown_cancel_token).await; + } + + // 3) Normal polling every 2 seconds + _ = time::sleep(time::Duration::from_secs(2)) => { + // Get runtime actions first + let runtime_actions = get_runtime_actions(); + + // Combine file-based and runtime actions; wrap in Arc for per-host re-evaluation + let mut combined_actions = specific_actions.clone(); + combined_actions.extend(runtime_actions); + let all_actions = Arc::new(combined_actions); + + // C3: Fetch all host data in ONE query per tick (replaces N×M×4 queries) + let host_snapshots = match ctx.get_all_host_snapshots().await { + Ok(s) => s, + Err(e) => { + tag_info(&format!("Workflow tick: failed to fetch host snapshots: {}", e)); + continue; + } + }; + + for snapshot in host_snapshots.values() { + let ip = &snapshot.ip; + let ports: Vec = snapshot.ports + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + + // Gate all specific_actions on reconnaissance completion (from snapshot, no DB call) + let recon_complete = snapshot.full_scan_complete + && snapshot.service_scan_complete + && snapshot.http_discovery_complete; + if !recon_complete { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Reconnaissance not complete for {}: full:{} service:{} http:{}", + ip, snapshot.full_scan_complete, snapshot.service_scan_complete, snapshot.http_discovery_complete)); + } + continue; + } + + // Build tag list from snapshot once per host (no DB calls) + let mut tag_vec: Vec = Vec::new(); + for t in snapshot.action_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.manual_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.refined_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + for t in snapshot.automated_tags.split(',').map(str::trim).filter(|s| !s.is_empty()) { + tag_vec.push(t.to_string()); + } + + let mut dispatched_for_host = 0usize; + + for action in all_actions.iter() { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Reconnaissance complete for {} - evaluating action '{}'", ip, action.name)); + } + + // Condition evaluation — synchronous, no DB calls + let trigger = if let Some(condition) = &action.condition { + evaluate_condition(condition, &ports, &tag_vec, snapshot) + } else { + true // No condition = always trigger (if recon complete) + }; + + if !trigger { + if VERBOSE.load(Ordering::SeqCst) { + tag_debug(&format!("Action '{}' condition not met for {}", action.name, ip)); + } + continue; + } + + const INTAKE_CAP: usize = MAX_CONCURRENT_TASKS * 3; + + // Determine spawn mode from condition: + // has_service: "name" (simple, no AND/OR) → per-port instance fan-out + // has_http_service: true → per-HTTP-service fan-out + // everything else → single spawn + let svc_name_for_iter = action.condition.as_ref() + .and_then(|c| c.has_service.as_deref()) + .filter(|s: &&str| !s.contains(" AND ") && !s.contains(" OR ")); + + let http_iter = action.condition.as_ref() + .and_then(|c| c.has_http_service) + .unwrap_or(false); + + if let Some(svc_name) = svc_name_for_iter { + // Per-instance: one task per port running this service + let port_services_map = crate::sql::hosts::parse_port_services(&snapshot.port_services); + let mut matching: Vec = port_services_map.iter() + .filter(|(_, v)| v.eq_ignore_ascii_case(svc_name)) + .map(|(k, _)| k.clone()) + .collect(); + matching.sort(); + + for svc_port in matching { + let composite_key = format!("{}__{}", action.name, svc_port); + let fired_key = (composite_key.clone(), ip.clone()); + + if fired.lock().unwrap().contains(&fired_key) { continue; } + if ctx.is_action_complete(ip, &composite_key).await.unwrap_or(false) { continue; } + + let current_depth = { + let active = ACTIVE_TASKS.lock().unwrap(); + let queued = QUEUED_TASK_INFOS.lock().unwrap(); + active.len() + queued.len() + }; + if current_depth >= INTAKE_CAP { continue; } + + tag_info(&format!("Triggering action '{}' on {}:{} ({} instance)", action.name, ip, svc_port, svc_name)); + + if !ctx.mark_action_started(ip, &composite_key).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(fired_key); + + let pending = PendingAction { + composite_key, + action_def: action.clone(), + foreach_instance: Some((svc_port, None)), + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(ip, pending, ctx, global_cancel_token.clone()); + dispatched_for_host += 1; + } + + } else if http_iter { + // Per-instance: one task per HTTP service (port + protocol) + for (svc_port, svc_proto) in &snapshot.http_service_list { + let composite_key = format!("{}__{}", action.name, svc_port); + let fired_key = (composite_key.clone(), ip.clone()); + + if fired.lock().unwrap().contains(&fired_key) { continue; } + if ctx.is_action_complete(ip, &composite_key).await.unwrap_or(false) { continue; } + + let current_depth = { + let active = ACTIVE_TASKS.lock().unwrap(); + let queued = QUEUED_TASK_INFOS.lock().unwrap(); + active.len() + queued.len() + }; + if current_depth >= INTAKE_CAP { continue; } + + tag_info(&format!("Triggering action '{}' on {} ({}://{}:{})", + action.name, ip, svc_proto, ip, svc_port)); + + if !ctx.mark_action_started(ip, &composite_key).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(fired_key); + + let pending = PendingAction { + composite_key, + action_def: action.clone(), + foreach_instance: Some((svc_port.clone(), Some(svc_proto.clone()))), + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(ip, pending, ctx, global_cancel_token.clone()); + dispatched_for_host += 1; + } + + } else { + // Single-spawn path + let key = (action.name.clone(), ip.clone()); + if fired.lock().unwrap().contains(&key) { continue; } + if ctx.is_action_complete(ip, &action.name).await.unwrap_or(false) { continue; } + + let current_depth = { + let active = ACTIVE_TASKS.lock().unwrap(); + let queued = QUEUED_TASK_INFOS.lock().unwrap(); + active.len() + queued.len() + }; + if current_depth >= INTAKE_CAP { continue; } + + tag_info(&format!("Triggering action '{}' on {} (reconnaissance complete + conditions met)", action.name, ip)); + + if !ctx.mark_action_started(ip, &action.name).await.unwrap_or(false) { continue; } + fired.lock().unwrap().insert(key.clone()); + + let pending = PendingAction { + composite_key: action.name.clone(), + action_def: action.clone(), + foreach_instance: None, + ports: ports.clone(), + port_services_json: snapshot.port_services.clone(), + all_actions: all_actions.clone(), + fired: fired.clone(), + }; + try_dispatch_or_queue(ip, pending, ctx, global_cancel_token.clone()); + dispatched_for_host += 1; + } + } + + // Nothing dispatched this tick for a recon-complete host → mark workflow done + // if it is also not currently running anything + if dispatched_for_host == 0 + && crate::workflow::host_queue::is_host_idle(ip) + { + ctx.mark_workflow_complete(ip).await.ok(); + } + } + } + } + } + } + +// Context: Runtime action management for dynamic workflow updates +// Operation: Retrieves and clears runtime actions to prevent duplicate execution +pub fn get_runtime_actions() -> Vec { + let mut runtime_actions = RUNTIME_ACTIONS.lock().unwrap(); + let actions = runtime_actions.clone(); + runtime_actions.clear(); // Clear after getting to avoid duplicate execution + actions +} + +// Context: YAML workflow file management with action persistence +// Operation: Safely appends new action to workflow file with proper formatting +pub async fn save_action_to_workflow(workflow_path: &str, action: &SpecificActionStep) -> Result<(), Box> { + // Verify the workflow file exists + if !tokio::fs::try_exists(workflow_path).await.unwrap_or(false) { + return Err(format!("Workflow file not found: {}", workflow_path).into()); + } + + // Read current workflow file + let mut workflow_content = tokio::fs::read_to_string(workflow_path).await?; + + let action_yaml = serde_yaml::to_string(action)?; + + // Find the specific_actions section and append (EXACT SAME LOGIC) + if let Some(_pos) = workflow_content.find("specific_actions:") { + // Simple append: add the new action with proper indentation + let indented_action = action_yaml + .lines() + .enumerate() + .map(|(i, line)| { + if i == 0 { + format!(" - {}", line) // First line gets "- " prefix + } else if !line.trim().is_empty() { + format!(" {}", line) // Other lines get extra indentation + } else { + line.to_string() + } + }) + .collect::>() + .join("\n"); + + workflow_content.push_str("\n"); + workflow_content.push_str(&indented_action); + } else { + // No specific_actions section exists, add it + workflow_content.push_str("\n\nspecific_actions:\n"); + let indented_action = action_yaml + .lines() + .enumerate() + .map(|(i, line)| { + if i == 0 { + format!(" - {}", line) + } else if !line.trim().is_empty() { + format!(" {}", line) + } else { + line.to_string() + } + }) + .collect::>() + .join("\n"); + workflow_content.push_str(&indented_action); + } + + // Write back to file + tokio::fs::write(workflow_path, workflow_content).await?; + + Ok(()) +} \ No newline at end of file diff --git a/templates/action_builder.html b/templates/action_builder.html new file mode 100755 index 0000000..82dc768 --- /dev/null +++ b/templates/action_builder.html @@ -0,0 +1,1034 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Action Builder{% endblock %} + +{% block content %} +
+
+

Live Action Builder

+
+

Create and execute custom actions dynamically. Build actions that mirror your workflow.yaml structure.

+
+
+ + +
+
+

Action Definition

+

Define the basic action properties and command

+
+
+
+
+ + + Unique identifier for this action +
+ +
+ + + Full command — Placeholders: {{IP}} {{PORT}} {{URL}} {{PROTOCOL}} — per-instance: {{SERVICE_PORT}} {{HTTP_URL}} {{HTTP_PORT}} {{HTTP_PROTOCOL}} +
+ +
+ + + Maximum execution time per host +
+ +
+
+ + +
+
+

Execution Conditions

+

Define when this action should execute

+
+
+
+ +
+

Port Conditions

+
+ + + Use AND/OR logic for multiple ports +
+
+ + + Exact and strict port match, ex : 22 -> host with only port 22 opened will be targeted ! +
+
+ + +
+

Tag Conditions

+
+ +
+ + +
+ Use AND/OR logic for multiple tags +
+
+ +
+
+ + +
+

Service Conditions

+
+ + + Single name → per-port fan-out. Use exact name: mssql, rdp, smb, ftp… AND/OR = boolean gate only. +
+
+ + + +
+
+
+
+
+ + +
+
+

Output Tag Patterns

+

Define what tags to apply based on text found in command output

+
+
+
+
+ +
+ +
+
+
+ + +
+
+

Target Preview

+

Hosts that will be targeted by this action

+
+
+
+
+
+ 0 + Hosts will be targeted +
+
+
+ + +
+
+
+ +
+
+
+ + + +
+ + + + + +{% endblock %} \ No newline at end of file diff --git a/templates/available_tags.html b/templates/available_tags.html new file mode 100755 index 0000000..0049cff --- /dev/null +++ b/templates/available_tags.html @@ -0,0 +1,1079 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Available Tags{% endblock %} + +{% block content %} +
+
+

Tags

+
+ {{ categories.len() }} categories discovered +
+
+ + +
+
+
+ + + {% if !search_query.is_empty() %} + Clear + {% endif %} +
+ + {% if !search_query.is_empty() %} + {% if search_results.len() > 0 %} +
+
+

Results for "{{ search_query }}"

+ {{ search_results.len() }} host(s) found +
+
+ + + + + + + + + + + + + {% for host in search_results %} + + + + + + + + + + {% endfor %} + +
IP AddressHostnamePortsTagsHTTP ServicesStatusActions
{{ host.ip }} + {% if host.hostname != "" %} + {{ host.hostname }} + {% if host.ad_domain != "" %} + ({{ host.ad_domain }}) + {% endif %} + {% else %} + - + {% endif %} + + {% if host.ports_count > 0 %} + {{ host.ports_count }} ports + {% else %} + No ports + {% endif %} + + {% if host.tags_with_source.len() > 0 %} +
+ {% for tag in host.tags_with_source %} + {{ tag.name }} + {% endfor %} +
+ {% else %} + No tags + {% endif %} + 		+ {% if host.http_services_count > 0 %} + {{ host.http_services_count }} services + {% else %} + - + {% endif %} + +
+ P + S + H + SC +
+ 		+ Details +
+ + {% else %} +
+ No hosts found with tag "{{ search_query }}" +
+ {% endif %} + {% else %} +

Enter a tag above to find matching hosts.

+ {% endif %} + + + {% if categories.len() > 0 %} +
+

Browse all discovered tags organized by category. Click any tag to search for hosts with that tag.

+
+ + +
+
+

Export Hosts by Tags

+

Select tags below and choose your export format

+
+ +
+
+

Selected Tags:

+
+ No tags selected - click tags below to add them +
+ +
+ +
+
+
+ + +
+ +
+
+
+
+ + +
+
+ + +
+
+ + +
+ {% for category in categories %} +
+

{{ category.name }}

+
+ {% for tag in category.tags %} +
+ {{ tag.name|trim }} + {{ tag.count }} +
+ + +
+
+ {% endfor %} +
+
+ {% endfor %} +
+ + + + + {% else %} +
+
+

No Tags Available

+

No tags have been discovered yet. Tags are automatically generated during reconnaissance based on:

+
    +
  • Operating Systems: Windows, Linux, FreeBSD
    • +
  • Web Servers: Apache, nginx, IIS
    • +
  • Databases: MySQL, PostgreSQL, MongoDB
    • +
  • Network Services: SSH, SMB, RDP, FTP
    • +
  • Applications: Jenkins, GitLab, Grafana
    • +
  • Hardware: Printers, Network devices
    • +
+

Tags will appear here as reconnaissance progresses and services are discovered.

+ View All Hosts +
+
+ {% endif %} + + + + + +{% endblock %} \ No newline at end of file diff --git a/templates/base.html b/templates/base.html new file mode 100755 index 0000000..cc984c4 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,1265 @@ + + + + + + ROFMAD - {% block title %}{% endblock %} + + + + {% block extra_head %}{% endblock %} + + + + + + +
+ {% block content %}{% endblock %} +
+ + + + {% block extra_js %}{% endblock %} + + \ No newline at end of file diff --git a/templates/dashboard.html b/templates/dashboard.html new file mode 100755 index 0000000..1242603 --- /dev/null +++ b/templates/dashboard.html @@ -0,0 +1,1134 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Dashboard{% endblock %} + +{% block content %} +
+ +
+

ROFMAD Dashboard

+

Real-time reconnaissance and vulnerability assessment overview

+
+ + +
+
+

Total Hosts

+ {{ stats.total_hosts }} + Discovered targets +
+
+

Total Subnets

+ {{ stats.total_subnets }} + Network segments +
+
+

Active Tasks

+ {{ stats.active_tasks.len() }} + Currently running +
+
+

Queued Tasks

+ {{ stats.queued_tasks_count }} + Pending execution +
+
+ + + {% if stats.total_hosts == 0 %} +
+
+

Welcome to ROFMAD

+

No targets have been discovered yet. Get started by adding targets or configuring network scanning.

+
+ Go to Scanners + Configure Settings +
+
+
+ {% endif %} + + +
+
+

+ Discovery Statistics +

+

Overview of reconnaissance progress and findings

+
+
+
+
+

Hosts with Open Ports

+
+ {{ stats.hosts_with_open_ports }} + {{ stats.total_open_ports }} total ports discovered +
+
+
+

Hosts with HTTP Services Found

+
+ {{ stats.hosts_with_http_services }} + Http services found by probbing +
+
+
+

Screenshots Captured

+
+ {{ stats.total_screenshots }} + web interface captures +
+
+
+
+
+ + +
+
+

+ Hosts Ready for Analysis +

+

Targets that have completed all reconnaissance phases and are ready for manual + security assessment

+
+
+ {% if stats.hosts_ready_for_analysis.len() > 0 %} +
+ {% for host in stats.hosts_ready_for_analysis %} +
+
+

{{ host.ip }}

+ {% if host.hostname != "" %} + {{ host.hostname }} + {% endif %} +
+
+
+ {{ host.ports_count }} + Ports +
+
+ {{ host.http_services_count }} + HTTP +
+
+ {{ host.action_outputs_count }} + Actions +
+
+
+ {% for tag in host.tags_with_source %} + {% if loop.index <= 6 %} {{ tag.name }} + {% endif %} + {% endfor %} + {% if host.tags_with_source.len() > 6 %} + +{{ host.tags_with_source.len() - 6 }} more + {% endif %} +
+
+ Analyze +
+
+ {% endfor %} +
+ {% else %} +
+
+

Scanning in progress

+

Hosts will appear here once all scanning phases and workflow actions are complete.

+
+
+ {% endif %} +
+
+ + +
+
+

Nuclei Findings

+

Hosts with nuclei vulnerability scan results ready for review

+
+
+ {% if stats.hosts_with_nuclei_findings.len() > 0 %} +
+ {% for host in stats.hosts_with_nuclei_findings %} +
+
+

{{ host.ip }}

+ {% if host.hostname != "" %} + {{ host.hostname }} + {% endif %} +
+
+ {% if host.nuclei_severity_counts.critical > 0 %} +
+ {{ host.nuclei_severity_counts.critical }} + Critical +
+ {% endif %} + {% if host.nuclei_severity_counts.high > 0 %} +
+ {{ host.nuclei_severity_counts.high }} + High +
+ {% endif %} + {% if host.nuclei_severity_counts.medium > 0 %} +
+ {{ host.nuclei_severity_counts.medium }} + Medium +
+ {% endif %} + {% if host.nuclei_severity_counts.low > 0 %} +
+ {{ host.nuclei_severity_counts.low }} + Low +
+ {% endif %} + {% if host.nuclei_severity_counts.info > 0 %} +
+ {{ host.nuclei_severity_counts.info }} + Info +
+ {% endif %} + {% if host.nuclei_severity_counts.unknown > 0 %} +
+ {{ host.nuclei_severity_counts.unknown }} + Unknown +
+ {% endif %} +
+
+ View Findings +
+
+ {% endfor %} +
+ {% else %} +
+
+

No Nuclei Findings

+

Use the "Run Nuclei" button on any HTTP service to scan for vulnerabilities.

+
+
+ {% endif %} +
+
+ + + {% if stats.active_tasks.len() > 0 %} +
+
+

+ Active Tasks +

+

Real-time monitoring of running reconnaissance tasks and their progress

+
+
+
+ + + + + + + + + + + {% for task in stats.active_tasks %} + + + + + + + {% endfor %} + +
TaskTargetDurationActions
{{ task.name }}{{ task.target }}{{ task.duration }} + Detail + +
+
+
+
+ {% endif %} + + +
+
+

+ System Performance +

+

Monitor system resources and task execution efficiency

+
+
+
+
+

Task Queue

+
+ {{ stats.active_tasks.len() }}/{{ stats.queued_tasks_count + }} + Active/Queued +
+
+
+

Discovery Progress

+
+ {{ stats.hosts_with_open_ports }}/{{ stats.total_hosts }} + Hosts Scanned +
+
+
+
+
+ + +
+
+

+ Concurrency Status +

+

Real-time semaphore usage and resource allocation

+
+
+
+ {% for group in stats.concurrency_stats.display_groups %} +
+

{{ group.name }}

+
+
+ {{ group.in_use }}/{{ group.total_limit }} +
+
+
+
+
+
+
+ {% endfor %} +
+
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/templates/host_detail.html b/templates/host_detail.html new file mode 100755 index 0000000..1a26cd3 --- /dev/null +++ b/templates/host_detail.html @@ -0,0 +1,1577 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}{{ host.ip }}{% endblock %} + +{% block content %} +
+ +
+
+
+

{{ host.ip }}

+ {% if host.short_hostname != "" %} +

{{ host.short_hostname }}

+ {% else if host.hostname != "" %} +

{{ host.hostname }}

+ {% endif %} + {% if host.ad_domain != "" %} +
Domain: {{ host.ad_domain }}
+ {% endif %} +
+ + +
+
+ {{ host.ports.len() }} + Open Ports +
+
+ {{ host.http_services.len() }} + HTTP Services +
+
+ {{ host.action_findings.len() + host.action_other.len() }} + Action Outputs +
+
+ {{ host.screenshots.len() }} + Screenshots +
+
+ {{ host.nuclei_findings_count }} + Nuclei Findings +
+
+
+ + +
+

Scan Progress

+
+
+ +
+
+
+ {{ host.progress.completion_percentage }}% +
+
+ +
+ + Port Discovery {% if host.progress.full_scan %}[done]{% else %}[...]{% endif %} + + + Service Detection {% if host.progress.service_scan %}[done]{% else %}[...]{% endif %} + + + HTTP Services {% if host.progress.http_discovery %}[done]{% else %}[...]{% endif %} + + + Screenshots {% if host.progress.screenshots %}[done]{% else %}[...]{% endif %} + +
+
+
+ + {% if host.tags.len() > 0 %} +
+

Tags (Manual & Refined)

+
+ {% for tag in host.tags %} + {{ tag.name }} + {% endfor %} +
+
+ {% endif %} + +
+

Manual Tags

+
+
+ + +
+
+
+ +
+
+ + +
+

Analysis Status

+
+
+
+ + Bookmark this host +
+
+
+
+ + +
+

Host Management

+
+ + + + {% if host.http_services.len() > 0 %} + + {% endif %} +
+
+ + + + + + {% if host.ports.len() > 0 %} +
+

Open Ports ({{ host.ports.len() }})

+
+ + + + + + + + + + + + {% for port in host.ports %} + + + + + + + + {% endfor %} + +
PortProtocolStateServiceVersion / Info
{{ port.port }}{{ port.protocol }} + {{ port.state }} + {{ port.service }} + {% if port.version != "" %}{{ port.version }}{% endif %} + {% if port.extrainfo != "" %}({{ port.extrainfo }}){% endif %} + {% if port.cpe != "" %}{{ port.cpe }}{% endif %} +
+
+
+ {% endif %} + + + {% if host.http_services.len() > 0 %} +
+

HTTP Services ({{ host.http_services.len() }})

+
+ {% for entry in host.http_services %} + {% if entry.1.probe_failed %} +
+
+

{{ entry.1.protocol }}://{{ host.ip }}:{{ entry.0 }}

+ Probe Failed +
+
+
+ Status: + Port open — HTTP probe failed during scan +
+
+ + {% if host.nuclei_scanned_ports.contains(entry.0) %} + Nuclei Scanned + + {% else %} + + {% endif %} +
+ {% else %} +
+
+

{{ entry.1.protocol }}://{{ host.ip }}:{{ entry.0 }}

+ {{ entry.1.status_code }} +
+
+
+ Server: + {{ entry.1.server_header }} +
+
+ Title: + {{ entry.1.title }} +
+
+ Response Time: + {{ entry.1.response_time_ms }}ms +
+
+ Content Length: + {{ entry.1.content_length }} bytes +
+
+ {% if host.nuclei_scanned_ports.contains(entry.0) %} + Nuclei Scanned + + {% else %} + + {% endif %} +
+ {% endif %} + {% endfor %} +
+
+ {% endif %} + + + {% if host.screenshots.len() > 0 %} +
+

Screenshots ({{ host.screenshots.len() }})

+
+ {% for screenshot in host.screenshots %} +
+
+

{{ screenshot.url }}

+ Port {{ screenshot.port }} +
+
+ Screenshot of {{ screenshot.url }} +
+
+ {% endfor %} +
+
+ {% endif %} + + + {% if host.nuclei_findings_count > 0 %} +
+

Nuclei Findings {{ host.nuclei_findings_count }}

+ {% for result in host.nuclei_results %} + {% if result.findings.len() > 0 %} +
+
+ Port {{ result.port }} + {% if result.url != "" %} + {{ result.url }} + {% endif %} + {{ result.findings.len() }} unique finding(s) +
+
+ {% for f in result.findings %} +
+
+ {{ f.severity|upper }} + {{ f.name }} + [{{ f.template_id }}] + {{ f.matched_urls.len() }} match(es) +
+
+ {% for url in f.matched_urls %} + {{ url }} + {% endfor %} +
+ {% if f.description != "" %} +

{{ f.description }}

+ {% endif %} + {% if f.references.len() > 0 %} +
+ {% for r in f.references %} + {{ r }} + {% endfor %} +
+ {% endif %} +
+ {% endfor %} +
+
+ {% endif %} + {% endfor %} +
+ {% endif %} + + + {% if host.action_findings.len() > 0 %} +
+

Action Findings {{ host.action_findings.len() }}

+ {% for action in host.action_findings %} +
+ + ► + {{ action.name }} + + exit {{ action.exit_code }} + {{ action.duration_ms }}ms + ✓ tagged + + + {% if action.command != "" %} +
+ {{ action.command }} + +
+ {% endif %} + 
{{ action.output }}
+
+ +
+
+ {% endfor %} +
+ {% endif %} + + + {% if host.action_other.len() > 0 %} +
+

Other Actions {{ host.action_other.len() }}

+ {% for action in host.action_other %} +
+ + ► + {{ action.name }} + + exit {{ action.exit_code }} + {{ action.duration_ms }}ms + + + {% if action.command != "" %} +
+ {{ action.command }} + +
+ {% endif %} + 
{{ action.output }}
+
+ +
+
+ {% endfor %} +
+ {% endif %} + + + {% if host.nmap_scan_raw != "" %} + {% if host.nmap_scan_raw != "{}" %} +
+

Nmap Scan Results

+
+
+ Raw Nmap Output + +
+ 
{{ host.nmap_scan_raw }}
+
+
+ {% endif %} + {% endif %} + +
+ + + + + +{% endblock %} \ No newline at end of file diff --git a/templates/hosts.html b/templates/hosts.html new file mode 100755 index 0000000..33af674 --- /dev/null +++ b/templates/hosts.html @@ -0,0 +1,682 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Hosts{% endblock %} + +{% block content %} +
+
+

Discovered Hosts

+
+ {{ hosts.len() }} hosts total + | + {{ hosts.len() }} showing +
+
+ + +
+
+ + +
+
+ +
+ Page 1 of 1 + + +
+
+
+ + +
+ Filter: + All + ★ Bookmarked + Not Bookmarked + Ready to Analyze + Analyzed +
+ + + {% if filter == "analyzed_ready" && hosts.is_empty() %} +
+ ◎ + Scanning in progress — no hosts have completed all workflow actions yet +
+ {% endif %} + + +
+ + + + + + + + + + + + + + + + + + {% for host in hosts %} + + + + + + + + + + + + + + {% endfor %} + +
+
+ IP Address +
+ ▲ + ▼ +
+
+ 		+
+ Hostname +
+ ▲ + ▼ +
+
+ 		Domain +
+ Ports +
+ ▲ + ▼ +
+
+ 		Tags +
+ HTTP Services +
+ ▲ + ▼ +
+
+ 		Status +
+ User Analysis +
+ ▲ + ▼ +
+
+ 		+
+ Nuclei Review +
+ ▲ + ▼ +
+
+ 		+
+ Bookmark +
+ ▲ + ▼ +
+
+ 		Actions
{{ host.ip }} + {% if !host.short_hostname.is_empty() %} + {{ host.short_hostname }} + {% else %} + - + {% endif %} + + {% if !host.ad_domain.is_empty() %} + {{ host.ad_domain }} + {% else %} + - + {% endif %} + + {% if host.ports_count > 0 %} + {{ host.ports_count }} ports + {% else %} + No ports + {% endif %} + + {% if host.tags_with_source.len() > 0 %} +
+ {% for tag in host.tags_with_source %} + {% if loop.index0 % 3 == 0 %}
{% endif %} + {{ tag.name }} + {% if loop.index0 % 3 == 2 || loop.last %}
{% endif %} + {% endfor %} +
+ {% else %} + No tags + {% endif %} + 		+ {% if host.http_services_count > 0 %} + {{ host.http_services_count }} + {% else %} + - + {% endif %} + +
+ P + S + H + SC +
+ 		+
+ {% if host.analyzed %} +
+ Analyzed (undo) +
+ {% else if host.full_scan_complete && host.service_scan_complete && host.http_discovery_complete && host.screenshots_complete %} +
+ Mark Analyzed +
+ {% else %} +
+ In Progress +
+ {% endif %} +
+ 		+
+ {% if host.nuclei_reviewed %} +
+ Reviewed (undo) +
+ {% else if host.has_nuclei_findings %} +
+ Mark Reviewed +
+ {% else %} +
+ N/A +
+ {% endif %} +
+ 		+ {% if host.interesting %} + + {% else %} + + {% endif %} + Details
+
+
+ + +{% endblock %} diff --git a/templates/hosts_rows.html b/templates/hosts_rows.html new file mode 100644 index 0000000..34216bf --- /dev/null +++ b/templates/hosts_rows.html @@ -0,0 +1,76 @@ +{% for host in hosts %} + + {{ host.ip }} + + {% if !host.hostname.is_empty() %}{{ host.hostname }}{% endif %} + {% if !host.ad_domain.is_empty() %}({{ host.ad_domain }}){% endif %} + {% if host.hostname.is_empty() && host.ad_domain.is_empty() %}-{% endif %} + + + {% if host.ports_count > 0 %} + {{ host.ports_count }} ports + {% else %} + No ports + {% endif %} + + + {% if host.tags_with_source.len() > 0 %} +
+ {% for tag in host.tags_with_source %} + {% if loop.index0 % 3 == 0 %}
{% endif %} + {{ tag.name }} + {% if loop.index0 % 3 == 2 || loop.last %}
{% endif %} + {% endfor %} +
+ {% else %} + No tags + {% endif %} + + + {% if host.http_services_count > 0 %} + {{ host.http_services_count }} + {% else %} + - + {% endif %} + + +
+ P + S + H + SC +
+ + +
+ {% if host.analyzed %} +
+ Analyzed (undo) +
+ {% else if host.full_scan_complete && host.service_scan_complete && host.http_discovery_complete && host.screenshots_complete %} +
+ Mark Analyzed +
+ {% else %} +
+ In Progress +
+ {% endif %} +
+ + +
+ {% if host.interesting %} +
+ Bookmarked +
+ {% else %} +
+ Bookmark +
+ {% endif %} +
+ + Details + +{% endfor %} diff --git a/templates/login.html b/templates/login.html new file mode 100755 index 0000000..2e73243 --- /dev/null +++ b/templates/login.html @@ -0,0 +1,233 @@ + + + + + + + ROFMAD — Login + + + + + + + + + + + \ No newline at end of file diff --git a/templates/logs.html b/templates/logs.html new file mode 100755 index 0000000..c4e66ef --- /dev/null +++ b/templates/logs.html @@ -0,0 +1,476 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}System Logs{% endblock %} + +{% block content %} +
+
+

System Logs

+
+ +
+ + +
+ + +
+
+
+ + + + {% if selected_level != "" %} + Clear + {% endif %} +
+
+ +
+ + +
+
+
+ + {% if logs.len() > 0 %} +
+
+ + {{ logs.len() }} entries + {% if selected_level != "" %} · filtered by {{ selected_level }}{% endif %} + + +
+
+ + + + + + + + + + {% for log in logs %} + + + + + + {% endfor %} + +
TimestampLevelMessage
{{ log.timestamp }} + {{ log.level }} + {{ log.message }}
+
+
+ {% else %} +
+

{% if selected_level != "" %}No {{ selected_level }} entries found.{% else %}No log entries yet.{% endif %}

+
+ {% endif %} +
+ + + +
+ + +{% endblock %} diff --git a/templates/queue.html b/templates/queue.html new file mode 100755 index 0000000..940ee57 --- /dev/null +++ b/templates/queue.html @@ -0,0 +1,528 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Task Queue{% endblock %} + +{% block content %} +
+
+

Task Queue Management

+
+ {{ active_tasks.len() }} active | {{ queued_tasks.len() }} queued +
+
+ + +
+
+

Semaphore Status

+
+
+
+ {% for group in concurrency_stats.display_groups %} +
+

{{ group.name }}

+
+
+ {{ group.in_use }}/{{ group.total_limit }} +
+
+
+
+
+
+ {% endfor %} +
+
+
+ + + {% if active_tasks.len() > 0 %} +
+
+

Active Tasks ({{ active_tasks.len() }})

+
+
+
+ + + + + + + + + + + {% for task in active_tasks %} + + + + + + + {% endfor %} + +
TaskTargetDurationActions
{{ task.name }}{{ task.target }}{{ task.duration }} + Detail + +
+
+
+
+ {% else %} +
+

No Active Tasks

+

No tasks are currently running. Tasks will appear here when system is executing work.

+
+ {% endif %} + + + {% if queued_tasks.len() > 0 %} +
+
+

Queued Tasks ({{ queued_tasks.len() }})

+
+
+
+ + + + + + + + + + + {% for task in queued_tasks %} + + + + + + + {% endfor %} + +
TaskTargetQueued DurationActions
{{ task.name }}{{ task.target }}{{ task.duration }} + Detail + +
+
+
+
+ {% else %} +
+

No Queued Tasks

+

All tasks are either running or completed. Queue will show here when system is under load.

+
+ {% endif %} +
+ + +{% endblock %} \ No newline at end of file diff --git a/templates/scanners.html b/templates/scanners.html new file mode 100755 index 0000000..95bb3e9 --- /dev/null +++ b/templates/scanners.html @@ -0,0 +1,1393 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Scanners{% endblock %} + +{% block content %} +
+
+
+

Network Discovery Arsenal

+

Launch comprehensive network reconnaissance and enumeration tools

+
+
+ +
+
+
+

Target Discovery

+

Add specific targets for focused reconnaissance operations

+
+
+ +
+
+
+
+

Individual Target

+

Add specific IP addresses or hostnames for targeted scanning

+
+
+
+
+ + +
+ +
+
+
+ +
+ +
+
+ +
+
+
+ + +
+ Separators: comma, semicolon, space, newline | Max 1000 + entries +
+
+ +
+
+
+
+ +
+
+
+

Subnet Scanning

+

Launch high-speed Masscan discovery across network ranges

+
+
+
+
+ + +
+ +
+
+
+ +
+
+ + Private Ranges: + 10.0.0.0/8 + 172.16.0.0/12 + 192.168.0.0/16 + +
+
+ Scans automatically exclude localhost, host interfaces, and + blacklisted ranges +
+ +
+
+ +
+
+
+ + +
+ Separators: comma, semicolon, space, newline | Max 1000 + entries +
+
+ +
+
+
+
+
+
+ +
+
+
+

Local Network Intelligence

+

Deploy passive and active reconnaissance tools for local network discovery

+
+
+ +
+
+
+
+

ARP Discovery

+

Actively discover hosts on the local network segment using ARP requests

+
+
+
+ +
+
+ +
+
+
+

DNS Intelligence

+

Extract DNS server information and discover network infrastructure

+
+
+
+ +
+
+ +
+
+
+

Interface Explorer

+

Analyze network interfaces, routes, and topology for reconnaissance planning

+
+
+
+ +
+
+ +
+
+
+

Passive Listener

+

Monitor network traffic for mDNS, LLMNR, NetBIOS, and broadcast discovery

+
+
+
+ +
+
+
+
+ +
+
+
+

Vulnerability Assessment

+

Run vulnerability scans across discovered HTTP services

+
+
+ +
+
+
+
+

Nuclei — All HTTP Services

+

Run nuclei against every discovered HTTP service. Optionally filter hosts by tag.

+
+
+
+ +
+
+
+
+
+ + + + +{% endblock %} \ No newline at end of file diff --git a/templates/settings.html b/templates/settings.html new file mode 100755 index 0000000..e6e632b --- /dev/null +++ b/templates/settings.html @@ -0,0 +1,1126 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Settings{% endblock %} + +{% block content %} +
+
+

Project Settings

+
+
+
+ + {% if current_user_role.as_str() == "admin" %} + +
+
+

User Management

+
+
+
+
+

Create Player Account

+

Create read-only accounts for team members to monitor reconnaissance + data

+
+
+ + + Minimum 3 characters, alphanumeric +
+
+ +
+ + +
+
+ +
+

Current Player Accounts

+
+ {% if player_users.len() > 0 %} + {% for user in player_users %} +
+ + +
+ {% endfor %} + {% else %} +
No player accounts created yet
+ {% endif %} +
+
+
+
+
+ {% endif %} + + +
+
+

Concurrency Limits

+
+
+ {% if current_user_role.as_str() == "admin" %} +
+ Controls simultaneous subprocess count. Lower per-type limits to preserve client network + bandwidth — avoid saturating the target /24 gateway. The only scanner-side hard limit is + the OS ephemeral port pool: {{ ephemeral_ports }} ports + available — approach this and outbound connections will stall. +
+ +
+
+ +
+
+ + process ceiling +
+ +
+ ≤100 safe + 101–200 caution + 201+ danger +
+

Hard ceiling on total concurrent subprocesses. Set above the sum of all per-type limits below.

+ suggested: 100 +
+ +
+
+ + ~300 SYN/scan +
+ +
+ ≤10 safe + 11–20 caution + 21+ danger +
+

nmap --privileged -p- -T4 per host. Raw SYN probes (~300 in-flight). No port pool cost — client network bandwidth is the concern.

+ suggested: 5 +
+ +
+
+ + ~30 TCP/scan +
+ +
+ ≤15 safe + 16–25 caution + 26+ danger +
+

nmap -sV -sC on open ports only (T3 default). Full TCP connects for banner grab — consumes ephemeral ports. Scripts may linger 1800 s.

+ suggested: 10 +
+ +
+
+ + ~150 SYN/sweep +
+ +
+ ≤5 safe + 6–10 caution + 11+ danger +
+

nmap --privileged 14-port sweep on /24 (254 hosts). Raw SYN — no port pool cost. Each concurrent sweep adds ~150 SYN probes on the client network.

+ suggested: 1 +
+ +
+
+ + ~400 SYN/sweep +
+ +
+ ≤2 safe + 3–4 caution + 5+ danger +
+

nmap --privileged -p- on entire /24. Raw SYN, most sustained scan — 2 concurrent can saturate a slower client uplink for hours.

+ suggested: 1 +
+ +
+
+ + ~3 TCP/probe +
+ +
+ ≤50 safe + 51–100 caution + 101+ danger +
+

curl per port (--max-time 15, -L --max-redirs 3). Full TCP — consumes ephemeral ports (~3 per probe). Minimal bandwidth impact on client network.

+ suggested: 10 +
+ +
+
+ +
+ +

Minimum delay (ms) enforced between successive subprocess launches, across ALL task types. Spreads out bursts (e.g. when masscan discovery fires hooks for many hosts at once) so CPU/RAM/network don't spike all at once. Set to 0 to turn this setting completely off.

+ suggested: 2000 +
+ +
+ +
+
+ Client network load: + — + +
+
+ Scanner port pool: + — + +
+
+ + +
+ + + + {% else %} +
+
+ Global Cap + {{ concurrency_config.global }} + process ceiling +
+
+ Nmap Full TCP + {{ concurrency_config.nmap_full_tcp }} + ~300 SYN/scan +
+
+ Nmap Service + {{ concurrency_config.nmap_service }} + ~30 TCP/scan +
+
+ Subnet Phase 1 + {{ concurrency_config.subnet_phase1 }} + ~150 SYN/sweep +
+
+ Subnet Phase 2 + {{ concurrency_config.subnet_phase2 }} + ~400 SYN/sweep +
+
+ HTTP Discovery + {{ concurrency_config.http_discovery }} + ~3 TCP/probe +
+
+ Launch Stagger + {{ concurrency_config.launch_stagger_ms }}ms +
+
+ {% endif %} +
+
+ + +
+
+

Blacklist Management

+
+
+
+
+

Add to Blacklist

+

Exclude IP addresses or ranges from all scanning operations

+
+
+ + + Individual IP or CIDR notation range to exclude from + scanning +
+ +
+
+ +
+

Current Blacklist

+
+
Loading blacklist entries...
+
+
+
+
+ + + +
+ {% endblock %} \ No newline at end of file diff --git a/templates/setup.html b/templates/setup.html new file mode 100644 index 0000000..67b1738 --- /dev/null +++ b/templates/setup.html @@ -0,0 +1,187 @@ + + + + + + + ROFMAD — Setup + + + + + + + + + + diff --git a/templates/styles.html b/templates/styles.html new file mode 100755 index 0000000..afb4aa9 --- /dev/null +++ b/templates/styles.html @@ -0,0 +1,1036 @@ +/* ROFMAD — Gruvbox Dark Hard / Professional TUI Theme */ + +@import url('https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@300;400;500;600;700&display=swap'); + +:root { + /* Gruvbox Dark Hard — backgrounds */ + --gb-bg-h: #1d2021; + --gb-bg: #282828; + --gb-bg-s: #32302f; + --gb-bg1: #3c3836; + --gb-bg2: #504945; + --gb-bg3: #665c54; + --gb-bg4: #7c6f64; + --gb-gray: #928374; + + /* Gruvbox Dark Hard — foreground */ + --gb-fg: #ebdbb2; + --gb-fg2: #d5c4a1; + --gb-fg3: #bdae93; + --gb-fg4: #a89984; + + /* Gruvbox accents */ + --gb-red: #cc241d; + --gb-red-b: #fb4934; + --gb-yellow: #d79921; + --gb-yellow-b: #fabd2f; + --gb-blue: #458588; + --gb-blue-b: #83a598; + --gb-purple: #b16286; + --gb-purple-b: #d3869b; + --gb-orange: #d65d0e; + --gb-orange-b: #fe8019; + --gb-aqua: #689d6a; + --gb-aqua-b: #8ec07c; + + /* Semantic aliases */ + --primary: var(--gb-blue); + --primary-b: var(--gb-blue-b); + --secondary: var(--gb-purple); + --secondary-b: var(--gb-purple-b); + + --bg-primary: var(--gb-bg-h); + --bg-secondary: var(--gb-bg); + --bg-tertiary: var(--gb-bg1); + --bg-card: var(--gb-bg-s); + --bg-hover: var(--gb-bg2); + + --text-primary: var(--gb-fg); + --text-secondary: var(--gb-fg2); + --text-muted: var(--gb-fg4); + --text-dim: var(--gb-gray); + + --border-color: var(--gb-bg1); + --border-dim: var(--gb-bg2); + + --shadow-sm: 0 1px 3px rgba(0,0,0,0.5); + --shadow-md: 0 3px 8px rgba(0,0,0,0.6); + + --success: var(--gb-blue-b); + --warning: var(--gb-orange-b); + --danger: var(--gb-red-b); + --info: var(--gb-blue-b); + + /* Legacy aliases — referenced in page-specific templates */ + --primary-purple: var(--gb-blue); + --dark-purple: #076678; + --light-purple: var(--gb-blue-b); + --accent-yellow: var(--gb-yellow); + --bright-yellow: var(--gb-yellow-b); + --dark-yellow: #b57614; + --success-green: var(--gb-blue-b); + --warning-orange: var(--gb-orange-b); + --danger-red: var(--gb-red-b); + --info-blue: var(--gb-blue-b); + --border-light: var(--gb-bg2); + + --border-radius: 0; + --border-radius-lg: 0; + --max-width: 1440px; +} + +/* ── Reset & Base ──────────────────────────────────────────── */ +* { margin: 0; padding: 0; box-sizing: border-box; } +html { font-size: 14px; scroll-behavior: smooth; } + +body { + font-family: 'JetBrains Mono', 'Fira Code', 'Cascadia Code', 'Consolas', monospace; + background-color: var(--bg-primary); + color: var(--text-primary); + line-height: 1.55; + font-weight: 400; + overflow-x: hidden; +} + +/* ── Typography ────────────────────────────────────────────── */ +h1, h2, h3, h4, h5, h6 { + font-family: 'JetBrains Mono', monospace; + font-weight: 700; + line-height: 1.25; + margin-bottom: 0.4rem; + letter-spacing: -0.01em; +} + +h1 { font-size: 1.8rem; color: var(--gb-blue-b); } +h2 { font-size: 1.4rem; color: var(--text-primary); } +h3 { font-size: 1.15rem; color: var(--text-primary); } +h4 { font-size: 1rem; color: var(--text-secondary); } +h5 { font-size: 0.9rem; color: var(--text-secondary); } +h6 { font-size: 0.82rem; color: var(--text-muted); } + +p { margin-bottom: 0.8rem; color: var(--text-secondary); } + +a { + color: var(--gb-blue-b); + text-decoration: none; + transition: color 0.1s; +} +a:hover { color: var(--gb-yellow-b); text-decoration: none; } + +/* ── Navigation ────────────────────────────────────────────── */ +.navbar { + background-color: var(--gb-bg); + padding: 0 1.5rem; + height: 44px; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid var(--gb-bg1); + position: sticky; + top: 0; + z-index: 100; +} + +.nav-brand h1 { + color: var(--gb-blue-b); + font-size: 1rem; + font-weight: 700; + letter-spacing: 0.12em; + text-transform: uppercase; + -webkit-text-fill-color: unset; + background: none; +} + +.nav-links { + display: flex; + gap: 0; + height: 100%; + align-items: stretch; +} + +.nav-links a { + color: var(--text-muted); + padding: 0 1rem; + font-size: 0.78rem; + font-weight: 500; + letter-spacing: 0.06em; + text-transform: uppercase; + display: flex; + align-items: center; + border-bottom: 2px solid transparent; + transition: color 0.1s, border-color 0.1s; +} + +.nav-links a:hover { + color: var(--text-primary); + border-bottom-color: var(--gb-blue); + text-decoration: none; +} + +/* ── Layout ────────────────────────────────────────────────── */ +.container { + max-width: var(--max-width); + margin: 0 auto; + padding: 1.5rem; +} + +/* ── Cards ─────────────────────────────────────────────────── */ +.card { + background-color: var(--bg-card); + border: 1px solid var(--border-color); + padding: 1.25rem; + transition: border-color 0.1s; +} +.card:hover { border-color: var(--gb-blue); } + +.section-card { + background-color: var(--bg-card); + border: 1px solid var(--border-color); + margin-bottom: 2rem; + overflow: hidden; +} + +.card-header { + background-color: var(--gb-bg1); + padding: 1rem 1.25rem; + border-bottom: 1px solid var(--border-color); +} +.card-header h2 { + color: var(--text-secondary); + margin: 0; + font-size: 0.82rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +/* ── Section Headers ───────────────────────────────────────── */ +.section-header { + background-color: var(--gb-bg); + padding: 1.25rem 1.5rem; + border-bottom: 1px solid var(--gb-blue); + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 2rem; +} + +.section-header::before { display: none; } + +.section-header h1, +.section-header h2 { + font-size: 1.2rem; + font-weight: 700; + color: var(--gb-blue-b); + margin-bottom: 0; + display: flex; + align-items: center; + gap: 0.6rem; + -webkit-text-fill-color: unset; + background: none; + background-clip: unset; + -webkit-background-clip: unset; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.section-title { + font-size: 0.82rem; + font-weight: 700; + color: var(--text-muted); + margin-bottom: 0.4rem; + display: flex; + align-items: center; + gap: 0.5rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.section-subtitle { color: var(--text-dim); font-size: 0.8rem; } +.section-content { padding: 1.75rem; } + +/* ── Stats Grid ────────────────────────────────────────────── */ +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 0.75rem; + margin-bottom: 2rem; +} + +.stat-card { + background-color: var(--gb-bg); + padding: 1.5rem; + text-align: left; + border: 1px solid var(--gb-bg1); + border-left: 3px solid var(--gb-blue); + transition: border-left-color 0.1s, background-color 0.1s; +} + +.stat-card:hover { + background-color: var(--gb-bg-s); + border-left-color: var(--gb-yellow); +} + +.stat-card::before { display: none; } +.stat-card::after { display: none; } + +.stat-card h3 { + color: var(--text-dim); + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.1em; + margin-bottom: 0.5rem; + font-weight: 500; +} + +.stat-number { + font-size: 2rem; + font-weight: 700; + color: var(--gb-blue-b); + display: block; + margin-bottom: 0; + font-family: 'JetBrains Mono', monospace; + filter: none; + background: none; + -webkit-text-fill-color: unset; + -webkit-background-clip: unset; + background-clip: unset; +} + +/* ── Progress ──────────────────────────────────────────────── */ +.progress-section { + background-color: var(--bg-card); + border: 1px solid var(--border-color); + padding: 1.25rem; + margin-bottom: 1.5rem; +} + +.progress-grid { display: grid; gap: 0.75rem; } + +.progress-item { + display: grid; + grid-template-columns: 160px 1fr 60px; + align-items: center; + gap: 1rem; +} + +.progress-item label { + color: var(--text-secondary); + font-weight: 500; + font-size: 0.82rem; +} + +.progress-bar { + background-color: var(--gb-bg1); + height: 12px; + border: 1px solid var(--border-color); + overflow: hidden; +} + +.progress-fill { + background-color: var(--gb-blue); + height: 100%; + transition: width 0.3s ease; +} + +.progress-fill::after { display: none; } + +.progress-item span { + color: var(--gb-blue-b); + font-weight: 600; + text-align: right; + font-size: 0.82rem; +} + +/* ── Tables ────────────────────────────────────────────────── */ +.table-container { + background-color: var(--bg-card); + border: 1px solid var(--border-color); + overflow: hidden; + margin-bottom: 1.5rem; +} + +.hosts-table, +.tasks-table { + width: 100%; + border-collapse: collapse; + font-size: 0.82rem; +} + +.hosts-table th, +.hosts-table td, +.tasks-table th, +.tasks-table td { + padding: 0.55rem 0.875rem; + text-align: left; + border-bottom: 1px solid var(--gb-bg1); +} + +.hosts-table th, +.tasks-table th { + background-color: var(--gb-bg1); + color: var(--text-muted); + font-weight: 600; + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.08em; + position: sticky; + top: 0; + z-index: 10; + border-bottom: 2px solid var(--gb-blue); +} + +.hosts-table tr:hover, +.tasks-table tr:hover { + background-color: var(--gb-bg1); +} + +.hosts-table tbody tr:last-child td, +.tasks-table tbody tr:last-child td { border-bottom: none; } + +/* ── Host Links ────────────────────────────────────────────── */ +.host-link { color: var(--gb-blue-b); font-weight: 600; font-size: 0.85rem; } +.host-link:hover { color: var(--gb-yellow-b); } + +/* ── Tags ──────────────────────────────────────────────────── */ +.tags { display: flex; flex-wrap: wrap; gap: 0.3rem; } + +.tag { + background-color: var(--gb-bg1); + color: var(--text-secondary); + padding: 0.15rem 0.5rem; + font-size: 0.72rem; + font-weight: 500; + border: 1px solid var(--gb-bg2); + transition: background-color 0.1s, color 0.1s; + cursor: pointer; + font-family: 'JetBrains Mono', monospace; + border-radius: 0; +} + +.tag:hover { + background-color: var(--gb-blue); + color: var(--gb-fg); + border-color: var(--gb-blue-b); +} + +.tech-tag, +.service-tag { + background-color: var(--gb-blue); + color: var(--gb-fg); + padding: 0.15rem 0.5rem; + font-size: 0.72rem; + font-weight: 600; + border-radius: 0; +} + +.tag-action { + background-color: var(--gb-orange) !important; + color: var(--gb-bg-h) !important; + border-color: var(--gb-orange-b) !important; + font-weight: 700 !important; +} + +.tag-manual { + background-color: var(--gb-yellow) !important; + color: var(--gb-bg-h) !important; + border-color: var(--gb-yellow-b) !important; + font-weight: 700 !important; +} + +.tag-refined { + background-color: var(--gb-purple) !important; + color: var(--gb-fg) !important; + border-color: var(--gb-purple-b) !important; +} + +.tag-automated { + background-color: var(--gb-bg1) !important; + color: var(--text-muted) !important; + border-color: var(--gb-bg2) !important; + font-size: 0.68rem !important; +} + +.tag-combined { + background-color: var(--gb-blue) !important; + color: var(--gb-fg) !important; + border-color: var(--gb-blue-b) !important; + font-weight: 500 !important; +} + +.tag.action-tag { + background-color: var(--gb-yellow); + color: var(--gb-bg-h); + border-color: var(--gb-yellow-b); + font-weight: 700; +} +.tag.action-tag:hover { + background-color: var(--gb-yellow-b); + color: var(--gb-bg-h); +} + +/* ── Status Indicators ─────────────────────────────────────── */ +.status-indicators { display: flex; gap: 0.25rem; } + +.status-indicator { + width: 22px; + height: 22px; + display: flex; + align-items: center; + justify-content: center; + font-size: 0.65rem; + font-weight: 700; + border: 1px solid; + border-radius: 0; + transition: opacity 0.1s; +} + +.status-indicator.complete { + background-color: var(--gb-blue); + border-color: var(--gb-blue-b); + color: var(--gb-fg); +} + +.status-indicator.incomplete { + background-color: var(--gb-bg1); + border-color: var(--gb-bg2); + color: var(--text-muted); +} + +.status-indicator:hover { opacity: 0.75; } + +/* ── Buttons ───────────────────────────────────────────────── */ +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.4rem; + padding: 0.45rem 1rem; + border: 1px solid transparent; + border-radius: 0; + font-size: 0.78rem; + font-weight: 600; + text-decoration: none; + cursor: pointer; + transition: background-color 0.1s, color 0.1s, border-color 0.1s; + min-width: 70px; + font-family: 'JetBrains Mono', monospace; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +.btn-primary { + background-color: var(--gb-blue); + color: var(--gb-fg); + border-color: var(--gb-blue-b); +} +.btn-primary:hover { + background-color: var(--gb-blue-b); + color: var(--gb-bg-h); + text-decoration: none; +} + +.btn-secondary { + background-color: var(--gb-bg1); + color: var(--text-secondary); + border-color: var(--gb-bg2); +} +.btn-secondary:hover { + background-color: var(--gb-bg2); + color: var(--text-primary); + border-color: var(--gb-blue); + text-decoration: none; +} + +.btn-detail { + background-color: var(--gb-purple); + color: var(--gb-fg); + border-color: var(--gb-purple-b); + font-weight: 700; +} +.btn-detail:hover { + background-color: var(--gb-purple-b); + color: var(--gb-bg-h); + text-decoration: none; +} + +.btn-warning { + background-color: var(--gb-orange); + color: var(--gb-fg); + border-color: var(--gb-orange-b); +} +.btn-warning:hover { + background-color: var(--gb-orange-b); + color: var(--gb-bg-h); + text-decoration: none; +} + +.btn-danger { + background-color: var(--gb-red); + color: var(--gb-fg); + border-color: var(--gb-red-b); +} +.btn-danger:hover { + background-color: var(--gb-red-b); + color: var(--gb-fg); + text-decoration: none; +} + +.btn-small { padding: 0.3rem 0.6rem; font-size: 0.7rem; min-width: auto; } + +/* ── Forms ─────────────────────────────────────────────────── */ +.form-group { margin-bottom: 1rem; } + +.form-group label { + display: block; + margin-bottom: 0.3rem; + color: var(--text-muted); + font-weight: 500; + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +input[type="text"], +input[type="password"], +input[type="email"], +input[type="number"], +select, +textarea { + width: 100%; + padding: 0.5rem 0.75rem; + background-color: var(--gb-bg-h); + border: 1px solid var(--gb-bg2); + border-radius: 0; + color: var(--text-primary); + font-size: 0.85rem; + font-family: 'JetBrains Mono', monospace; + transition: border-color 0.1s; +} + +input:focus, +select:focus, +textarea:focus { + outline: none; + border-color: var(--gb-blue); + box-shadow: 0 0 0 1px var(--gb-blue); + background-color: var(--gb-bg); +} + +select option { background-color: var(--gb-bg); } + +.form-actions { + display: flex; + gap: 0.75rem; + justify-content: flex-end; + margin-top: 1.25rem; +} + +.form-help { color: var(--text-dim); font-size: 0.72rem; margin-top: 0.2rem; display: block; } +.required { color: var(--gb-red-b); } + +/* ── Tabs ──────────────────────────────────────────────────── */ +.tab-nav { + display: flex; + background-color: var(--gb-bg-h); + border: 1px solid var(--border-color); + border-bottom: 2px solid var(--gb-blue); + overflow-x: auto; +} + +.tab-btn { + padding: 0.6rem 1.25rem; + background: none; + border: none; + color: var(--text-muted); + cursor: pointer; + font-size: 0.72rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.08em; + transition: background-color 0.1s, color 0.1s; + border-right: 1px solid var(--border-color); + white-space: nowrap; + font-family: 'JetBrains Mono', monospace; +} + +.tab-btn:last-child { border-right: none; } +.tab-btn:hover { background-color: var(--gb-bg1); color: var(--text-secondary); } + +.tab-btn.active { + background-color: var(--gb-blue); + color: var(--gb-fg); +} + +.tab-content { + display: none; + background-color: var(--bg-card); + border: 1px solid var(--border-color); + border-top: none; + padding: 1.25rem; +} +.tab-content.active { display: block; } + +/* ── Code ──────────────────────────────────────────────────── */ +pre, code { + background-color: var(--gb-bg-h); + border: 1px solid var(--border-color); + font-family: 'JetBrains Mono', 'Consolas', monospace; + font-size: 0.82rem; + line-height: 1.5; + color: var(--gb-fg3); + border-radius: 0; +} + +pre { padding: 0.875rem; overflow-x: auto; margin: 0.75rem 0; } +code { padding: 0.1rem 0.35rem; display: inline; } + +/* ── Modals ────────────────────────────────────────────────── */ +.modal-header { + background-color: var(--gb-bg1); + padding: 0.75rem 1.25rem; + border-bottom: 1px solid var(--border-color); + display: flex; + justify-content: space-between; + align-items: center; +} +.modal-body { padding: 1.25rem; } + +/* Discovery Modal */ +.discovery-modal-overlay { + position: fixed; + top: 0; left: 0; + width: 100%; height: 100%; + background: rgba(0,0,0,0.85); + display: flex; + align-items: center; + justify-content: center; + z-index: 10000; +} + +.discovery-modal { + background: var(--gb-bg); + border: 1px solid var(--gb-blue); + border-top: 2px solid var(--gb-blue-b); + width: 90%; + max-width: 1000px; + max-height: 80%; + display: flex; + flex-direction: column; + box-shadow: var(--shadow-md); +} + +.discovery-modal-header { + background: var(--gb-bg1); + padding: 0.75rem 1.25rem; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid var(--border-color); +} + +.discovery-modal-title { + color: var(--gb-blue-b); + font-weight: 700; + margin: 0; + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.discovery-modal-close { + background: var(--gb-red); + color: var(--gb-fg); + border: 1px solid var(--gb-red-b); + padding: 0.3rem 0.7rem; + cursor: pointer; + font-weight: 700; + font-size: 0.72rem; + font-family: 'JetBrains Mono', monospace; + text-transform: uppercase; + letter-spacing: 0.05em; + border-radius: 0; +} +.discovery-modal-close:hover { background: var(--gb-red-b); } + +.discovery-modal-body { flex: 1; padding: 1.25rem; overflow-y: auto; } +.discovery-section { margin-bottom: 1.25rem; } + +.discovery-section h3 { + color: var(--gb-blue-b); + margin-bottom: 0.6rem; + padding-bottom: 0.35rem; + border-bottom: 1px solid var(--border-color); + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.discovery-table { + width: 100%; + border-collapse: collapse; + background: var(--gb-bg-h); + border: 1px solid var(--border-color); + font-size: 0.82rem; +} + +.discovery-table th { + background: var(--gb-bg1); + color: var(--text-muted); + padding: 0.5rem 0.875rem; + text-align: left; + font-weight: 600; + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.06em; + border-bottom: 2px solid var(--gb-blue); +} + +.discovery-table td { + padding: 0.5rem 0.875rem; + border-bottom: 1px solid var(--border-color); + color: var(--text-secondary); +} +.discovery-table tr:hover { background: var(--gb-bg1); } + +.discovery-actions { display: flex; gap: 0.4rem; flex-wrap: wrap; } + +.discovery-btn { + padding: 0.3rem 0.65rem; + border: 1px solid transparent; + font-size: 0.72rem; + font-weight: 600; + cursor: pointer; + transition: background-color 0.1s; + font-family: 'JetBrains Mono', monospace; + text-transform: uppercase; + letter-spacing: 0.04em; + border-radius: 0; +} + +.discovery-btn-primary { background: var(--gb-blue); color: var(--gb-fg); border-color: var(--gb-blue-b); } +.discovery-btn-secondary{ background: var(--gb-bg1); color: var(--text-primary); border-color: var(--gb-bg2); } +.discovery-btn-danger { background: var(--gb-red); color: var(--gb-fg); border-color: var(--gb-red-b); } +.discovery-btn-warning { background: var(--gb-orange); color: var(--gb-fg); border-color: var(--gb-orange-b); } +.discovery-btn:hover { opacity: 0.82; } + +.discovery-empty { text-align: center; padding: 2rem; color: var(--text-dim); font-size: 0.82rem; } + +/* Terminal Modal */ +.terminal-modal-overlay { + position: fixed; + top: 0; left: 0; + width: 100%; height: 100%; + background: rgba(0,0,0,0.88); + display: flex; + align-items: center; + justify-content: center; + z-index: 10000; +} + +.terminal-modal { + background: var(--gb-bg-h); + border: 1px solid var(--gb-blue); + border-top: 2px solid var(--gb-blue-b); + width: 80%; + max-width: 900px; + height: 70%; + display: flex; + flex-direction: column; + box-shadow: var(--shadow-md); +} + +.terminal-btn { + background: var(--gb-bg1); + color: var(--text-secondary); + border: 1px solid var(--gb-bg2); + padding: 0.3rem 0.65rem; + cursor: pointer; + font-size: 0.72rem; + font-family: 'JetBrains Mono', monospace; + text-transform: uppercase; + letter-spacing: 0.04em; + transition: background-color 0.1s; + border-radius: 0; +} +.terminal-btn:hover { background: var(--gb-blue); color: var(--gb-fg); border-color: var(--gb-blue-b); } +.terminal-close { background: var(--gb-red); color: var(--gb-fg); border-color: var(--gb-red-b); } +.terminal-close:hover { background: var(--gb-red-b); } + +.terminal-output { + flex: 1; + background: var(--gb-bg-h); + color: var(--gb-fg3); + font-family: 'JetBrains Mono', 'Consolas', monospace; + font-size: 0.82rem; + padding: 0.875rem; + overflow-y: auto; + white-space: pre-wrap; + line-height: 1.5; +} + +.terminal-modal-status { + background: var(--gb-bg1); + color: var(--text-dim); + padding: 0.35rem 0.875rem; + font-size: 0.7rem; + border-top: 1px solid var(--border-color); + font-family: 'JetBrains Mono', monospace; +} + +/* ── Host Header ───────────────────────────────────────────── */ +.host-header { + background-color: var(--gb-bg); + border: 1px solid var(--border-color); + border-left: 3px solid var(--gb-blue); + padding: 1.25rem; + margin-bottom: 1.5rem; +} + +.host-header h1 { font-size: 1.8rem; margin-bottom: 0.2rem; } +.host-header h2 { color: var(--text-muted); font-weight: 400; margin-bottom: 0.6rem; font-size: 0.9rem; } + +/* Hostname display */ +.no-hostname { color: var(--text-dim); font-style: italic; font-size: 0.82rem; } +.ad-domain { color: var(--gb-yellow); font-style: normal; font-size: 0.78rem; font-weight: 600; margin-left: 0.3rem; } +.no-hostname + .ad-domain { color: var(--gb-blue-b); font-weight: 700; } + +/* ── Search Preview ────────────────────────────────────────── */ +.search-preview { + margin-top: 1.25rem; + background-color: var(--bg-card); + border: 1px solid var(--border-color); + padding: 1rem; +} + +.search-preview h4 { + color: var(--text-muted); + margin-bottom: 0.6rem; + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.tag-preview-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(130px, 1fr)); + gap: 0.3rem; +} + +.tag-preview-item { + background: var(--gb-bg1); + border: 1px solid var(--border-color); + padding: 0.35rem; + text-align: center; + font-size: 0.72rem; + color: var(--text-secondary); + transition: background-color 0.1s, color 0.1s; + cursor: pointer; + font-family: 'JetBrains Mono', monospace; +} +.tag-preview-item:hover { + background: var(--gb-blue); + color: var(--gb-fg); + border-color: var(--gb-blue-b); +} + +/* Tag name/count */ +.tag-name { font-weight: 600; font-size: 0.82rem; } +.tag-count { + background: var(--gb-blue); + color: var(--gb-fg); + padding: 0.1rem 0.4rem; + font-size: 0.7rem; + font-weight: 700; +} + +/* ── Empty States ──────────────────────────────────────────── */ +.empty-placeholder { text-align: center; padding: 2rem; } +.empty-icon { font-size: 2rem; margin-bottom: 0.75rem; opacity: 0.4; } +.empty-placeholder h3 { color: var(--text-secondary); margin-bottom: 0.3rem; font-size: 1rem; } +.empty-placeholder p { color: var(--text-muted); margin-bottom: 0.875rem; font-size: 0.82rem; } + +.empty-stats { display: flex; gap: 1.5rem; justify-content: center; } +.empty-stat { color: var(--gb-blue-b); font-size: 0.82rem; font-weight: 600; } + +/* ── Notifications ─────────────────────────────────────────── */ +.notification { + position: fixed; + bottom: 1.5rem; + right: 1.5rem; + padding: 0.6rem 1.25rem; + background-color: var(--gb-bg1); + border: 1px solid var(--gb-blue); + border-left: 3px solid var(--gb-blue-b); + color: var(--text-primary); + font-size: 0.82rem; + z-index: 9999; + font-family: 'JetBrains Mono', monospace; + box-shadow: var(--shadow-md); + border-radius: 0; +} +.notification.success { border-left-color: var(--gb-blue-b); } +.notification.error { border-left-color: var(--gb-red-b); border-color: var(--gb-red); } +.notification.warning { border-left-color: var(--gb-orange-b); border-color: var(--gb-orange); } + +/* ── Utilities ─────────────────────────────────────────────── */ +.text-center { text-align: center; } +.text-right { text-align: right; } +.text-left { text-align: left; } + +.text-muted { color: var(--text-muted); } +.text-success { color: var(--gb-blue-b); } +.text-warning { color: var(--gb-orange-b); } +.text-danger { color: var(--gb-red-b); } + +.no-data { color: var(--text-dim); font-style: italic; font-size: 0.82rem; } + +.small { font-size: 0.82rem; } +.smaller { font-size: 0.72rem; } + +.mb-0 { margin-bottom: 0; } +.mb-1 { margin-bottom: 0.4rem; } +.mb-2 { margin-bottom: 0.8rem; } +.mb-3 { margin-bottom: 1.25rem; } +.mb-4 { margin-bottom: 1.75rem; } + +.mt-0 { margin-top: 0; } +.mt-1 { margin-top: 0.4rem; } +.mt-2 { margin-top: 0.8rem; } +.mt-3 { margin-top: 1.25rem; } +.mt-4 { margin-top: 1.75rem; } + +/* ── Responsive ────────────────────────────────────────────── */ +@media (max-width: 1024px) { + .container { padding: 1rem; } + .stats-grid { grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); } + .progress-item { grid-template-columns: 1fr; gap: 0.4rem; text-align: center; } + .hosts-table { font-size: 0.75rem; } + .hosts-table th, .hosts-table td { padding: 0.45rem 0.5rem; } +} + +@media (max-width: 768px) { + .navbar { padding: 0 0.875rem; } + .nav-links { flex-wrap: wrap; } + .nav-links a { padding: 0 0.6rem; font-size: 0.7rem; } + .container { padding: 0.75rem; } + .section-header { flex-direction: column; gap: 0.75rem; text-align: left; } + .75rem; } + .hosts-table th, .hosts-table td { padding: 0.4rem 0.3rem; font-size: 0.7rem; } + .tab-btn { font-size: 0.68rem; padding: 0.5rem 0.875rem; } + .status-indicators { justify-content: center; } + .status-indicator { width: 18px; height: 18px; font-size: 0.6rem; } + .discovery-modal { width: 95%; height: 88%; } + .discovery-actions { flex-direction: column; } + .discovery-btn { width: 100%; } + .terminal-modal { width: 95%; height: 82%; } +} + +@media (max-width: 480px) { + .stat-card { padding: 0.875rem; } + .stat-number { font-size: 1.6rem; } + .progress-section, .host-header { padding: 0.875rem; } + .hosts-table { display: block; overflow-x: auto; white-space: nowrap; } + .stats-grid { grid-template-columns: 1fr; } + .navbar { height: auto; padding: 0.5rem 0.75rem; flex-direction: column; gap: 0.4rem; } + .nav-links { width: 100%; justify-content: center; } +} diff --git a/templates/task_detail.html b/templates/task_detail.html new file mode 100755 index 0000000..801706f --- /dev/null +++ b/templates/task_detail.html @@ -0,0 +1,554 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} +{% block title %}Task Queue{% endblock %} + +{% block content %} +
+ +
+
+

{{ task.name }}

+
+ {{ task.target }} + Calculating... + Running +
+
+
+ + + +
+
+ + +
+
+
{{ task.name }} @ {{ task.target }}
+
+ Connected + Lines: 0 +
+
+ +
+ +
+
+ + +
+
+

Task Information

+
+
+ + {{ task.id }} +
+
+ + {{ task.name }} +
+
+ + {{ task.target }} +
+
+ + {{ task.start_epoch }} +
+
+
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/templates/task_output.html b/templates/task_output.html new file mode 100644 index 0000000..dde334d --- /dev/null +++ b/templates/task_output.html @@ -0,0 +1,100 @@ +{% extends "base.html" %} + +{% block extra_head %} + +{% endblock %} + +{% block title %}{{ task_name }} — {{ target }}{% endblock %} + +{% block content %} +
+
+

{{ task_name }}

+ {{ target }} + {{ status }} + +
+ {% if output.len() > 0 %} + 
{{ output }}
+ {% else %} + 
No output captured for this task.
+ {% endif %} +
+ + +{% endblock %} diff --git a/workflow-example.yaml b/workflow-example.yaml new file mode 100755 index 0000000..07d043d --- /dev/null +++ b/workflow-example.yaml @@ -0,0 +1,228 @@ +specific_actions: + # ============================================================================ + # EXAMPLE A — Port-based condition (well-known, stable ports) + # Use contains: when you know the port never changes in practice. + # Single spawn per host — no fan-out. + # ============================================================================ + - name: "smb_example" + condition: + contains: "445 AND 139" # must have BOTH ports open + # contains: "80 OR 443" # must have ANY of these ports open + # equals: "22" # must have EXACTLY this one port (rare) + has_tag: "smb AND Windows" # tag AND logic + # has_tag: "web OR database" # tag OR logic + has_not_tag: "anonymous-smb" # skip if already tagged (avoids re-running) + command: "nxc smb {{IP}} -u '' -p '' --shares" + timeout_s: 120 + tags: + - pattern: "STATUS_SUCCESS" + tag: "anonymous-smb" + - pattern: "SMB.*signing.*False" + tag: "smb-signing-disabled" + - pattern: "(?i)domain.*corp" # (?i) = case-insensitive match + tag: "domain-corp-member" + - pattern: "VULNERABLE.*(MS17-010|EternalBlue)" + tag: "vulnerable-ms17-010" + + + # ============================================================================ + # EXAMPLE B — Service-based condition: fires once per matching service instance + # has_service: "" implies per-instance fan-out automatically. + # One task per port nmap identified as that service — handles non-standard ports + # and multiple simultaneous instances (e.g. MSSQL named instances). + # {{SERVICE_PORT}} resolves to the current instance's port. + # ============================================================================ + - name: "mssql_example" + condition: + has_service: "mssql" # fires once per port nmap identified as mssql + # 1 instance → 1 task; 3 instances → 3 tasks + # {{SERVICE_PORT}} → the actual port for this instance + # works whether mssql is on 1433, 1434, or 49152 + command: "nxc mssql {{IP}} -p {{SERVICE_PORT}} -u '' -p ''" + timeout_s: 60 + tags: + - pattern: "\\[\\+\\]" + tag: "mssql-anonymous-access" + + - name: "rdp_example" + condition: + has_service: "rdp" # catches 3389 AND any non-standard RDP port + command: "nxc rdp {{IP}} -p {{SERVICE_PORT}}" + timeout_s: 30 + tags: + - pattern: "NLA.*False" + tag: "rdp-nla-disabled" + + - name: "ftp_example" + condition: + has_service: "ftp" + command: "nxc ftp {{IP}} -p {{SERVICE_PORT}} -u 'anonymous' -p 'anonymous'" + timeout_s: 30 + tags: + - pattern: "\\[\\+\\]" + tag: "ftp-anonymous-access" + + + # ============================================================================ + # EXAMPLE C — HTTP service fan-out + # has_http_service: true fires once per discovered HTTP service on the host. + # {{HTTP_URL}} resolves to the correct http:// or https:// URL per iteration. + # Outputs stored as "nuclei_scan__80", "nuclei_scan__443", etc. + # ============================================================================ + - name: "nuclei_scan" + condition: + has_http_service: true # fires per HTTP service; optional tags filter further + has_tag: "interesting" # host-level tag condition still applies + command: "nuclei -u {{HTTP_URL}} -silent -severity medium,high,critical" + timeout_s: 300 + tags: + - pattern: "\\[critical\\]" + tag: "nuclei-critical" + - pattern: "\\[high\\]" + tag: "nuclei-high" + + - name: "ffuf_fuzz" + condition: + has_http_service: true + command: "ffuf -u {{HTTP_URL}}/FUZZ -w /opt/wordlists/common.txt -mc 200,301,302 -s" + timeout_s: 600 + + - name: "curl_banner" + condition: + has_http_service: true + # Mix {{HTTP_PROTOCOL}} and {{HTTP_PORT}} freely + command: "curl -sk -I {{HTTP_PROTOCOL}}://{{IP}}:{{HTTP_PORT}}" + timeout_s: 15 + tags: + - pattern: "(?i)X-Powered-By:" + tag: "http-powered-by-header" + + +# ============================================================================ +# PLACEHOLDERS +# ============================================================================ +# {{IP}} / {{ip}} / Target IP address +# {{PORT}} First port in the raw open port list +# (order-dependent — prefer {{PORT:service}}) +# {{PORT:service}} First port where nmap identified 'service' +# Useful in single-spawn actions to cross-reference +# another service's port +# {{PORTS:service}} All ports running 'service', comma-separated +# e.g. "139,445" when smb runs on both +# {{URL}} / {{BASE_URL}} http(s)://IP:PORT (first HTTP service) +# {{PROTOCOL}} http or https (first HTTP service) +# +# Per-instance placeholders — available in has_service and has_http_service actions: +# {{SERVICE_PORT}} Port for the current instance +# {{SERVICE_PROTOCOL}} Protocol for the current instance (HTTP only) +# {{HTTP_URL}} http(s)://IP:PORT for the current HTTP instance +# {{HTTP_PORT}} Port for the current HTTP instance +# {{HTTP_PROTOCOL}} http or https for the current HTTP instance + + +# ============================================================================ +# CONDITIONS — field reference +# ============================================================================ +# contains: "445" Port 445 is in the open port list +# contains: "80 AND 443" Port 80 AND 443 both open +# contains: "80 OR 443" Port 80 OR 443 open +# equals: "22" ONLY port 22 open +# has_tag: "windows" Tag present in automated/manual/refined tags +# has_tag: "smb AND windows" Both tags present +# has_tag: "smb OR netbios" Either tag present +# has_not_tag: "anonymous-smb" Skip host if tag IS present (inverse gate) +# has_not_tag: "exploited AND owned" Skip if BOTH tags present +# has_not_tag: "low-value OR noise" Skip if EITHER tag present +# has_service: "mssql" nmap -sV identified mssql — fires per instance +# Single name only: AND/OR stays a boolean gate +# has_service: "rdp OR winrm" Either service present (single spawn, no fan-out) +# has_http_service: true/false HTTP service confirmed — fires per HTTP service +# reconnaissance_complete: true Always enforced — no need to specify +# +# All specified fields must pass simultaneously (AND between fields). +# OR/AND logic applies within a single field value only. +# +# Fan-out behaviour: +# has_service: "mssql" → 1 task per port running mssql (0 tasks if absent) +# has_http_service: true → 1 task per HTTP service (0 tasks if absent) +# all other conditions → single spawn per host + + +# ============================================================================ +# SERVICE NAMES — after normalize_service_name() (use these in has_service:) +# Matching is exact, case-insensitive. "MSSQL" matches "mssql"; "sql" does NOT. +# ============================================================================ +# +# Core Windows services: +# smb (nmap: microsoft-ds, cifs) +# mssql (nmap: ms-sql-s, ms-sql-m, ms-sql-monitor, ms-sql-ds) +# rdp (nmap: ms-wbt-server) +# winrm (nmap: wsman) +# kerberos (nmap: kerberos-sec) +# netbios (nmap: netbios-ssn, netbios-ns) +# rpc (nmap: msrpc, epmap, ncacn_http, ncacn_ip_tcp, msrpc-base) +# dns (nmap: domain) +# +# Common Linux/cross-platform (pass-through, already clean): +# ssh, ftp, http, https, smtp, pop3, imap, ldap, mysql, postgresql +# vnc, telnet, snmp, redis, mongodb, nfs, ntp +# +# Pass-through but worth knowing: +# rpcbind Unix portmapper (port 111) — intentionally NOT mapped to "rpc" +# (different tooling from Windows msrpc; use has_service: "rpcbind") +# oracle-tns Oracle TNS listener — clean identifier, no normalization needed +# ajp13 Tomcat AJP connector (port 8009) — clean identifier +# +# TLS variants — normalized to base service: +# ldaps → ldap | imaps → imap | smtps → smtp | pop3s → pop3 | ftps → ftp +# ssl/http → http | ssl/ldap → ldap (ssl/ prefix stripped automatically) +# +# HTTP on alternate ports: +# http-alt → http (use has_service: "http") +# http-proxy kept as-is — Squid/HAProxy, NOT generic http +# +# Niche/application-specific services (elasticsearch, squid, etc.): +# Use the action builder — workflow.yaml covers systematic services only + + +# ============================================================================ +# HOW AUTOMATED TAGS ARE GENERATED FROM NMAP +# ============================================================================ +# nmap -sV produces two distinct fields per port — both become automated tags: +# +# 1. service name → normalize_service_name() → short canonical tag +# Raw: "ms-sql-s" Tag: "mssql" +# Raw: "microsoft-ds" Tag: "smb" +# Raw: "ms-wbt-server" Tag: "rdp" +# See SERVICE NAMES section above for the full mapping table. +# +# 2. product string → lowercased → full-string tag (refined by hardcoded rules) +# nmap: product="Apache httpd" Tag: "apache" (hardcoded rule) +# nmap: product="nginx" Tag: "nginx" (hardcoded rule) +# nmap: product="Microsoft IIS httpd" Tag: "iis" (hardcoded rule) +# nmap: product="Squid http proxy" Tag: "squid http proxy" (no rule — stays as-is) +# +# Well-known products normalized by hardcoded rules (produces short tags): +# smb, rdp, winrm, dns, kerberos, ldap, rpc, mssql, iis, apache, nginx, +# tomcat, mysql, mariadb, postgresql, ssh +# +# Niche products (no hardcoded rule) — tag is the full product string lowercased. +# Target them in the action builder: browse available tags to find the exact value, +# or use has_service: for broader protocol-level matching. +# +# Examples: +# Squid proxy: has_service: "http-proxy" (all proxies) +# has_tag: "squid http proxy" (specifically Squid, via action builder) +# HAProxy: has_service: "http-proxy" (same service name) +# has_tag: "haproxy" (if nmap product is just "HAProxy") + + +# ============================================================================ +# TAG PATTERN SYNTAX +# ============================================================================ +# - Standard regex +# - Case-sensitive by default — prefix (?i) for case-insensitive +# - Matched against full stdout + stderr of the command +# - Multiple patterns are independent — each applies its own tag +# - Tags are written immediately after the action completes; +# subsequent actions can depend on them via has_tag: diff --git a/workflows/workflow.yaml b/workflows/workflow.yaml new file mode 100755 index 0000000..3ac1c70 --- /dev/null +++ b/workflows/workflow.yaml @@ -0,0 +1,108 @@ +specific_actions: + # ============================================================================ + # SMB — port 445 is reliable in practice, keep contains condition + # ============================================================================ + + - name: "smb_signing_check" + condition: + contains: "445" + command: "nxc smb {{IP}}" + timeout_s: 60 + tags: + - pattern: "signing:False" + tag: "smb-signing-disabled" + - pattern: "SMBv1:True" + tag: "smb-v1-enabled" + + - name: "smb_anonymous_check" + condition: + contains: "445" + command: "nxc smb {{IP}}" + timeout_s: 60 + tags: + - pattern: "Null Auth:True" + tag: "anonymous-smb-access" + + - name: "smb_guest_check" + condition: + contains: "445" + has_not_tag: "anonymous-smb-access" + command: "nxc smb {{IP}} -u 'guest' -p ''" + timeout_s: 60 + tags: + - pattern: "\\[\\+\\]" + tag: "guest-smb-access" + + - name: "smb_share_enum_anonymous" + condition: + contains: "445" + has_tag: "anonymous-smb-access" + command: "nxc smb {{IP}} -u '' -p '' --shares" + timeout_s: 90 + tags: + - pattern: "READ|WRITE" + tag: "unauthenticated-shares" + + - name: "smb_share_enum_guest" + condition: + contains: "445" + has_tag: "guest-smb-access" + command: "nxc smb {{IP}} -u 'guest' -p '' --shares" + timeout_s: 90 + tags: + - pattern: "READ|WRITE" + tag: "unauthenticated-shares" + + - name: "smb_user_enum_anonymous" + condition: + contains: "445" + has_tag: "anonymous-smb-access" + command: "nxc smb {{IP}} -u '' -p '' --users" + timeout_s: 90 + tags: + - pattern: "Enumerated" + tag: "unauthenticated-user-enum" + + - name: "smb_user_enum_guest" + condition: + contains: "445" + has_tag: "guest-smb-access" + command: "nxc smb {{IP}} -u 'guest' -p '' --users" + timeout_s: 90 + tags: + - pattern: "Enumerated" + tag: "unauthenticated-user-enum" + + - name: "smb_vuln_check" + condition: + contains: "445" + has_tag: "Windows" + command: "nmap --script smb-vuln-ms17-010.nse -p445 {{IP}}" + timeout_s: 300 + tags: + - pattern: "ms17-010.*VULNERABLE" + tag: "vulnerable-ms17-010" + + # ============================================================================ + # MSSQL — port varies (1433, 1434, named instances, dynamic ports) + # has_service with a single name → fires once per port running mssql + # {{SERVICE_PORT}} resolves to each instance's port in turn + # ============================================================================ + + - name: "mssql_sa_auth" + condition: + has_service: "mssql" + command: "nxc mssql {{IP}} -p {{SERVICE_PORT}} -u 'sa' -p 'sa'" + timeout_s: 60 + tags: + - pattern: "\\[\\+\\]" + tag: "mssql-anonymous-access" + + - name: "mssql_sa_auth-local-auth" + condition: + has_service: "mssql" + command: "nxc mssql {{IP}} -p {{SERVICE_PORT}} -u 'sa' -p 'sa' --local-auth" + timeout_s: 60 + tags: + - pattern: "\\[\\+\\]" + tag: "mssql-anonymous-access" \ No newline at end of file