Merge branch 'master' into bee-add-config-file

.travis.yml

@@ -3,7 +3,7 @@ language: rust
 os:
   - linux
   - osx
-  - windows
+  # - windows
 
 arch:
   - amd64

Cargo.lock

@@ -774,6 +774,7 @@ dependencies = [
  "rlimit",
  "shell-words",
  "structopt",
+ "toml",
 ]
 
 [[package]]
@@ -782,6 +783,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
 
+[[package]]
+name = "serde"
+version = "1.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
+
 [[package]]
 name = "shell-words"
 version = "1.0.0"
@@ -874,6 +881,15 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.6.0"

Cargo.toml

@@ -21,17 +21,19 @@ colored = "2.0.0"
 structopt = "0.3.16"
 async-std = "1.6.3"
 futures = "0.3"
+rlimit = "0.4.0"
 shell-words = "1.0.0"
 log = "0.4.0"
 env_logger = "0.7.1"
 dirs = "3.0.1"
+toml = "0.5.6"
 gcd = "2.0.1"
 rand = "0.7.3"
 colorful = "0.2.1"
 ansi_term = "0.12.1"
-
-[target."cfg(not(windows))".dependencies]
-rlimit = "0.4.0"
+ureq = "1.3.0"
+serde = "1.0.115"
+serde_derive = "1.0.115"
 
 [package.metadata.deb]
 depends = "$auto, nmap"

README.md

@@ -213,23 +213,25 @@ rustscan -h
 ```
 
 ```
-rustscan 1.7.1
 Fast Port Scanner built in Rust. WARNING Do not use this program against sensitive infrastructure since the specified
 server may not be able to handle this many socket connections at once. - Discord https://discord.gg/GFrQsGy - GitHub
 https://github.com/RustScan/RustScan
 
 USAGE:
-    rustscan [FLAGS] [OPTIONS] <ips>... [-- <command>...]
+    rustscan [FLAGS] [OPTIONS] <ips-or-hosts>... [-- <command>...]
 
 FLAGS:
-    -h, --help       Prints help information
-    -q, --quiet      Quiet mode. Only output the ports. No Nmap. Useful for grep or outputting to a file
-    -V, --version    Prints version information
+    -a, --accessible
+    -h, --help          Prints help information
+    -q, --quiet         Quiet mode. Only output the ports. No Nmap. Useful for grep or outputting to a file
+    -V, --version       Prints version information
 
 OPTIONS:
     -b, --batch-size <batch-size>    The batch size for port scanning, it increases or slows the speed of scanning.
                                      Depends on the open file limit of your OS.  If you do 65535 it will do every port
                                      at the same time. Although, your OS may not support this [default: 4500]
+    -p, --ports <ports>...           A list of comma separed ports to be scanned. Example: 80,443,8080.
+    -r, --range <range>              A range of ports with format start-end. Example: 1-1000
         --scan-order <scan-order>    The order of scanning to be performed. The "serial" option will scan ports in
                                      ascending order while the "random" option will scan ports randomly [default:
                                      serial]  [possible values: Serial, Random]
@@ -237,11 +239,11 @@ OPTIONS:
     -u, --ulimit <ulimit>            Automatically ups the ULIMIT with the value you provided
 
 ARGS:
-    <ips>...        A list of comma separated IP addresses to be scanned
-    <command>...    The Nmap arguments to run. To use the argument -A, end RustScan's args with '-- -A'. Example:
-                    'rustscan -T 1500 127.0.0.1 -- -A -sC'. This command adds -Pn -vvv -p $PORTS automatically to
-                    nmap. For things like --script '(safe and vuln)' enclose it in quotations marks \"'(safe and
-                    vuln)'\"")
+    <ips-or-hosts>...    A list of comma separated IP addresses or hosts to be scanned
+    <command>...         The Nmap arguments to run. To use the argument -A, end RustScan's args with '-- -A'.
+                         Example: 'rustscan -T 1500 127.0.0.1 -- -A -sC'. This command adds -Pn -vvv -p $PORTS
+                         automatically to nmap. For things like --script '(safe and vuln)' enclose it in quotations
+                         marks \"'(safe and vuln)'\"")
 ```
 
 The format is `rustscan -b 500 -T 1500 192.168.0.1` to scan 192.168.0.1 with 500 batch size with a timeout of 1500ms. The timeout is how long RustScan waits for a response until it assumes the port is closed.
@@ -338,6 +340,22 @@ To increase accuracy, the easiest way is to increase the timeout. The default is
 
 Decreasing accuracy gives some speed bonus, but my testing found that batch size dramatically changed the speed whereas timeout did, but not so much.
 
+## 🤖 Config File
+
+RustScan uses a Config file to make things easier for you. And to also improve itself.
+The config file is used as the basis of the _adaptive learning_ mechanisms. This means that RustScan _learns_ from your scans and improves itself over time. (Don't worry, no bloated AI framework here 😉)
+
+The config file should be placed in APPDIRS. To find this folder, run RustScan and look out for RustScan saying "couldn't find config at <location>". That location is where your config file should be.
+
+Once you have this location, copy and paste this:
+
+```toml
+
+# The top ports
+[top_ports]
+80, 60, 443
+```
+
 # 🎪 Community
 
 Howdy Space Cow-Person 🤠🌌

docker/ci_test/Dockerfile

@@ -0,0 +1,28 @@
+FROM archlinux:latest
+
+RUN yes | pacman -Syu
+RUN yes | pacman -S fakeroot binutils sudo make gcc gettext awk git
+
+WORKDIR /opt
+RUN git clone https://aur.archlinux.org/yay-git.git
+
+RUN useradd -m tecmint
+RUN passwd -d tecmint
+RUN printf 'tecmint ALL=(ALL) ALL\n' | tee -a /etc/sudoers
+
+WORKDIR /opt/yay-git
+RUN chown -R tecmint /opt/yay-git
+USER tecmint
+RUN makepkg -si --noconfirm
+
+# AUR
+RUN yay -S rustscan --noconfirm
+RUN rustscan 127.0.0.1 -q -b 250 -t 1
+RUN yay -R rustscan --noconfirm
+
+# homebrew
+RUN yes | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+ENV PATH="/home/linuxbrew/.linuxbrew/bin:${PATH}"
+RUN yes | brew tap brandonskerritt/rustscan
+RUN yes | brew install rustscan
+RUN rustscan 127.0.0.1 -q -b 250 -t 1

docker/rustscan-debbuilder/entrypoint.sh

@@ -5,6 +5,7 @@ git pull --force
 
 #amd64
 cargo deb
+yes | dpkg -i *.deb
 
 #arm64
 rustup target add arm-unknown-linux-gnueabihf

src/main.rs

@@ -11,17 +11,22 @@ use port_strategy::PortStrategy;
 use colorful::Color;
 use colorful::Colorful;
 use futures::executor::block_on;
-
-#[cfg(not(target_os = "windows"))]
-use rlimit::{getrlimit, setrlimit, Resource};
-
+use rlimit::Resource;
+use rlimit::{getrlimit, setrlimit};
+use serde_derive::Deserialize;
 use std::collections::HashMap;
-#[cfg(not(target_os = "windows"))]
-use std::convert::TryInto;
+use std::net::ToSocketAddrs;
 use std::process::Command;
 use std::str::FromStr;
-use std::{net::IpAddr, net::ToSocketAddrs, time::Duration};
+use std::{
+    fs::File,
+    io::{Read, Write},
+    net::IpAddr,
+    path::PathBuf,
+    time::Duration,
+};
 use structopt::{clap::arg_enum, StructOpt};
+use ureq;
 
 extern crate colorful;
 extern crate dirs;
@@ -29,9 +34,9 @@ extern crate dirs;
 const LOWEST_PORT_NUMBER: u16 = 1;
 const TOP_PORT_NUMBER: u16 = 65535;
 // Average value for Ubuntu
-const DEFAULT_FILE_DESCRIPTORS_LIMIT: u32 = 8000;
+const DEFAULT_FILE_DESCRIPTORS_LIMIT: rlimit::rlim = 8000;
 // Safest batch size based on experimentation
-const AVERAGE_BATCH_SIZE: u32 = 3000;
+const AVERAGE_BATCH_SIZE: rlimit::rlim = 3000;
 
 #[macro_use]
 extern crate log;
@@ -101,6 +106,10 @@ struct Opts {
     #[structopt(short, long)]
     accessible: bool,
 
+    ///Config file usage
+    #[structopt(short, long)]
+    config: bool,
+
     /// The batch size for port scanning, it increases or slows the speed of
     /// scanning. Depends on the open file limit of your OS.  If you do 65535
     /// it will do every port at the same time. Although, your OS may not
@@ -114,7 +123,7 @@ struct Opts {
 
     /// Automatically ups the ULIMIT with the value you provided.
     #[structopt(short, long)]
-    ulimit: Option<u32>,
+    ulimit: Option<rlimit::rlim>,
 
     /// The order of scanning to be performed. The "serial" option will
     /// scan ports in ascending order while the "random" option will scan
@@ -131,6 +140,11 @@ struct Opts {
     command: Vec<String>,
 }
 
+#[derive(Deserialize)]
+struct ConfigFile {
+    ip: IpAddr,
+}
+
 #[cfg(not(tarpaulin_include))]
 /// Faster Nmap scanning with Rust
 /// If you're looking for the actual scanning, check out the module Scanner
@@ -153,6 +167,12 @@ fn main() {
         print_opening();
     }
 
+    /*
+    TODO when releasing the new config file feature
+    turn this on.
+    if opts.config {
+        let config = get_config_file();
+    }*/
     let ips: Vec<IpAddr> = parse_ips(&opts);
 
     if ips.is_empty() {
@@ -160,8 +180,7 @@ fn main() {
         std::process::exit(1);
     }
 
-    let ulimit: u32 = adjust_ulimit_size(&opts);
-
+    let ulimit: rlimit::rlim = adjust_ulimit_size(&opts);
     let batch_size: u16 = infer_batch_size(&opts, ulimit);
 
     let scanner = Scanner::new(
@@ -250,21 +269,65 @@ Faster Nmap scanning with Rust."#;
  --------------------------------------"#;
     println!("{}", info.gradient(Color::Yellow).bold());
     funny_opening!();
+    // println!("{}", loaded_config_file);
+}
 
+fn get_config_file() {
+    let result_get_location = get_location_config();
+    let location = match result_get_location {
+        Ok(path) => path,
+        Err(_) => panic!("Your system does not have appdirs."),
+    };
+    info!("The location of config is {:?}", location);
+
+    detail!(format!("The config file is at {:?}", location));
+
+    load_and_parse_config_file(location);
+}
+
+fn get_location_config() -> Result<std::path::PathBuf, &'static str> {
+    info!("Getting location of config file");
     let config_path = match dirs::config_dir() {
         Some(mut path) => {
             path.push("rustscan");
             path.push("config.toml");
-            path
+            Ok(path)
         }
-        None => panic!("Couldn't find config dir."),
+        None => Err("Your system does not have APPDIRS"),
     };
-
-    detail!(format!(
-        "{} {:?}",
-        "The config file is expected to be at", config_path
-    ));
+    config_path
 }
+
+fn load_and_parse_config_file(config_path: PathBuf) -> ConfigFile {
+    // Gets the file and returns the string of TOML
+    // TODO quiet mode
+
+    let mut file = File::open(&config_path);
+    let mut file: File = match file {
+        Ok(result) => result,
+        Err(_) => download_place_config_file(config_path),
+    };
+    let mut contents = String::new();
+    file.read_to_string(&mut contents).unwrap();
+    let config: ConfigFile = toml::from_str(&contents).unwrap();
+
+    config
+}
+// Downloads config file
+// Places into Appdirs
+
+fn download_place_config_file(config_location: PathBuf) -> std::fs::File {
+    info!("Placing config file");
+    let body = ureq::get("https://raw.githubusercontent.com/RustScan/RustScan/master/config.toml")
+        .call()
+        .into_string()
+        .unwrap();
+
+    let mut file = File::create(config_location).unwrap();
+    file.write(body.as_bytes()).unwrap();
+    file
+}
+
 #[cfg(not(tarpaulin_include))]
 fn build_nmap_arguments<'a>(
     addr: &'a str,
@@ -305,10 +368,9 @@ fn parse_ips(opts: &Opts) -> Vec<IpAddr> {
     ips
 }
 
-#[cfg(not(target_os = "windows"))]
-fn adjust_ulimit_size(opts: &Opts) -> u32 {
+fn adjust_ulimit_size(opts: &Opts) -> rlimit::rlim {
     if opts.ulimit.is_some() {
-        let limit: rlimit::rlim = opts.ulimit.unwrap().into();
+        let limit: rlimit::rlim = opts.ulimit.unwrap();
 
         match setrlimit(Resource::NOFILE, limit, limit) {
             Ok(_) => {
@@ -323,21 +385,14 @@ fn adjust_ulimit_size(opts: &Opts) -> u32 {
 
     let (rlim, _) = getrlimit(Resource::NOFILE).unwrap();
 
-    rlim.try_into().unwrap()
+    rlim
 }
 
-// Rlimit does not support Windows
-// set to 1000 if Windows is used
-#[cfg(target_os = "windows")]
-fn adjust_ulimit_size(_opts: &Opts) -> u32 {
-    1000
-}
-
-fn infer_batch_size(opts: &Opts, ulimit: u32) -> u16 {
-    let mut batch_size: u32 = opts.batch_size.into();
+fn infer_batch_size(opts: &Opts, ulimit: rlimit::rlim) -> u16 {
+    let mut batch_size: rlimit::rlim = opts.batch_size.into();
 
     // Adjust the batch size when the ulimit value is lower than the desired batch size
-    if ulimit < batch_size && !(cfg!(windows)) {
+    if ulimit < batch_size {
         warning!("File limit is lower than default batch size. Consider upping with --ulimt. May cause harm to sensitive servers",
             opts.quiet
         );
@@ -349,11 +404,7 @@ fn infer_batch_size(opts: &Opts, ulimit: u32) -> u16 {
             // ulimit is smaller than aveage batch size
             // user must have very small ulimit
             // decrease batch size to half of ulimit
-            if !(cfg!(windows)) {
-                warning!("Your file limit is very small, which negatively impacts RustScan's speed. Use the Docker image, or up the Ulimit with '--ulimit 5000'. ");
-            } else {
-                warning!("Windows is known to be much slower than scanning on Unix systems. Use the Docker image if you need faster speeds.");
-            }
+            warning!("Your file limit is very small, which negatively impacts RustScan's speed. Use the Docker image, or up the Ulimit with '--ulimit 5000'. ");
             info!("Halving batch_size because ulimit is smaller than average batch size");
             batch_size = ulimit / 2
         } else if ulimit > DEFAULT_FILE_DESCRIPTORS_LIMIT {
@@ -379,7 +430,6 @@ fn infer_batch_size(opts: &Opts, ulimit: u32) -> u16 {
 mod tests {
     use crate::{adjust_ulimit_size, infer_batch_size, parse_ips, print_opening, Opts, ScanOrder};
 
-    #[cfg(not(target_os = "windows"))]
     #[test]
     fn batch_size_lowered() {
         let opts = Opts {
@@ -391,6 +441,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -399,7 +450,6 @@ mod tests {
         assert!(batch_size < 50_000);
     }
 
-    #[cfg(not(target_os = "windows"))]
     #[test]
     fn batch_size_lowered_average_size() {
         let opts = Opts {
@@ -411,6 +461,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -418,7 +469,6 @@ mod tests {
 
         assert!(batch_size == 3_000);
     }
-    #[cfg(not(target_os = "windows"))]
     #[test]
     fn batch_size_equals_ulimit_lowered() {
         // because ulimit and batch size are same size, batch size is lowered
@@ -432,6 +482,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -439,7 +490,6 @@ mod tests {
 
         assert!(batch_size == 4_900);
     }
-    #[cfg(not(target_os = "windows"))]
     #[test]
     fn batch_size_adjusted_2000() {
         // ulimit == batch_size
@@ -452,6 +502,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -476,6 +527,7 @@ mod tests {
             timeout: 1_000,
             ulimit: None,
             command: Vec::new(),
+            config: false,
             accessible: true,
             scan_order: ScanOrder::Serial,
         };
@@ -498,6 +550,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);
 
@@ -517,6 +570,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);
 
@@ -536,6 +590,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);