Merge branch 'master' into bee-add-config-file

.all-contributorsrc

@@ -115,44 +115,6 @@
         "doc",
         "design"
       ]
-    },
-    {
-      "login": "Ferryistaken",
-      "name": "Alessandro Ferrari",
-      "avatar_url": "https://avatars3.githubusercontent.com/u/47927670?v=4",
-      "profile": "https://github.com/Ferryistaken",
-      "contributions": [
-        "content"
-      ]
-    },
-    {
-      "login": "Phenomite",
-      "name": "Phenomite",
-      "avatar_url": "https://avatars2.githubusercontent.com/u/8285537?v=4",
-      "profile": "https://github.com/Phenomite",
-      "contributions": [
-        "content"
-      ]
-    },
-    {
-      "login": "SuperSandro2000",
-      "name": "Sandro",
-      "avatar_url": "https://avatars2.githubusercontent.com/u/7258858?v=4",
-      "profile": "https://supersandro.de/",
-      "contributions": [
-        "content"
-      ]
-    },
-    {
-      "login": "caass",
-      "name": "Cass",
-      "avatar_url": "https://avatars2.githubusercontent.com/u/25358963?v=4",
-      "profile": "https://swag.lgbt",
-      "contributions": [
-        "platform",
-        "code",
-        "bug"
-      ]
     }
   ],
   "contributorsPerLine": 7,

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/rustscan_scripting_engine"]
-	path = src/rustscan_scripting_engine
-	url = https://github.com/RustScan/rustscan_scripting_engine

.travis.yml

@@ -54,14 +54,3 @@ after_failure:
   - wget https://raw.githubusercontent.com/DiscordHooks/travis-ci-discord-webhook/master/send.sh
   - chmod +x send.sh
   - ./send.sh failure $WEBHOOK_URL
-
-after_success: |
-  if [ -n "${GITHUB_TOKEN}" ]; then
-    [ "${TRAVIS_BRANCH} " = master ] &&
-    [ "${TRAVIS_PULL_REQUEST}" = false ] &&
-    cargo doc &&
-    echo "<meta http-equiv=refresh content=0;url=rustscan/index.html>" > target/doc/index.html &&
-    sudo pip install ghp-import &&
-    ghp-import -n target/doc &&
-    git push -fq https://${GITHUB_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages
-  fi

Cargo.lock

@@ -774,6 +774,7 @@ dependencies = [
  "rlimit",
  "shell-words",
  "structopt",
+ "toml",
 ]
 
 [[package]]
@@ -782,6 +783,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
 
+[[package]]
+name = "serde"
+version = "1.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
+
 [[package]]
 name = "shell-words"
 version = "1.0.0"
@@ -814,9 +821,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.17"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cc388d94ffabf39b5ed5fadddc40147cb21e605f53db6f8f36a625d27489ac5"
+checksum = "de5472fb24d7e80ae84a7801b7978f95a19ec32cb1876faea59ab711eb901976"
 dependencies = [
  "clap",
  "lazy_static",
@@ -825,9 +832,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.10"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e2513111825077552a6751dfad9e11ce0fba07d7276a3943a037d7e93e64c5f"
+checksum = "1e0eb37335aeeebe51be42e2dc07f031163fbabfa6ac67d7ea68b5c2f68d5f99"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -874,6 +881,15 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.6.0"

Cargo.toml

@@ -18,7 +18,7 @@ exclude = [
 
 [dependencies]
 colored = "2.0.0"
-structopt = "0.3.17"
+structopt = "0.3.16"
 async-std = "1.6.3"
 futures = "0.3"
 rlimit = "0.4.0"
@@ -26,10 +26,14 @@ shell-words = "1.0.0"
 log = "0.4.0"
 env_logger = "0.7.1"
 dirs = "3.0.1"
+toml = "0.5.6"
 gcd = "2.0.1"
 rand = "0.7.3"
 colorful = "0.2.1"
 ansi_term = "0.12.1"
+ureq = "1.3.0"
+serde = "1.0.115"
+serde_derive = "1.0.115"
 
 [package.metadata.deb]
 depends = "$auto, nmap"

README.md

@@ -67,7 +67,7 @@ RustScans **only** job is to reduce the friction between finding open ports and
 
 | Installation Guide | Documentation | Discord |
 | ------------------ | ------------- | ------- |
-| 📖 [Installation Guide](https://github.com/RustScan/RustScan#-full-installation-guide) | 📚 [Documentation](https://rustscan.github.io/RustScan/) | 🦜 [Discord](https://discord.gg/GFrQsGy)
+| 📖 [Installation Guide](https://github.com/RustScan/RustScan#-full-installation-guide) | 📚 [Documentation](https://github.com/RustScan/RustScan/issues/89) | 🦜 [Discord](https://discord.gg/GFrQsGy)
 
 ## 🙋 Table of Contents
 * 📖 [Installation Guide](https://github.com/RustScan/RustScan#-full-installation-guide)
@@ -241,7 +241,7 @@ OPTIONS:
 ARGS:
     <ips-or-hosts>...    A list of comma separated IP addresses or hosts to be scanned
     <command>...         The Nmap arguments to run. To use the argument -A, end RustScan's args with '-- -A'.
-                         Example: 'rustscan -t 1500 127.0.0.1 -- -A -sC'. This command adds -Pn -vvv -p $PORTS
+                         Example: 'rustscan -T 1500 127.0.0.1 -- -A -sC'. This command adds -Pn -vvv -p $PORTS
                          automatically to nmap. For things like --script '(safe and vuln)' enclose it in quotations
                          marks \"'(safe and vuln)'\"")
 ```
@@ -340,6 +340,22 @@ To increase accuracy, the easiest way is to increase the timeout. The default is
 
 Decreasing accuracy gives some speed bonus, but my testing found that batch size dramatically changed the speed whereas timeout did, but not so much.
 
+## 🤖 Config File
+
+RustScan uses a Config file to make things easier for you. And to also improve itself.
+The config file is used as the basis of the _adaptive learning_ mechanisms. This means that RustScan _learns_ from your scans and improves itself over time. (Don't worry, no bloated AI framework here 😉)
+
+The config file should be placed in APPDIRS. To find this folder, run RustScan and look out for RustScan saying "couldn't find config at <location>". That location is where your config file should be.
+
+Once you have this location, copy and paste this:
+
+```toml
+
+# The top ports
+[top_ports]
+80, 60, 443
+```
+
 # 🎪 Community
 
 Howdy Space Cow-Person 🤠🌌
@@ -381,7 +397,7 @@ Please read the [contributing.md file](contributing.md)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -404,12 +420,6 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://tgotwig.me"><img src="https://avatars0.githubusercontent.com/u/30773779?v=4" width="100px;" alt=""/><br /><sub><b>Thomas Gotwig</b></sub></a><br /><a href="#platform-TGotwig" title="Packaging/porting to new platform">📦</a></td>
     <td align="center"><a href="https://github.com/remigourdon"><img src="https://avatars3.githubusercontent.com/u/2874133?v=4" width="100px;" alt=""/><br /><sub><b>Rémi Gourdon</b></sub></a><br /><a href="https://github.com/RustScan/RustScan/commits?author=remigourdon" title="Documentation">📖</a> <a href="https://github.com/RustScan/RustScan/commits?author=remigourdon" title="Code">💻</a></td>
     <td align="center"><a href="https://cmnatic.co.uk"><img src="https://avatars3.githubusercontent.com/u/4163116?v=4" width="100px;" alt=""/><br /><sub><b>Ben (CMNatic)</b></sub></a><br /><a href="https://github.com/RustScan/RustScan/commits?author=cmnatic" title="Code">💻</a> <a href="https://github.com/RustScan/RustScan/commits?author=cmnatic" title="Documentation">📖</a> <a href="#design-cmnatic" title="Design">🎨</a></td>
-    <td align="center"><a href="https://github.com/Ferryistaken"><img src="https://avatars3.githubusercontent.com/u/47927670?v=4" width="100px;" alt=""/><br /><sub><b>Alessandro Ferrari</b></sub></a><br /><a href="#content-Ferryistaken" title="Content">🖋</a></td>
-    <td align="center"><a href="https://github.com/Phenomite"><img src="https://avatars2.githubusercontent.com/u/8285537?v=4" width="100px;" alt=""/><br /><sub><b>Phenomite</b></sub></a><br /><a href="#content-Phenomite" title="Content">🖋</a></td>
-    <td align="center"><a href="https://supersandro.de/"><img src="https://avatars2.githubusercontent.com/u/7258858?v=4" width="100px;" alt=""/><br /><sub><b>Sandro</b></sub></a><br /><a href="#content-SuperSandro2000" title="Content">🖋</a></td>
-  </tr>
-  <tr>
-    <td align="center"><a href="https://swag.lgbt"><img src="https://avatars2.githubusercontent.com/u/25358963?v=4" width="100px;" alt=""/><br /><sub><b>Cass</b></sub></a><br /><a href="#platform-caass" title="Packaging/porting to new platform">📦</a> <a href="https://github.com/RustScan/RustScan/commits?author=caass" title="Code">💻</a> <a href="https://github.com/RustScan/RustScan/issues?q=author%3Acaass" title="Bug reports">🐛</a></td>
   </tr>
 </table>
 

docker/ci_test/Dockerfile

@@ -0,0 +1,28 @@
+FROM archlinux:latest
+
+RUN yes | pacman -Syu
+RUN yes | pacman -S fakeroot binutils sudo make gcc gettext awk git
+
+WORKDIR /opt
+RUN git clone https://aur.archlinux.org/yay-git.git
+
+RUN useradd -m tecmint
+RUN passwd -d tecmint
+RUN printf 'tecmint ALL=(ALL) ALL\n' | tee -a /etc/sudoers
+
+WORKDIR /opt/yay-git
+RUN chown -R tecmint /opt/yay-git
+USER tecmint
+RUN makepkg -si --noconfirm
+
+# AUR
+RUN yay -S rustscan --noconfirm
+RUN rustscan 127.0.0.1 -q -b 250 -t 1
+RUN yay -R rustscan --noconfirm
+
+# homebrew
+RUN yes | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+ENV PATH="/home/linuxbrew/.linuxbrew/bin:${PATH}"
+RUN yes | brew tap brandonskerritt/rustscan
+RUN yes | brew install rustscan
+RUN rustscan 127.0.0.1 -q -b 250 -t 1

docker/rustscan-debbuilder/entrypoint.sh

@@ -5,6 +5,7 @@ git pull --force
 
 #amd64
 cargo deb
+yes | dpkg -i *.deb
 
 #arm64
 rustup target add arm-unknown-linux-gnueabihf

src/main.rs

@@ -13,11 +13,20 @@ use colorful::Colorful;
 use futures::executor::block_on;
 use rlimit::Resource;
 use rlimit::{getrlimit, setrlimit};
+use serde_derive::Deserialize;
 use std::collections::HashMap;
+use std::net::ToSocketAddrs;
 use std::process::Command;
 use std::str::FromStr;
-use std::{net::IpAddr, net::ToSocketAddrs, time::Duration};
+use std::{
+    fs::File,
+    io::{Read, Write},
+    net::IpAddr,
+    path::PathBuf,
+    time::Duration,
+};
 use structopt::{clap::arg_enum, StructOpt};
+use ureq;
 
 extern crate colorful;
 extern crate dirs;
@@ -97,6 +106,10 @@ struct Opts {
     #[structopt(short, long)]
     accessible: bool,
 
+    ///Config file usage
+    #[structopt(short, long)]
+    config: bool,
+
     /// The batch size for port scanning, it increases or slows the speed of
     /// scanning. Depends on the open file limit of your OS.  If you do 65535
     /// it will do every port at the same time. Although, your OS may not
@@ -120,13 +133,18 @@ struct Opts {
 
     /// The Nmap arguments to run.
     /// To use the argument -A, end RustScan's args with '-- -A'.
-    /// Example: 'rustscan -t 1500 127.0.0.1 -- -A -sC'.
+    /// Example: 'rustscan -T 1500 127.0.0.1 -- -A -sC'.
     /// This command adds -Pn -vvv -p $PORTS automatically to nmap.
     /// For things like --script '(safe and vuln)' enclose it in quotations marks \"'(safe and vuln)'\"")
     #[structopt(last = true)]
     command: Vec<String>,
 }
 
+#[derive(Deserialize)]
+struct ConfigFile {
+    ip: IpAddr,
+}
+
 #[cfg(not(tarpaulin_include))]
 /// Faster Nmap scanning with Rust
 /// If you're looking for the actual scanning, check out the module Scanner
@@ -149,6 +167,12 @@ fn main() {
         print_opening();
     }
 
+    /*
+    TODO when releasing the new config file feature
+    turn this on.
+    if opts.config {
+        let config = get_config_file();
+    }*/
     let ips: Vec<IpAddr> = parse_ips(&opts);
 
     if ips.is_empty() {
@@ -187,7 +211,7 @@ fn main() {
 
         let x = format!("{} Looks like I didn't find any open ports for {:?}. This is usually caused by a high batch size.
         \n*I used {} batch size, consider lowering to {} with {} or a comfortable number for your system.
-        \n Alternatively, increase the timeout if your ping is high. Rustscan -t 2000 for 2000 milliseconds (2s) timeout.\n",
+        \n Alternatively, increase the timeout if your ping is high. Rustscan -t 2000 for 2000 second timeout.\n",
         "ERROR",
         ip,
         opts.batch_size,
@@ -245,21 +269,65 @@ Faster Nmap scanning with Rust."#;
  --------------------------------------"#;
     println!("{}", info.gradient(Color::Yellow).bold());
     funny_opening!();
+    // println!("{}", loaded_config_file);
+}
 
+fn get_config_file() {
+    let result_get_location = get_location_config();
+    let location = match result_get_location {
+        Ok(path) => path,
+        Err(_) => panic!("Your system does not have appdirs."),
+    };
+    info!("The location of config is {:?}", location);
+
+    detail!(format!("The config file is at {:?}", location));
+
+    load_and_parse_config_file(location);
+}
+
+fn get_location_config() -> Result<std::path::PathBuf, &'static str> {
+    info!("Getting location of config file");
     let config_path = match dirs::config_dir() {
         Some(mut path) => {
             path.push("rustscan");
             path.push("config.toml");
-            path
+            Ok(path)
         }
-        None => panic!("Couldn't find config dir."),
+        None => Err("Your system does not have APPDIRS"),
     };
-
-    detail!(format!(
-        "{} {:?}",
-        "The config file is expected to be at", config_path
-    ));
+    config_path
 }
+
+fn load_and_parse_config_file(config_path: PathBuf) -> ConfigFile {
+    // Gets the file and returns the string of TOML
+    // TODO quiet mode
+
+    let mut file = File::open(&config_path);
+    let mut file: File = match file {
+        Ok(result) => result,
+        Err(_) => download_place_config_file(config_path),
+    };
+    let mut contents = String::new();
+    file.read_to_string(&mut contents).unwrap();
+    let config: ConfigFile = toml::from_str(&contents).unwrap();
+
+    config
+}
+// Downloads config file
+// Places into Appdirs
+
+fn download_place_config_file(config_location: PathBuf) -> std::fs::File {
+    info!("Placing config file");
+    let body = ureq::get("https://raw.githubusercontent.com/RustScan/RustScan/master/config.toml")
+        .call()
+        .into_string()
+        .unwrap();
+
+    let mut file = File::create(config_location).unwrap();
+    file.write(body.as_bytes()).unwrap();
+    file
+}
+
 #[cfg(not(tarpaulin_include))]
 fn build_nmap_arguments<'a>(
     addr: &'a str,
@@ -325,7 +393,7 @@ fn infer_batch_size(opts: &Opts, ulimit: rlimit::rlim) -> u16 {
 
     // Adjust the batch size when the ulimit value is lower than the desired batch size
     if ulimit < batch_size {
-        warning!("File limit is lower than default batch size. Consider upping with --ulimit. May cause harm to sensitive servers",
+        warning!("File limit is lower than default batch size. Consider upping with --ulimt. May cause harm to sensitive servers",
             opts.quiet
         );
 
@@ -373,6 +441,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -392,6 +461,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -412,6 +482,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -431,6 +502,7 @@ mod tests {
             timeout: 1_000,
             ulimit: Some(2_000),
             command: Vec::new(),
+            config: false,
             accessible: false,
             scan_order: ScanOrder::Serial,
         };
@@ -455,6 +527,7 @@ mod tests {
             timeout: 1_000,
             ulimit: None,
             command: Vec::new(),
+            config: false,
             accessible: true,
             scan_order: ScanOrder::Serial,
         };
@@ -477,6 +550,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);
 
@@ -496,6 +570,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);
 
@@ -515,6 +590,7 @@ mod tests {
             command: Vec::new(),
             accessible: false,
             scan_order: ScanOrder::Serial,
+            config: false,
         };
         let ips = parse_ips(&opts);