From 73224a588106c8867bd6a66e3258eae1fd857d87 Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Mon, 20 Feb 2017 15:38:45 +0100 Subject: [PATCH] get rid of data/ and utils/ directories in favor of stuff/ Following a summary of the changes in this commit: * Moved data/android/ to stuff/android/ as I'm not sure what else to do with it for now. * Moved data/src/binpackage/ to stuff/execsc/ as it contains the source code for the execsc.exe binary that we feature in our repository. * Moved data/strace.stp to stuff/strace.stp as it's required for STAP-based analysis of Linux binaries. * Removed data/test-internet.vbs as it's much easier to do this logic in Python and we have existing scripts for this in our unit tests. * Moved utils/darwin/ to stuff/darwin/ as I'm not sure what else to do with it right now. * Moved utils/rawdb.py to stuff/ as it may still be useful. * Removed utils/service.sh as we've superseded the Upstart scripts by a supervisor-based configuration file. * Removed utils/setup.py as we've superseded the entire setup process with a SaltStack-based module. * Removed utils/smtp_sinkhole.py as this should be done in Postfix. * Removed utils/{start,stop}-distributed.sh as this functionality has been replaced by a single worker in combination with supervisor. * Removed utils/stats.py as you can also get this information from the Cuckoo API and/or Cuckoo Web Interface. * Moved utils/suricata.sh to stuff/. * Moved utils/vpncheck.py to stuff/. --- data/__init__.py | 4 - data/test-internet.vbs | 37 -- .../android/anti-vm/fake-build.prop | 0 {data => stuff}/android/anti-vm/fake-cpuinfo | 0 {data => stuff}/android/anti-vm/fake-drivers | 0 .../android/apps/ImportContacts.apk | Bin {data => stuff}/android/apps/Superuser.apk | Bin ...bv.android.xposed.installer_v33_36570c.apk | Bin {data => stuff}/android/binaries/su | Bin {data => stuff}/android/create_guest_avd.sh | 0 {data => stuff}/android/hooking/Droidmon.apk | Bin .../android/hooking/EmulatorAntiDetect.apk | Bin {utils => stuff}/darwin/bootstrap_guest.sh | 0 {utils => stuff}/darwin/bootstrap_host.sh | 0 .../src/binpackage => stuff/execsc}/Makefile | 1 - .../src/binpackage => stuff/execsc}/execsc.c | 0 {utils => stuff}/rawdb.py | 8 +- {data => stuff}/strace.stp | 0 {utils => stuff}/suricata.sh | 0 {utils => stuff}/vpncheck.py | 14 +- utils/service.sh | 342 ----------- utils/setup.sh | 581 ------------------ utils/smtp_sinkhole.py | 52 -- utils/start-distributed.sh | 14 - utils/stats.py | 67 -- utils/stop-distributed.sh | 14 - 26 files changed, 12 insertions(+), 1122 deletions(-) delete mode 100644 data/__init__.py delete mode 100644 data/test-internet.vbs rename {data => stuff}/android/anti-vm/fake-build.prop (100%) rename {data => stuff}/android/anti-vm/fake-cpuinfo (100%) rename {data => stuff}/android/anti-vm/fake-drivers (100%) rename {data => stuff}/android/apps/ImportContacts.apk (100%) rename {data => stuff}/android/apps/Superuser.apk (100%) rename {data => stuff}/android/apps/de.robv.android.xposed.installer_v33_36570c.apk (100%) rename {data => stuff}/android/binaries/su (100%) rename {data => stuff}/android/create_guest_avd.sh (100%) rename {data => stuff}/android/hooking/Droidmon.apk (100%) rename {data => stuff}/android/hooking/EmulatorAntiDetect.apk (100%) rename {utils => stuff}/darwin/bootstrap_guest.sh (100%) rename {utils => stuff}/darwin/bootstrap_host.sh (100%) rename {data/src/binpackage => stuff/execsc}/Makefile (98%) rename {data/src/binpackage => stuff/execsc}/execsc.c (100%) rename {utils => stuff}/rawdb.py (64%) rename {data => stuff}/strace.stp (100%) mode change 100755 => 100644 rename {utils => stuff}/suricata.sh (100%) rename {utils => stuff}/vpncheck.py (88%) delete mode 100755 utils/service.sh delete mode 100755 utils/setup.sh delete mode 100755 utils/smtp_sinkhole.py delete mode 100755 utils/start-distributed.sh delete mode 100755 utils/stats.py delete mode 100755 utils/stop-distributed.sh diff --git a/data/__init__.py b/data/__init__.py deleted file mode 100644 index 57edaa85f..000000000 --- a/data/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (C) 2010-2013 Claudio Guarnieri. -# Copyright (C) 2014-2016 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. diff --git a/data/test-internet.vbs b/data/test-internet.vbs deleted file mode 100644 index 9078a2252..000000000 --- a/data/test-internet.vbs +++ /dev/null @@ -1,37 +0,0 @@ -' http://www.motobit.com/tips/detpg_read-write-binary-files/ -Function SaveBinaryData(FileName, ByteArray) - Const adTypeBinary = 1 - Const adSaveCreateOverWrite = 2 - - ' create Stream object - dim BinaryStream - set BinaryStream = CreateObject("ADODB.Stream") - - ' specify stream type - we want to save binary data. - BinaryStream.Type = adTypeBinary - - ' open the stream and write binary data To the object - BinaryStream.Open - BinaryStream.Write ByteArray - - ' save binary data to disk - BinaryStream.SaveToFile FileName, adSaveCreateOverWrite -End Function - -' http://stackoverflow.com/questions/5907089/how-to-post-https-request-using-vbscript -Function DownloadFile(FileName, Url) - dim http - set http = createobject("MSXML2.ServerXMLHTTP") - - http.Open "GET", Url, False - - ' 2 stands for SXH_OPTION_IGNORE_SERVER_SSL_CERT_ERROR_FLAGS - ' 13056 means ignore all server side cert error - http.setOption 2, 13056 - http.Send - - ' read response body - SaveBinaryData FileName, http.responseBody -End Function - -DownloadFile "index.html", "http://google.com/" diff --git a/data/android/anti-vm/fake-build.prop b/stuff/android/anti-vm/fake-build.prop similarity index 100% rename from data/android/anti-vm/fake-build.prop rename to stuff/android/anti-vm/fake-build.prop diff --git a/data/android/anti-vm/fake-cpuinfo b/stuff/android/anti-vm/fake-cpuinfo similarity index 100% rename from data/android/anti-vm/fake-cpuinfo rename to stuff/android/anti-vm/fake-cpuinfo diff --git a/data/android/anti-vm/fake-drivers b/stuff/android/anti-vm/fake-drivers similarity index 100% rename from data/android/anti-vm/fake-drivers rename to stuff/android/anti-vm/fake-drivers diff --git a/data/android/apps/ImportContacts.apk b/stuff/android/apps/ImportContacts.apk similarity index 100% rename from data/android/apps/ImportContacts.apk rename to stuff/android/apps/ImportContacts.apk diff --git a/data/android/apps/Superuser.apk b/stuff/android/apps/Superuser.apk similarity index 100% rename from data/android/apps/Superuser.apk rename to stuff/android/apps/Superuser.apk diff --git a/data/android/apps/de.robv.android.xposed.installer_v33_36570c.apk b/stuff/android/apps/de.robv.android.xposed.installer_v33_36570c.apk similarity index 100% rename from data/android/apps/de.robv.android.xposed.installer_v33_36570c.apk rename to stuff/android/apps/de.robv.android.xposed.installer_v33_36570c.apk diff --git a/data/android/binaries/su b/stuff/android/binaries/su similarity index 100% rename from data/android/binaries/su rename to stuff/android/binaries/su diff --git a/data/android/create_guest_avd.sh b/stuff/android/create_guest_avd.sh similarity index 100% rename from data/android/create_guest_avd.sh rename to stuff/android/create_guest_avd.sh diff --git a/data/android/hooking/Droidmon.apk b/stuff/android/hooking/Droidmon.apk similarity index 100% rename from data/android/hooking/Droidmon.apk rename to stuff/android/hooking/Droidmon.apk diff --git a/data/android/hooking/EmulatorAntiDetect.apk b/stuff/android/hooking/EmulatorAntiDetect.apk similarity index 100% rename from data/android/hooking/EmulatorAntiDetect.apk rename to stuff/android/hooking/EmulatorAntiDetect.apk diff --git a/utils/darwin/bootstrap_guest.sh b/stuff/darwin/bootstrap_guest.sh similarity index 100% rename from utils/darwin/bootstrap_guest.sh rename to stuff/darwin/bootstrap_guest.sh diff --git a/utils/darwin/bootstrap_host.sh b/stuff/darwin/bootstrap_host.sh similarity index 100% rename from utils/darwin/bootstrap_host.sh rename to stuff/darwin/bootstrap_host.sh diff --git a/data/src/binpackage/Makefile b/stuff/execsc/Makefile similarity index 98% rename from data/src/binpackage/Makefile rename to stuff/execsc/Makefile index e052c988b..748f7c148 100644 --- a/data/src/binpackage/Makefile +++ b/stuff/execsc/Makefile @@ -1,3 +1,2 @@ execsc.exe: execsc.c i586-mingw32msvc-cc -Wall -o $@ $< - diff --git a/data/src/binpackage/execsc.c b/stuff/execsc/execsc.c similarity index 100% rename from data/src/binpackage/execsc.c rename to stuff/execsc/execsc.c diff --git a/utils/rawdb.py b/stuff/rawdb.py similarity index 64% rename from utils/rawdb.py rename to stuff/rawdb.py index 45b718916..770c2931c 100755 --- a/utils/rawdb.py +++ b/stuff/rawdb.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2014-2016 Cuckoo Foundation. +# Copyright (C) 2015-2017 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -7,13 +7,11 @@ import IPython import os.path import sys -sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "..")) - from cuckoo.core.database import * -from cuckoo.misc import set_cwd +from cuckoo.misc import decide_cwd if __name__ == "__main__": - set_cwd(os.path.expanduser("~/.cuckoo")) + decide_cwd(exists=True) db = Database() db.connect() diff --git a/data/strace.stp b/stuff/strace.stp old mode 100755 new mode 100644 similarity index 100% rename from data/strace.stp rename to stuff/strace.stp diff --git a/utils/suricata.sh b/stuff/suricata.sh similarity index 100% rename from utils/suricata.sh rename to stuff/suricata.sh diff --git a/utils/vpncheck.py b/stuff/vpncheck.py similarity index 88% rename from utils/vpncheck.py rename to stuff/vpncheck.py index 40fbc10df..31b464b43 100755 --- a/utils/vpncheck.py +++ b/stuff/vpncheck.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2016 Cuckoo Foundation. +# Copyright (C) 2016-2017 Cuckoo Foundation. # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org # See the file 'docs/LICENSE' for copying permission. @@ -10,8 +10,6 @@ import socket import struct import sys -sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "..")) - from cuckoo.core.rooter import rooter, vpns from cuckoo.core.startup import init_rooter, init_routing @@ -49,7 +47,10 @@ if __name__ == "__main__": ipaddr = get_ip_address(vpns[vpn].interface) - rooter("forward_enable", vpns[vpn].interface, vpns[vpn].interface, ipaddr) + rooter( + "forward_enable", vpns[vpn].interface, + vpns[vpn].interface, ipaddr + ) rooter("srcroute_enable", vpns[vpn].rt_table, ipaddr) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -66,7 +67,10 @@ if __name__ == "__main__": print>>sys.stderr, "Unable to connect through VPN", vpn error = 1 - rooter("forward_disable", vpns[vpn].interface, vpns[vpn].interface, ipaddr) + rooter( + "forward_disable", vpns[vpn].interface, + vpns[vpn].interface, ipaddr + ) rooter("srcroute_disable", vpns[vpn].rt_table, ipaddr) exit(error) diff --git a/utils/service.sh b/utils/service.sh deleted file mode 100755 index d3e53d05f..000000000 --- a/utils/service.sh +++ /dev/null @@ -1,342 +0,0 @@ -#!/bin/sh -set -e - -echo "### NOTICE ###" >&2 -echo "This script is a work-in-progress, has not been yet documented, " >&2 -echo "and may not work as expected." >&2 -echo "### END OF NOTICE ###" >&2 - -# TODO Load Virtual Machines into tmpfs, if enabled. - -_about_upstart() { - echo "Using Upstart technique.." -} - -_install_configuration() { - if [ -f "/etc/default/cuckoo" ]; then - # TODO Ask yes/no to force overwrite. - echo "Not overwriting existing configuration.." - return - fi - - cat > /etc/default/cuckoo << EOF -# Configuration file for the Cuckoo Sandbox service(s). - -# It is possible to allow the virtual machines to connect to the entire -# internet through the vmcloak-iptables script. Enable by uncommenting and -# setting the following value. Give the network interface(s) that can allow -# internet access to the virtual machines. -# VMINTERNET="eth0 wlan0" - -# IP address and port of the Cuckoo API. Cuckoo API is by default -# turned *OFF*. Enable by uncommenting and setting the value. -# APIADDR="127.0.0.1" -# APIPORT=8090 - -# IP address and port of the Cuckoo Web Interface. The Cuckoo Web Interface -# is by default turned *OFF*. Enable by uncommenting and setting the value. -# WEBADDR="127.0.0.1" -# WEBPORT=8000 - -# Run Suricata in the background? -SURICATA="0" - -# Start Cuckoo in verbose mode. Toggle to 1 to enable verbose mode. -VERBOSE="0" -EOF -} - -_remove_configuration() { - rm -f /etc/default/cuckoo -} - -_install_upstart() { - cat > /etc/init/cuckoo.conf << EOF -# Cuckoo daemon service. - -description "cuckoo daemon" -start on runlevel [2345] -chdir "$CUCKOO" - -# Give Cuckoo time to cleanup. -kill signal SIGINT -kill timeout 600 - -# Restart Cuckoo if it exits. -respawn - -# Upstart ignores limits found in /etc/security/limits.conf. -limit nofile 499999 999999 - -env CONFFILE="$CONFFILE" -env VMINTERNET="" -env CHECKVMS="/etc/default/cuckoo-setup" - -pre-start script - . "\$CONFFILE" - - vmcloak-vboxnet0 - - if [ -n "\$VMINTERNET" ]; then - vmcloak-iptables 192.168.56.1/24 "\$VMINTERNET" - fi - - # Check up on all VMs and fix any if required. - if [ -f "\$CHECKVMS" ]; then - ./utils/setup.sh -S "\$CHECKVMS" -V - fi -end script - -script - . "\$CONFFILE" - - if [ "\$VERBOSE" -eq 0 ]; then - exec ./cuckoo.py -u "$USERNAME" - else - exec ./cuckoo.py -u "$USERNAME" -d - fi -end script -EOF - - cat > /etc/init/cuckoo-process.conf << EOF -# Cuckoo results processing service. - -description "start cuckoo results processing" -start on started cuckoo -stop on stopped cuckoo - -env PROCESSES=4 - -pre-start script - echo STARTING - for i in \$(seq 1 \$PROCESSES); do - start cuckoo-process2 INSTANCE=process\$i - done -end script -EOF - - cat > /etc/init/cuckoo-process2.conf << EOF -# Cuckoo results processing service. - -description "cuckoo results processing" -stop on stopping cuckoo-process -setuid "$USERNAME" -chdir "$CUCKOO" -instance \$INSTANCE - -# Restart Cuckoo report processing if it exits unexpectedly. -respawn - -env CONFFILE="$CONFFILE" - -script - . "\$CONFFILE" - - exec ./utils/process2.py "\$INSTANCE" -end script -EOF - - cat > /etc/init/cuckoo-api.conf << EOF -# Cuckoo API server service. - -description "cuckoo api server" -start on started cuckoo -stop on stopped cuckoo -setuid "$USERNAME" -chdir "$CUCKOO" - -env CONFFILE="$CONFFILE" -env APIADDR="" -env APIPORT=8090 - -script - . "\$CONFFILE" - - if [ -n "\$APIADDR" ]; then - exec ./utils/api.py -H "\$APIADDR" -p "\$APIPORT" - fi -end script -EOF - - cat > /etc/init/cuckoo-distributed-instance.conf << EOF -# Cuckoo distributed API node instance service. - -description "cuckoo distributed api node instance service" -setuid "$USERNAME" -chdir "$CUCKOO/distributed" -instance \$INSTANCE -respawn - -env CONFFILE="$CONFFILE" - -script - . "\$CONFFILE" - - if [ "\$VERBOSE" -eq 0 ]; then - exec ./instance.py "\$INSTANCE" - else - exec ./instance.py "\$INSTANCE" -v - fi -end script -EOF - - cat > /etc/uwsgi/apps-available/cuckoo-distributed.ini << EOF -[uwsgi] -plugins = python -chdir = $CUCKOO/distributed -file = app.py -uid = $USERNAME -gid = $USERNAME -EOF - - ln -s /etc/uwsgi/apps-available/cuckoo-distributed.ini \ - /etc/uwsgi/apps-enabled/cuckoo-distributed.ini - - cat > /etc/nginx/sites-available/cuckoo-distributed << EOF -upstream _uwsgi_cuckoo_distributed { - server unix:/run/uwsgi/app/cuckoo-distributed/socket; -} - -server { - # If required, prepend a listening IP address. - listen 9003; - - location / { - client_max_body_size 100M; - uwsgi_pass _uwsgi_cuckoo_distributed; - include uwsgi_params; - } -} -EOF - - ln -s /etc/nginx/sites-available/cuckoo-distributed \ - /etc/nginx/sites-enabled/cuckoo-distributed - - cat > /etc/init/cuckoo-web.conf << EOF -# Cuckoo Web Interface server. - -description "cuckoo web interface service" -start on started cuckoo -stop on stopped cuckoo -setuid "$USERNAME" -chdir "$(readlink -f "$CUCKOO/web/")" - -env CONFFILE="$CONFFILE" -env WEBADDR="" -env WEBPORT=8000 - -script - . "\$CONFFILE" - - if [ -n "\$WEBADDR" ]; then - exec ./manage.py runserver "\$WEBADDR:\$WEBPORT" - fi -end script -EOF - echo "Cuckoo Service scripts installed!" -} - -_remove_upstart() { - rm -f /etc/init/cuckoo.conf - rm -f /etc/init/cuckoo-api.conf - rm -f /etc/init/cuckoo-process.conf - rm -f /etc/init/cuckoo-process2.conf - rm -f /etc/init/cuckoo-distributed-instance.conf - rm -f /etc/init/cuckoo-web.conf -} - -_reload_upstart() { - initctl reload-configuration -} - -_start_upstart() { - initctl start cuckoo -} - -_stop_upstart() { - initctl stop cuckoo -} - -_restart_upstart() { - initctl restart cuckoo -} - -case "$(lsb_release -is)" in - Ubuntu) - alias _about=_about_upstart - alias _install=_install_upstart - alias _remove=_remove_upstart - alias _reload=_reload_upstart - alias _start=_start_upstart - alias _stop=_stop_upstart - alias _restart=_restart_upstart - ;; - - *) - echo "Unsupported Linux distribution.." - exit 1 -esac - -if [ "$#" -eq 0 ]; then - echo "Usage: $0 " - echo "-u --username: Username from which to run Cuckoo." - echo "-c --cuckoo: Directory where Cuckoo is located." - exit 1 -fi - -if [ "$(id -u)" -ne 0 ]; then - echo "This script should be run as root." - exit 1 -fi - -USERNAME="cuckoo" -CONFFILE="/etc/default/cuckoo" -CUCKOO="/home/cuckoo/cuckoo/" - -# Note that this way the variables have to be set before the -# actions are invoked. -while [ "$#" -ne 0 ]; do - ACTION="$1" - shift - - case "$ACTION" in - install) - _about - _install - _install_configuration - _reload - ;; - - remove) - _remove - _remove_configuration - _reload - ;; - - start) - _start - ;; - - stop) - _stop - ;; - - restart) - _restart - ;; - - -u|--username) - USERNAME="$1" - shift - ;; - - -c|--cuckoo) - CUCKOO="$1" - shift - ;; - - *) - echo "Requested invalid action." - exit 1 - esac -done diff --git a/utils/setup.sh b/utils/setup.sh deleted file mode 100755 index 3d85305ce..000000000 --- a/utils/setup.sh +++ /dev/null @@ -1,581 +0,0 @@ -#!/bin/bash - -echo "### NOTICE ###" >&2 -echo "This script is a work-in-progress, has not been yet documented, " >&2 -echo "and may not work as expected." >&2 -echo "### END OF NOTICE ###" >&2 - -# Default values. -WIN7="0" -VMCOUNT="40" -ISOFILE="" -SERIALKEY="" -TMPFS="0" -TAGS="" -INTERFACES="eth0 wlan0" -CLEAN="0" -DEPENDENCIES="" -BASEDIR="/home/cuckoo" -VMCHECKUP="0" -LONGTERM="0" -CPUCOUNT="1" - -usage() { - echo "Usage: $0 [options...]" - echo "-S --settings: Load configuration from a file." - echo "-7 --win7: Create Windows 7 x64 Virtual Machines." - echo " --win7x86: Create Windows 7 x86 Virtual Machines." - echo "-c --vmcount: Amount of Virtual Machines to be created." - echo "-i --iso: Path to the Windows Installer ISO." - echo "-s --serial-key: Serial Key for the given Windows XP version." - echo "-t --tmpfs: Indicate tmpfs should be used for snapshots." - echo "-T --tags: Tags for the Virtual Machines." - echo "-I --interfaces: Interfaces to route Virtual Machine internet" - echo " through. Defaults to eth0 wlan0." - echo "-C --clean: Clean the Cuckoo setup." - echo "-d --dependencies: Dependencies to install in the Virtual Machine." - echo "-b --basedir: Base directory for Virtual Machine files." - echo "-V --vms: Only check Virtual Machines, don't re-setup." - echo "-l --longterm: Indicate this is a longterm analysis setup." - echo "-u --cpucount: Amount of CPUs per Virtual Machine." - exit 1 -} - -EGGNAME="winxp" -MOUNTOS="winxp" -WINOS="--winxp" - -while [ "$#" -gt 0 ]; do - option="$1" - shift - - case "$option" in - -h|--help) - usage - ;; - - -7|--win7) - WIN7="1" - EGGNAME="win7x64" - MOUNTOS="win7" - WINOS="--win7x64" - ;; - - --win7x86) - WIN7="1" - EGNNAME="win7x86" - MOUNTOS="win7" - WINOS="--win7" - ;; - - -c|--vmcount) - VMCOUNT="$1" - shift - ;; - - -i|--iso) - ISOFILE="$1" - shift - ;; - - -s|--serial-key) - SERIALKEY="$1" - shift - ;; - - -t|--tmpfs) - TMPFS="1" - ;; - - -T|--tags) - TAGS="$1" - shift - ;; - - -I|--interfaces) - INTERFACES="$1" - shift - ;; - - -C|--clean) - CLEAN="1" - ;; - - -d|--dependencies) - DEPENDENCIES="$1" - shift - ;; - - -b|--basedir) - BASEDIR="$1" - shift - ;; - - -S|--settings) - SETTINGS="$1" - shift - ;; - - -V|--vms) - VMCHECKUP="1" - ;; - - -l|--longterm) - LONGTERM="1" - ;; - - -u|--cpucount) - CPUCOUNT="$1" - shift - ;; - - *) - echo "$0: Invalid argument.. $1" >&2 - usage - exit 1 - ;; - esac -done - -if [ "$(id -u)" -ne 0 ]; then - echo "You'll probably want to run this script as root." - exit 1 -fi - -if [ "$CLEAN" -ne 0 ]; then - yes|sudo -u cuckoo -i vmcloak-removevms - umount /home/cuckoo/vmmount - rm -rf /home/cuckoo/{.config,.vmcloak,vmmount} - rm -rf /home/cuckoo/{cuckoo,cuckoo.git} - rm -rf "$BASEDIR/vms" "$BASEDIR/vmbackup" - rm -rf /opt/cuckoo - exit 0 -fi - -# Load configuration settings from a file if one has been provided. -# Note that any values in the configuration file will overwrite -# settings provided on the command-line. -if [ -n "$SETTINGS" ]; then - . "$SETTINGS" -fi - -if [ -z "$ISOFILE" ]; then - echo "Please specify the path to a Windows XP or Windows 7 Installer ISO." - exit 1 -fi - -if [ "$WIN7" -eq 0 ] && [ -z "$SERIALKEY" ]; then - echo "Please specify a working serial key." - exit 1 -fi - -if [ "$TMPFS" -ne 0 ] && [ "$LONGTERM" -ne 0 ]; then - echo "It is not recommended to use tmpfs in a longterm setup." - echo "Please update your settings or remove this check." - exit 1 -fi - -if [ "$LONGTERM" -ne 0 ] && [ "$VMCOUNT" -ne 0 ]; then - echo "In longterm mode virtual machines should not be created through the" - echo "setup.sh script - this is handled by the vm cronjob." - exit 1 -fi - -_clone_cuckoo() { - local gitrepo="" - - # Fetch Cuckoo or in the case of a longterm setup, longcuckoo. - if [ "$LONGTERM" -eq 0 ]; then - sudo -u cuckoo -i \ - git clone --bare https://github.com/cuckoosandbox/cuckoo.git - - gitrepo="cuckoo.git" - else - sudo -u cuckoo -i \ - git clone --bare https://github.com/jbremer/longcuckoo.git - - gitrepo="longcuckoo.git" - fi - - sudo -u cuckoo -i tee "$gitrepo/hooks/post-receive" << EOF -#!/bin/bash - -read commit - -if ! echo \$commit|grep production$; then - echo -e "\\e[31mWe only check out changes to the production branch.\\e[0m" - exit -fi - -GIT_WORK_TREE=/opt/cuckoo git checkout -f production -EOF - - mkdir -p "/opt/cuckoo" - chown cuckoo:cuckoo "/opt/cuckoo" - chmod 755 "/home/cuckoo/" "/opt/cuckoo" - sudo -u cuckoo -i chmod +x "$gitrepo/hooks/post-receive" - - # Checkout master branch of the repository (if this was not done already). - if [ -z "$(ls -A /opt/cuckoo)" ]; then - sudo -u cuckoo -i \ - git --work-tree /opt/cuckoo --git-dir "$gitrepo" checkout -f master - fi - - # Add the Suricata reboot crontab entry. - if ! grep suricata.sh <(crontab -l); then - (crontab -l ; echo @reboot /opt/cuckoo/utils/suricata.sh)|crontab - - fi - - # Delete the cuckoo1 machine that is included in the VirtualBox - # configuration by default. - sudo -u cuckoo -i "/opt/cuckoo/utils/machine.py" --delete cuckoo1 -} - -_setup() { - # All functionality related to setting up the machine - this is not - # required when doing a Virtual Machine checkup. - - # Add the VirtualBox apt repository. - if [ ! -e /etc/apt/sources.list.d/virtualbox.list ]; then - # Update our apt repository with VirtualBox "contrib". - DEBVERSION="$(lsb_release -cs)" - echo "deb http://download.virtualbox.org/virtualbox/debian" \ - "$DEBVERSION contrib" >> /etc/apt/sources.list.d/virtualbox.list - - # Add the VirtualBox public key to our apt repository. - wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- \ - | apt-key add - - fi - - # Add the ElasticSearch apt repository. - if [ ! -e /etc/apt/sources.list.d/elasticsearch.list ]; then - echo "deb http://packages.elastic.co/elasticsearch/2.x/debian" \ - "stable main" >> /etc/apt/sources.list.d/elasticsearch.list - - wget -q https://packages.elastic.co/GPG-KEY-elasticsearch -O- \ - | sudo apt-key add - - fi - - # Update apt repository and install required packages. - apt-get update -y --force-yes - apt-get install -y --force-yes sudo git python-dev python-pip postgresql \ - libpq-dev python-dpkt vim tcpdump libcap2-bin genisoimage pwgen \ - htop tig mosh mongodb uwsgi uwsgi-plugin-python nginx virtualbox-4.3 \ - libffi-dev libxml2-dev libxslt1-dev libjpeg-dev samba-common-bin \ - ethtool elasticsearch linux-headers-$(uname -r) - - # Create the main postgresql cluster. In recent versions of Ubuntu Server - # 14.04 you have to do this manually. If it already exists this command - # will simply fail. - pg_createcluster 9.3 main --start - - # Install the VirtualBox Extension Pack for VRDE support. - VBOXVERSION="$(VBoxManage --version|sed s/r/\-/)" - PRETTYNAME="Oracle_VM_VirtualBox_Extension_Pack-$VBOXVERSION" - wget "http://cuckoo.sh/vmcloak-files/${PRETTYNAME}.vbox-extpack" - VBoxManage extpack install --replace "${PRETTYNAME}.vbox-extpack" - rm -f "${PRETTYNAME}.vbox-extpack" - - # Allow tcpdump to dump packet captures when executed as a normal user. - setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump - - # Setup a Cuckoo user. - useradd cuckoo -d /home/cuckoo -s /bin/bash - mkdir -p /home/cuckoo - chown cuckoo:cuckoo /home/cuckoo - - # Copy any authorized keys from the current user to the cuckoo user. - mkdir -p /home/cuckoo/.ssh - cp ~/.ssh/authorized_keys /home/cuckoo/.ssh/authorized_keys - chown cuckoo:cuckoo /home/cuckoo/.ssh/authorized_keys - - # Add the www-data user to the cuckoo group. - adduser www-data cuckoo - - # TODO Somehow vmtemp is not properly propagated into the vmcloak - # configuration thing, having to run the setup.sh script twice to - # actually start creating the bird. - VMTEMP="$(mktemp -d "/home/cuckoo/tempXXXXXX")" - - chown cuckoo:cuckoo "/home/cuckoo/" - chown -R cuckoo:cuckoo "$VMTEMP" - chmod 755 "/home/cuckoo/" "$VMTEMP" - - # Install required packages part two. - pip install --upgrade mitmproxy psycopg2 vmcloak - - # Clone the Cuckoo repository and initialize it. - _clone_cuckoo - - # Install required packages part three. - pip install -r "/opt/cuckoo/requirements.txt" - - # Create a random password. - # PASSWORD="$(pwgen -1 16)" - PASSWORD="cuckoo" - - sql_query() { - echo "$1"|sudo -u postgres psql - } - - sql_query "DROP DATABASE cuckoo" - sql_query "CREATE DATABASE cuckoo" - - sql_query "DROP USER cuckoo" - sql_query "CREATE USER cuckoo WITH PASSWORD '$PASSWORD'" - - # Install the Upstart/SystemV scripts. - /opt/cuckoo/utils/service.sh -c /opt/cuckoo install - - # Fetch the community signatures and monitoring binaries. - sudo -u cuckoo -i /opt/cuckoo/utils/community.py -wafb master - - # Add "nmi_watchdog=0" to the GRUB commandline if it's not in there already. - if ! grep nmi_watchdog /etc/default/grub; then - cat >> /etc/default/grub << EOF - -# Add nmi_watchdog=0 to the GRUB commandline to prevent -# VirtualBox from kernel panicing when the load increases. -GRUB_CMDLINE_LINUX_DEFAULT="\$GRUB_CMDLINE_LINUX_DEFAULT nmi_watchdog=0" -EOF - fi - - # Recreate the GRUB configuration. - grub-mkconfig -o /boot/grub/grub.cfg - - # Increase the file descriptor limits. TODO How to set it for just the cuckoo - # user? Doesn't seem to work when using 'cuckoo' instead of '*'. - if ! grep "* soft nofile" /etc/security/limits.conf; then - cat >> /etc/security/limits.conf << EOF - -# Set the file descriptor limit fairly high. -* soft nofile 499999 -* hard nofile 999999 -EOF - fi - - # For longterm setups we install the VM provisioning script and cronjob. - if [ "$LONGTERM" -ne 0 ]; then - CRONJOB="/home/cuckoo/vmprovision.sh" - - # Install the machine cronjob. - "/opt/cuckoo/utils/experiment.py" machine-cronjob install \ - "cpucount=$CPUCOUNT" "path=$CRONJOB" "basedir=$BASEDIR" - chown cuckoo:cuckoo "$CRONJOB" - chmod +x "$CRONJOB" - - # We want to run the vm provisioning cronjob every five minutes. - # Ensure that we only install the cronjob entry once. - CRONTAB="$(crontab -u cuckoo -l)" - if [[ ! "$CRONTAB" =~ "vmprovision.sh" ]]; then - (echo "$CRONTAB" ; echo "*/5 * * * * $CRONJOB")|crontab -u cuckoo - - fi - fi - - # TODO Should be automated away. - echo "PostgreSQL connection string: " \ - "postgresql://cuckoo:$PASSWORD@localhost/cuckoo" -} - -_create_virtual_machines() { - # Prepare the machine for virtual machines and actually create them. - - # Ensure that vboxnet0 is up and running. - vmcloak-vboxnet0 - - # Kill all VirtualBox processes as otherwise the listening port for - # vmcloak-clone might still be in use.. TODO This can probably be removed. - vmcloak-killvbox - - if [ -z "$SERIALKEY" ]; then - serial="" - else - serial="--serial-key $SERIALKEY" - fi - - options="$serial --cpus $CPUCOUNT" - - # Attempt to create a new image if one does not already exist. - sudo -u cuckoo -i vmcloak init "${EGGNAME}_bird" "$WINOS" $options - if [ "$?" -eq 0 ]; then - sudo -u cuckoo -i vmcloak install "${EGGNAME}_bird" $DEPENDENCIES - fi - - # Create various Virtual Machine eggs. - for i in $(seq 1 "$VMCOUNT"); do - name="${EGGNAME}_egg$i" - - # Ensure this Virtual Machine has not already been created. - if grep '"'$name'"' <(sudo -u cuckoo -i VBoxManage list vms); then - continue - fi - - # As vmcloak-clone will add an entry for this node we remove it just - # in case it did already exist. - "/opt/cuckoo/utils/machine.py" --delete "$name" - - # Delete any remaining files for this Virtual Machine just in case - # they were still present. - rm -rf "$VMBACKUP/$name" - - echo "Creating Virtual Machine $name.." - sudo -u cuckoo -i vmcloak snapshot "${EGGNAME}_bird" \ - "$name" "192.168.56.$((2+$i))" - - echo "Registering Virtual Machine $name.." - sudo -u cuckoo -i vmcloak register "$name" /opt/cuckoo - done - - rm -rf "$VMCLOAKCONF" "$VMTEMP" - - # In longterm modes we have a different script that provides VMs for us. - if [ "$LONGTERM" -ne 0 ]; then - sudo -u cuckoo -i sh "$CRONJOB" - fi - - # Remove all Virtual Machine related logfiles, we're not interested - # in keeping those. - rm -f $(find "$VMBACKUP" -type f|grep "\.log") -} - -_setup_vms() { - # Create directories in the target directory for each directory found in - # the source directory. Then create a symlink for all .vdi and .sav files - # and copy over all .vbox files. - if [ ! -d "$1" ] || [ ! -d "$2" ]; then - echo "Missing parameter(s) for setup-vms.." - exit 1 - fi - - local source="$1" target="$2" - - # Create each directory. - for dirname in $(cd "$source" && find * -type d); do - sudo -u cuckoo mkdir -p "$target/$dirname" - done - - # Make symlinks of all files .vdi and .sav files, these are considered - # readonly files by VirtualBox due to the immutable disk property. - for filename in $(cd "$source" && find bird*.vdi */*/*.vdi */*/*.sav); do - sudo -u cuckoo ln -fs "$source/$filename" "$target/$filename" - done - - # Copy all .vbox files, these may be modified by VirtualBox. In the case - # that somehow we end up with out of diskspace issues and the updated - # .vbox files can not be written to disk, then the virtual machine is - # essentially bricked. These files are only a couple of kilobytes anyway. - for filename in $(cd "$source" && find */*.vbox); do - sudo -u cuckoo cp "$source/$filename" "$target/$filename" - done -} - -_setup_vmmount() { - if [ ! -d "$1" ] || [ ! -d "$2" ]; then - echo "Missing parameter(s) for setup-vmmount.." - exit 1 - fi - - local source="$1" target="$2" - - # Unmount just in case it was mounted. - umount "$target" - - # Calculate the required size for the tmpfs mount. Round up to megabytes. - REQSIZE="$(du -s "$source"|cut -f1)" - REQSIZE="$(($REQSIZE/1024+1))M" - - mount -o "size=$REQSIZE,uid=cuckoo,gid=cuckoo" -t tmpfs tmpfs "$target" - - # Copy all files from the backup to the mount. - sudo -u cuckoo cp -rf "$source/." "$target" -} - -_initialize_from_backup() { - # Initialize our setup from our backup, if available. - local tmpfs="$1" - - if [ "$tmpfs" -ne 0 ]; then - # Initialize a mount with all the files from the backup directory. - _setup_vmmount "$VMBACKUP" "$VMMOUNT" - - # Initialize vms to point to the mount directory. - _setup_vms "$VMMOUNT" "$VMS" - else - # Initialize vms to point to the backup directory. - _setup_vms "$VMBACKUP" "$VMS" - fi -} - -# Initialize various variables. -MOUNT="/mnt/$MOUNTOS/" -VMS="$BASEDIR/vms/" -VMBACKUP="$BASEDIR/vmbackup/" -VMMOUNT="/home/cuckoo/vmmount/" - -# In Upstart scripts the $HOME variable may not have been set. If so, set it. -if [ -z "$HOME" ]; then - export HOME="/home/cuckoo" -fi - -# If $LC_ALL is not set then we force it to en_US.UTF-8, otherwise PostgreSQL -# and possibly others will complain about a missing language. -if [ -z "$LC_ALL" ]; then - export LC_ALL="en_US.UTF-8" -fi - -# First of all, setup the machine with all required packages etc -# if asked to do so. (Or, actually, if not asked to not do). -if [ "$VMCHECKUP" -eq 0 ]; then - _setup - - # We store this exact configuration so that we can later on recreate - # the cluster in case that's required. E.g., when VMs break or after - # a reboot when using tmpfs support (which is actually an instance of - # VMs breaking). - cat > /etc/default/cuckoo-setup << EOF -# Direct configuration values. -WIN7="$WIN7" -VMCOUNT="$VMCOUNT" -ISOFILE="$ISOFILE" -SERIALKEY="$SERIALKEY" -TMPFS="$TMPFS" -TAGS="$TAGS" -INTERFACES="$INTERFACES" -DEPENDENCIES="$DEPENDENCIES" -BASEDIR="$BASEDIR" -LONGTERM="$LONGTERM" -CPUCOUNT="$CPUCOUNT" - -# Values set based on configuration values. -EGGNAME="$EGGNAME" -MOUNTOS="$MOUNTOS" -WINOS="$WINOS" -EOF -fi - -# The mount directory must exist and it must not be empty. -if [ ! -d "$MOUNT" ] || [ -z "$(ls -A "$MOUNT")" ]; then - mkdir -p "$MOUNT" - mount -o loop,ro "$ISOFILE" "$MOUNT" -fi - -# Remove the vms directory as we don't want broken remainders -# in there later on such as symbolic links. -rm -rf "$VMS" - -mkdir -p "$VMS" "$VMBACKUP" "$VMMOUNT" -chown cuckoo:cuckoo "$VMS" "$VMBACKUP" "$VMMOUNT" - -# In the case of tmpfs support we first setup the vmmount and all that so to -# recover any VMs that were not actually broken, but were just missing -# symbolic links and some files copied over. At this point, however, we do not -# setup the entire tmpfs thing yet, though, as that would just require a lot -# of file copying even though we might have to do this right again after -# creating new VMs. -_initialize_from_backup 0 - -# We then create new VMs if required. -_create_virtual_machines - -# Initialize all VMs. -_initialize_from_backup "$TMPFS" diff --git a/utils/smtp_sinkhole.py b/utils/smtp_sinkhole.py deleted file mode 100755 index aaa5fbfa4..000000000 --- a/utils/smtp_sinkhole.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2014-2016 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import asyncore -import argparse -import os - -from datetime import datetime -from smtpd import SMTPServer - -class SmtpSink(SMTPServer): - """SMTP Sinkhole server.""" - - # Where mails should be saved. - mail_dir = None - - def process_message(self, peer, mailfrom, rcpttos, data): - """Custom mail processing used to save mails to disk.""" - # Save message to disk only if path is passed. - if self.mail_dir: - file_name = "%s" % datetime.now().strftime("%Y%m%d%H%M%S") - - # Duplicate check. - i = 0 - while os.path.exists(os.path.join(self.mail_dir, file_name + str(i))): - i += 1 - - file_name = file_name + str(i) - with open(os.path.join(self.mail_dir, file_name), "w") as mail: - mail.write(data) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(prog="smtp_sinkhole.py", - usage="%(prog)s [host [port]]", - description="SMTP Sinkhole") - parser.add_argument("host", nargs="?", default="127.0.0.1") - parser.add_argument("port", nargs="?", type=int, default=1025) - parser.add_argument("--dir", default=None, - help="Directory used to dump emails.") - - args = parser.parse_args() - - s = SmtpSink((args.host, args.port), None) - s.mail_dir = args.dir - - try: - asyncore.loop() - except KeyboardInterrupt: - pass diff --git a/utils/start-distributed.sh b/utils/start-distributed.sh deleted file mode 100755 index e8715ed24..000000000 --- a/utils/start-distributed.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -DISTADDR="127.0.0.1" -. /etc/default/cuckoo - -sudo service uwsgi start cuckoo-distributed -sudo service nginx start - -sudo start cuckoo-distributed-instance INSTANCE=dist.status -sudo start cuckoo-distributed-instance INSTANCE=dist.scheduler - -for worker in $(curl -s "$DISTADDR:9003/api/node?mode=workers"); do - sudo start cuckoo-distributed-instance "INSTANCE=$worker" -done diff --git a/utils/stats.py b/utils/stats.py deleted file mode 100755 index 9b01edc9f..000000000 --- a/utils/stats.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2014-2016 Cuckoo Foundation. -# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org -# See the file 'docs/LICENSE' for copying permission. - -import os.path -import sys -import time - -sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "..")) - -from cuckoo.core.database import Database, TASK_PENDING, TASK_RUNNING -from cuckoo.core.database import TASK_COMPLETED, TASK_RECOVERED, TASK_REPORTED -from cuckoo.core.database import TASK_FAILED_ANALYSIS, TASK_FAILED_PROCESSING, TASK_FAILED_REPORTING - -def timestamp(dt): - """Returns the timestamp of a datetime object.""" - return time.mktime(dt.timetuple()) - -def main(): - db = Database() - - print("%d samples in db" % db.count_samples()) - print("%d tasks in db" % db.count_tasks()) - - states = ( - TASK_PENDING, TASK_RUNNING, - TASK_COMPLETED, TASK_RECOVERED, TASK_REPORTED, - TASK_FAILED_ANALYSIS, TASK_FAILED_PROCESSING, TASK_FAILED_REPORTING - ) - - for state in states: - print("%s %d tasks" % (state, db.count_tasks(state))) - - # Later on we might be interested in only calculating stats for all - # tasks starting at a certain offset, because the Cuckoo daemon may - # have been restarted at some point in time. - offset = None - - # For the following stats we're only interested in completed tasks. - tasks = db.list_tasks(offset=offset, status=TASK_COMPLETED) - tasks += db.list_tasks(offset=offset, status=TASK_REPORTED) - - if tasks: - # Get the time when the first task started. - started = min(timestamp(task.started_on) for task in tasks) - - # Get the time when the last task completed. - stamps = [] - for task in tasks: - try: - stamps.append(timestamp(task.completed_on)) - except AttributeError: - pass - - completed = max(stamps) - - # Get the amount of tasks that actually completed. - finished = len(tasks) - - hourly = 60 * 60 * finished / (completed - started) - - print("roughly %d tasks an hour" % int(hourly)) - print("roughly %d tasks a day" % int(24 * hourly)) - -if __name__ == "__main__": - main() diff --git a/utils/stop-distributed.sh b/utils/stop-distributed.sh deleted file mode 100755 index 4a89b95d8..000000000 --- a/utils/stop-distributed.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -DISTADDR="127.0.0.1" -. /etc/default/cuckoo - -for worker in $(curl -s "$DISTADDR:9003/api/node?mode=workers"); do - sudo stop cuckoo-distributed-instance "INSTANCE=$worker" -done - -sudo stop cuckoo-distributed-instance INSTANCE=dist.status -sudo stop cuckoo-distributed-instance INSTANCE=dist.scheduler - -sudo service uwsgi stop cuckoo-distributed -sudo service nginx stop