Re-org'd project, build-system, and added tests

- "--heatmap" flag under prediction

.gitignore

@@ -113,3 +113,4 @@ venv.bak/
 *.model
 *.csv
 *.txt
+!requirements.txt

Makefile

@@ -0,0 +1,95 @@
+###############################################################################
+# Parts of this are borrowed from the HTTPie Makefile. Thanks!
+###############################################################################
+
+REQUIREMENTS="requirements-dev.txt"
+TAG="\n\n\033[0;32m\#\#\# "
+END=" \#\#\# \033[0m\n"
+
+PIP=$(shell which pip3)
+CONDA=$(shell which conda)
+CONDA_HOME="$(HOME)/anaconda3"
+VENV="eyeballer_venv"
+
+SHELL := /bin/bash
+.ONE_SHELL:
+
+all: test
+
+# Target-specific assignments
+conda: PIP := "$(CONDA_HOME)/envs/$(VENV)/bin/pip"
+conda init: uninstall-eyeballer
+
+ifeq ($(strip $(MAKECMDGOALS)),conda)
+	@echo -e $(TAG)Conda detected at: $(CONDA)$(END)
+	conda create -yn $(VENV) | true
+
+	source $(CONDA_HOME)/etc/profile.d/conda.sh; \
+	conda activate $(VENV); \
+	conda install -y pip
+endif
+	@echo -e $(TAG)Installing dev requirements$(END)
+	$(PIP) install --upgrade -r $(REQUIREMENTS)
+
+	@echo -e $(TAG)Installing Eyeballer$(END)
+	$(PIP) install --upgrade --editable .
+
+ifeq ($(strip $(MAKECMDGOALS)),conda)
+	@echo -e $(TAG)Activate virtualenv to run eyeballer: $(END)
+	@echo conda activate $(VENV)
+endif
+
+	@echo
+
+clean:
+	@echo -e $(TAG)Cleaning up$(END)
+	rm -rf *.egg dist build .cache .pytest_cache eyeballer.egg-info
+	find . -name '__pycache__' -delete -print -o -name '*.pyc' -delete -print
+ifneq ($(strip $(CONDA)),)
+	conda env remove -y --name eyeballer_venv | true
+endif
+	@echo
+
+
+###############################################################################
+# Testing
+###############################################################################
+
+
+test:
+	@echo -e $(TAG)Running tests on the current Python interpreter with coverage $(END)
+	python -m unittest tests
+	@echo
+
+
+# test-all is meant to test everything — even this Makefile
+test-all: uninstall-all clean init test pycodestyle
+	@echo
+
+
+pycodestyle:
+	which pycodestyle || $(PIP) install pycodestyle
+	pycodestyle
+	@echo
+
+
+###############################################################################
+# Uninstalling
+###############################################################################
+
+uninstall-eyeballer:
+	@echo using pip3 located at: $(PIP)
+	@echo -e $(TAG)Uninstalling eyeballer$(END)
+	- $(PIP) uninstall --yes eyeballer &2>/dev/null
+
+	@echo "Verifying…"
+	cd .. && ! python -m eyeballer --version &2>/dev/null
+
+	@echo "Done"
+	@echo
+
+
+uninstall-all: uninstall-eyeballer
+
+	@echo -e $(TAG)Uninstalling development requirements$(END)
+	- $(PIP) uninstall --yes -r $(REQUIREMENTS)

README.md

@@ -0,0 +1,56 @@
+# Eyeballer
+Give those screenshots of yours a quick eyeballing.
+
+Eyeballer is meant for large-scope network penetration tests where you need to find "interesting" targets from a huge set of web-based hosts. Go ahead and use your favorite screenshotting tool like normal (EyeWitness or GoWitness) and then run them through Eyeballer to tell you what's likely to contain vulnerabilities, and what isn't.
+
+## Installation
+
+To install in a python venv (Recommended):
+```
+python3 -m venv venv
+source venv/bin/activate
+make init
+```
+
+To install for Conda (automatically creates conda venv):
+
+```
+make conda
+```
+
+## Running Tests
+
+```
+make test
+```
+
+## Predicting Labels
+To eyeball some screenshots, just run the "predict" mode:
+
+```
+eyeballer --weights YOUR_WEIGHTS.h5 predict YOUR_FILE.png
+```
+
+Or for a whole directory of files:
+
+```
+eyeballer --weights YOUR_WEIGHTS.h5 predict PATH_TO/YOUR_FILES/
+```
+
+## Training
+To train a new model, run:
+```
+eyeballer train
+```
+
+You'll want a machine with a good GPU for this to run in a reasonable amount of time. Setting that up is outside the scope of this readme, however.
+
+This will output a new model file (weights.h5 by default).
+
+## Evaluation
+
+You just trained a new model, cool! Let's see how well it performs against some images it's never seen before, across a variety of metrics:
+
+```
+eyeballer --weights YOUR_WEIGHTS.h5 evaluate
+```

eyeballer/__init__.py

@@ -0,0 +1 @@
+from eyeballer.model import EyeballModel  # noqa: F401

eyeballer/augmentation.py

@@ -0,0 +1,18 @@
+import numpy as np
+
+from Augmentor.Operations import Operation
+from keras.applications.mobilenet import preprocess_input
+
+
+class EyeballerAugmentation(Operation):
+    def __init__(self, probability=1):
+        Operation.__init__(self, probability)
+
+    # Class must implement the perform_operation method
+    def perform_operation(self, images):
+        return_list = []
+        for image in images:
+            image_array = np.array(image).astype('uint8')
+            image_array = preprocess_input(image_array)
+            return_list.append(image_array)
+        return return_list

eyeballer/eyeballer.py

@@ -1,276 +1,83 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
-import argparse
-import os
-import random
-import time
+import click
+import csv
 
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-
-from keras import regularizers
-from keras.applications import MobileNet
-from keras.applications.mobilenet import preprocess_input
-from keras.callbacks import ModelCheckpoint
-from keras.layers import Dropout, Dense
-from keras.layers import GlobalAveragePooling2D
-from keras.models import Model
-from keras.optimizers import Adam
-from keras.preprocessing import image
-from keras.preprocessing.image import ImageDataGenerator
-from sklearn.metrics import precision_score, recall_score
-
-# Parse the arguments
-parser = argparse.ArgumentParser(description='Give those screenshots of yours a quick eyeballing')
-parser.add_argument("mode", help="Mode of operation. One of \"train\", \"predict\", or \"evaluate\".")
-parser.add_argument("--weights", help="Weights file for input/output")
-parser.add_argument("--batchsize", help="Batch size", default=32, type=int)
-parser.add_argument("--epochs", help="Number of epochs", default=20, type=int)
-parser.add_argument("--screenshot", help="File to predict")
-parser.add_argument("--summary", help="Print model summary at start", action='store_true')
-parser.add_argument("--graphs", help="Save accuracy and loss graphs to file", action='store_true')
-parser.add_argument("--seed", help="RNG seed for data shuffling and transformations", default=0, type=int)
-
-args = parser.parse_args()
+from eyeballer.model import EyeballModel, DATA_LABELS
+from eyeballer.visualization import HeatMap
 
 
-class EyeballModel:
-    checkpoint_file = "weights-snapshot.h5"
-    image_dir = "images_224x224/"
-    image_width, image_height = 224, 224
-    input_shape = (image_width, image_height, 3)
+@click.group(invoke_without_command=True)
+@click.option('--weights', default=None, type=click.Path(), help="Weights file for input/output")
+@click.option('--summary/--no-summary', default=False, help="Print model summary at start")
+@click.option('--seed', default=None, type=int, help="RNG seed for data shuffling and transformations, defaults to random value")
+@click.pass_context
+def cli(ctx, weights, summary, seed):
+    model_kwargs = {"weights_file": weights,
+                    "print_summary": summary,
+                    "seed": seed}
 
-    def __init__(self, print_summary=False, weights_file="weights.h5", seed=0):
-        # Build the model
-        base_model = MobileNet(weights='imagenet', include_top=False, input_shape=self.input_shape)
-        x = base_model.output
-        x = GlobalAveragePooling2D()(x)
-        x = Dense(256, activation="relu", name="HiddenLayer1")(x)
-        x = Dropout(0.5)(x)
-        x = Dense(128, activation="relu", name="HiddenLayer2", kernel_regularizer=regularizers.l2(0.01))(x)
-        x = Dropout(0.2)(x)
-        output_layer = Dense(3, activation="sigmoid", name="OutputLayer")(x)
-        self.model = Model(inputs=base_model.input, outputs=output_layer)
-
-        # for layer in base_model.layers:
-        #    layer.trainable = False
-
-        adam = Adam(lr=0.0005)
-        self.model.compile(optimizer=adam, loss="binary_crossentropy", metrics=["accuracy"])
-
-        if print_summary:
-            print(self.model.summary())
-
-        # Pull out our labels for use in generators later
-        data = pd.read_csv("labels.csv")
-        self.training_labels = data.loc[data['evaluation'] == False]
-        self.evaluation_labels = data.loc[data['evaluation'] == True]
-
-        # Shuffle the training labels
-        self.seed = seed
-        print("Using seed: " + str(seed))
-        random.seed(self.seed)
-        self.training_labels = self.training_labels.sample(frac=1)
-
-        if weights_file is not None and os.path.isfile(weights_file):
-            self.model.load_weights(weights_file)
-            print("Loaded model from file.")
-        else:
-            print("No model loaded from file")
-
-    # Training
-    def train(self, epochs=20, batch_size=32, print_graphs=False):
-        print("Training with seed: " + str(self.seed))
-
-        # Data augmentation
-        data_generator = ImageDataGenerator(
-            preprocessing_function=preprocess_input,
-            shear_range=0.0,
-            zoom_range=0.2,
-            samplewise_center=True,
-            validation_split=0.2,
-            horizontal_flip=False)
-
-        training_generator = data_generator.flow_from_dataframe(
-            self.training_labels,
-            directory=self.image_dir,
-            x_col="filename",
-            y_col=["custom404", "login", "homepage"],
-            target_size=(self.image_width, self.image_height),
-            batch_size=batch_size,
-            subset='training',
-            shuffle=True,
-            seed=self.seed,
-            class_mode="other")
-        validation_generator = data_generator.flow_from_dataframe(
-            self.training_labels,
-            directory=self.image_dir,
-            x_col="filename",
-            y_col=["custom404", "login", "homepage"],
-            target_size=(self.image_width, self.image_height),
-            batch_size=batch_size,
-            subset='validation',
-            shuffle=False,
-            seed=self.seed,
-            class_mode="other")
-
-        # Model checkpoint - Saves model weights when validation accuracy improves
-        callbacks = [ModelCheckpoint(self.checkpoint_file, monitor='val_loss', verbose=1, save_best_only=True,
-                                     save_weights_only=True, mode='min')]
-
-        history = self.model.fit_generator(
-            training_generator,
-            steps_per_epoch=len(training_generator.filenames) // batch_size,
-            epochs=epochs,
-            validation_data=validation_generator,
-            validation_steps=len(validation_generator.filenames) // batch_size,
-            callbacks=callbacks,
-            verbose=1)
-
-        if print_graphs:
-            # Plot training & validation accuracy values
-            plt.plot(history.history['acc'])
-            plt.plot(history.history['val_acc'])
-            plt.title('Model accuracy')
-            plt.ylabel('Accuracy')
-            plt.xlabel('Epoch')
-            plt.legend(['Train', 'Validation'], loc='upper left')
-            plt.savefig("accuracy.png")
-            plt.clf()
-            plt.cla()
-            plt.close()
-
-            # Plot training & validation loss values
-            plt.plot(history.history['loss'])
-            plt.plot(history.history['val_loss'])
-            plt.title('Model loss')
-            plt.ylabel('Loss')
-            plt.xlabel('Epoch')
-            plt.legend(['Train', 'Validation'], loc='upper left')
-            plt.savefig("loss.png")
-
-    # Single predict on a file
-    def predict(self, filename):
-        # Load the image into memory
-        img = image.load_img(filename, target_size=(self.image_width, self.image_height))
-        img = image.img_to_array(img)
-        img = np.expand_dims(img, axis=0)
-        img = preprocess_input(img)
-
-        time_start = time.time()
-        prediction = self.model.predict(img, batch_size=1)
-        time_end = time.time()
-        print("Predictions:")
-        print("\tCustom 404: " + str(round(prediction[0][0] * 100, 2)) + "%")
-        print("\tLogin Page: " + str(round(prediction[0][1] * 100, 2)) + "%")
-        print("\tHome Page: " + str(round(prediction[0][2] * 100, 2)) + "%")
-        print("Prediction Took (seconds): ", time_end - time_start)
-
-    # Evaluate performance against the persistent evaluation data set
-    def evaluate(self, threshold=0.5):
-        # Prepare the data
-        data_generator = ImageDataGenerator(
-            preprocessing_function=preprocess_input)
-        evaluation_generator = data_generator.flow_from_dataframe(
-            self.evaluation_labels,
-            directory=self.image_dir,
-            x_col="filename",
-            y_col=["custom404", "login", "homepage"],
-            target_size=(self.image_width, self.image_height),
-            shuffle=False,
-            batch_size=1,
-            class_mode="other")
-        # If a seed was selected, then also evaluate on the validation set for that seed
-        if args.seed > 0:
-            print("Using validation set...")
-            # Data augmentation
-            data_generator = ImageDataGenerator(
-                preprocessing_function=preprocess_input,
-                shear_range=0.0,
-                zoom_range=0.2,
-                samplewise_center=True,
-                validation_split=0.2,
-                horizontal_flip=False)
-            evaluation_generator = data_generator.flow_from_dataframe(
-                self.training_labels,
-                directory=self.image_dir,
-                x_col="filename",
-                y_col=["custom404", "login", "homepage"],
-                target_size=(self.image_width, self.image_height),
-                batch_size=1,
-                subset='validation',
-                shuffle=False,
-                seed=self.seed,
-                class_mode="other")
-        else:
-            print("Using evaluation set...")
-
-        predictions = self.model.predict_generator(evaluation_generator, verbose=1, steps=len(evaluation_generator))
-        predictions = predictions > threshold
-
-        all_or_nothing_success = 0
-
-        correct_per_category_counts = [0, 0, 0]
-        total_correct_count = 0
-
-        labels = evaluation_generator.data.tolist()
-
-        # Zip them together, so we can evaluate easier
-        for row in zip(labels, predictions.tolist()):
-            # All or nothing count
-            if row[0] == row[1]:
-                all_or_nothing_success += 1
-
-            # Per-category count
-            for i in range(len(row[0])):
-                if row[0][i] == row[1][i]:
-                    correct_per_category_counts[i] += 1
-                    total_correct_count += 1
-
-        custom404_labels = np.reshape(np.array(labels)[:, 0:1], len(labels)).tolist()
-        custom404_predicitons = np.reshape(np.array(predictions)[:, 0:1], len(predictions)).tolist()
-
-        login_labels = np.reshape(np.array(labels)[:, 1:2], len(labels)).tolist()
-        login_predicitons = np.reshape(np.array(predictions)[:, 1:2], len(predictions)).tolist()
-
-        homepage_labels = np.reshape(np.array(labels)[:, 2:3], len(labels)).tolist()
-        homepage_predicitons = np.reshape(np.array(predictions)[:, 2:3], len(predictions)).tolist()
-
-        custom404_precision = precision_score(custom404_labels, custom404_predicitons)
-        custom404_recall = recall_score(custom404_labels, custom404_predicitons)
-        print("Custom404 Precision Score: " + str(round(custom404_precision * 100, 2)) + "%")
-        print("Custom404 Recall Score: " + str(round(custom404_recall * 100, 2)) + "%")
-
-        login_precision = precision_score(login_labels, login_predicitons)
-        login_recall = recall_score(login_labels, login_predicitons)
-        print("Login Precision Score: " + str(round(login_precision * 100, 2)) + "%")
-        print("Login Recall Score: " + str(round(login_recall * 100, 2)) + "%")
-
-        homepage_precision = precision_score(homepage_labels, homepage_predicitons)
-        homepage_recall = recall_score(homepage_labels, homepage_predicitons)
-        print("Homepage Precision Score: " + str(round(homepage_precision * 100, 2)) + "%")
-        print("Homepage Recall Score: " + str(round(homepage_recall * 100, 2)) + "%")
-
-        print("Overall Binary Accuracy: " + str(
-            round((total_correct_count * 100) / (len(evaluation_generator) * 3), 2)) + "%")
-        print("All or nothing accuracy: " + str(
-            round((all_or_nothing_success / len(evaluation_generator)) * 100, 2)) + "%")
+    #  pass the model to subcommands
+    ctx.ensure_object(dict)
+    # We only pass the kwargs so we can be lazy and make the model later after the subcommand cli is parsed. This
+    # way, the user doesn't have to wait for tensorflow if they are just calling --help on a subcommand.
+    ctx.obj['model_kwargs'] = model_kwargs
 
 
-seed = args.seed
-if args.seed == 0:
-    seed = random.randint(0, 999999)
+@cli.command()
+@click.option('--graphs/--no-graphs', default=False, help="Save accuracy and loss graphs to file")
+@click.option('--epochs', default=20, type=int, help="Number of epochs")  # TODO better help string
+@click.option('--batchsize', default=32, type=int, help="Batch size")  # TODO better help string
+@click.pass_context
+def train(ctx, graphs, batchsize, epochs):
+    model = EyeballModel(**ctx.obj['model_kwargs'])
+    model.train(print_graphs=graphs, batch_size=batchsize, epochs=epochs)
 
-model = EyeballModel(weights_file=args.weights, print_summary=args.summary, seed=seed)
-if args.mode == "train":
-    model.train(print_graphs=args.graphs, batch_size=args.batchsize, epochs=args.epochs)
-elif args.mode == "predict":
-    if args.screenshot is None:
-        print("Error: Use the --screenshot flag to specify what file you want to predict on")
+
+@cli.command()
+@click.argument('screenshot')
+@click.option('--heatmap', default=False, is_flag=True, help="Create a heatmap graphfor the prediction")
+@click.pass_context
+def predict(ctx, screenshot, heatmap):
+    model = EyeballModel(**ctx.obj['model_kwargs'])
+    results = model.predict(screenshot)
+
+    if heatmap:
+        # Generate a heatmap
+        HeatMap(screenshot, model).generate()
+
+    if not results:
+        print("Error: Input file does not exist")
+    if len(results) == 1:
+        print(results)
     else:
-        model.predict(args.screenshot)
-elif args.mode == "evaluate":
-    model.evaluate(threshold=0.5)
-else:
-    print("Error: " + args.mode + " is not a valid mode. Try one of \"train\", \"predict\", or \"evaluate\"")
+        with open("results.csv", "w", newline="") as csvfile:
+            fieldnames = ["filename", "custom404", "login", "homepage", "oldlooking"]
+            labelwriter = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            labelwriter.writeheader()
+            labelwriter.writerows(results)
+        print("Output written to results.csv")
+
+
+def pretty_print_evaluation(results):
+    """Print a human-readable summary of the evaluation"""
+    # We use 4.2% to handle all the way from "  0.00%" (7chars) to "100.00%" (7chars)
+    for label in DATA_LABELS:
+        print(f"{label} Precision Score: {results[label]['precision']:4.2%}")
+        print(f"{label} Recall Score: {results[label]['recall']:4.2%}")
+    print(f"Overall Binary Accuracy: {results['total_binary_accuracy']:4.2%}")
+    print(f"All or nothing accuracy: {results['all_or_nothing_accuracy']:4.2%}")
+    print(f"Top 10 worst predictions: {results['top_10_worst'][1]}")
+
+
+@cli.command()
+@click.option('--threshold', default=.5, type=float, help="Threshold confidence for labeling")
+@click.pass_context
+def evaluate(ctx, threshold):
+    model = EyeballModel(**ctx.obj['model_kwargs'])
+    results = model.evaluate(threshold)
+    pretty_print_evaluation(results)
+
+
+if __name__ == '__main__':
+    cli()

eyeballer/model.py

@@ -0,0 +1,358 @@
+import os
+import random
+
+# Prevent Tkinter Dependency
+import matplotlib
+matplotlib.use('agg')  # noqa: E402
+import matplotlib.pyplot as plt
+
+import numpy as np
+import pandas as pd
+import Augmentor
+
+from keras import regularizers
+from keras.applications import MobileNet
+from keras.applications.mobilenet import preprocess_input
+from keras.callbacks import ModelCheckpoint
+from keras.layers import Dropout, Dense
+from keras.layers import GlobalAveragePooling2D
+from keras.models import Model
+from keras.optimizers import Adam
+from keras.preprocessing import image
+from keras.preprocessing.image import ImageDataGenerator
+from sklearn.metrics import classification_report, accuracy_score, hamming_loss
+from eyeballer.augmentation import EyeballerAugmentation
+from keras.callbacks import Callback
+
+DATA_LABELS = ["custom404", "login", "homepage", "oldlooking"]
+
+
+class PredictionManager(Callback):
+    def __init__(self):
+        super(PredictionManager, self).__init__()
+        self.predictions = []
+
+    def getPredictions(self):
+        return np.array(self.predictions)
+
+    def on_predict_batch_end(self, batch, logs=None):
+        self.predictions.append(logs['outputs'][0].tolist()[0])
+
+
+class EyeballModel:
+    """The primary model class of Eyeballer.
+
+    Contains high-level functions for training, evaluating, and predicting.
+    """
+    graphs_directory = "graphs/"
+    checkpoint_file = "weights.h5"
+    image_dir = "images_224x224/"
+    image_width, image_height = 224, 224
+    input_shape = (image_width, image_height, 3)
+
+    def __init__(self, print_summary=False, weights_file="weights.h5", seed=None):
+        """Constructor for model class.
+
+        Keyword arguments:
+        print_summary -- Whether or not to print to stdout the keras model summary, containing a detailed description of every model layer
+        weights_file -- A filename for where to load the model's weights.
+        seed -- PRNG seed, useful for repeating a previous run and using the same data. Training/Validation split is determined randomly.
+        """
+        # Build the model
+        base_model = MobileNet(weights='imagenet', include_top=False, input_shape=self.input_shape)
+        x = base_model.output
+        x = GlobalAveragePooling2D()(x)
+        x = Dense(256, activation="relu", name="HiddenLayer1")(x)
+        x = Dropout(0.5)(x)
+        x = Dense(128, activation="relu", name="HiddenLayer2", kernel_regularizer=regularizers.l2(0.01))(x)
+        x = Dropout(0.2)(x)
+        output_layer = Dense(len(DATA_LABELS), activation="sigmoid", name="OutputLayer")(x)
+        self.model = Model(inputs=base_model.input, outputs=output_layer)
+
+        # for layer in base_model.layers:
+        #     layer.trainable = False
+
+        adam = Adam(lr=0.0005)
+        self.model.compile(optimizer=adam, loss="binary_crossentropy", metrics=["accuracy"])
+
+        if print_summary:
+            print(self.model.summary())
+
+        # Pull out our labels for use in generators later
+        data = pd.read_csv("labels.csv")
+        self.training_labels = data.loc[data['evaluation'] == False]  # noqa: E712
+        self.evaluation_labels = data.loc[data['evaluation'] == True]  # noqa: E712
+
+        # Shuffle the training labels
+        self.random_seed = False
+        self.seed = seed
+        if self.seed is None:
+            self.random_seed = True
+            self.seed = random.randint(0, 999999)
+            print("No seed set, ", end='')
+        print(f"using seed: {self.seed}")
+        random.seed(self.seed)
+        self.training_labels = self.training_labels.sample(frac=1)
+
+        if weights_file is not None and os.path.isfile(weights_file):
+            self.model.load_weights(weights_file)
+            print("Loaded model from file.")
+        else:
+            print("No model loaded from file")
+
+        # Data augmentation
+        augmentor = Augmentor.Pipeline()
+        augmentor.set_seed(self.seed)
+        augmentor.zoom(probability=0.75, min_factor=0.8, max_factor=1.2)
+        augmentor.random_color(probability=0.75, min_factor=0.5, max_factor=1.0)
+        augmentor.random_contrast(probability=0.75, min_factor=0.8, max_factor=1.0)
+        augmentor.random_brightness(probability=0.75, min_factor=0.8, max_factor=1.2)
+        augmentor.random_erasing(probability=0.75, rectangle_area=0.15)
+
+        # Finalizes the augmentation with a custom operation to prepare the image for the specific pretrained model we're using
+        training_augmentation = EyeballerAugmentation()
+        augmentor.add_operation(training_augmentation)
+        self.preprocess_training_function = augmentor.keras_preprocess_func()
+
+    def train(self, epochs=20, batch_size=32, print_graphs=False):
+        """Train the model, making a new weights file at each successfull checkpoint. You'll probably need a GPU for this to realistically run.
+
+        Keyword arguments:
+        epochs -- The number of epochs to train for. (An epoch is one pass-through of the dataset)
+        batch_size -- How many images to batch together when training. Generally speaking, the higher the better, until you run out of memory.
+        print_graphs --- Whether or not to create accuracy and loss graphs. If true, they'll be written to accuracy.png and loss.png
+        """
+        print("Training with seed: " + str(self.seed))
+
+        data_generator = ImageDataGenerator(
+            preprocessing_function=self.preprocess_training_function,
+            validation_split=0.2,
+            samplewise_center=True)
+
+        training_generator = data_generator.flow_from_dataframe(
+            self.training_labels,
+            directory=self.image_dir,
+            x_col="filename",
+            y_col=DATA_LABELS,
+            target_size=(self.image_width, self.image_height),
+            batch_size=batch_size,
+            subset='training',
+            shuffle=True,
+            seed=self.seed,
+            class_mode="other")
+        validation_generator = data_generator.flow_from_dataframe(
+            self.training_labels,
+            directory=self.image_dir,
+            x_col="filename",
+            y_col=DATA_LABELS,
+            target_size=(self.image_width, self.image_height),
+            batch_size=batch_size,
+            subset='validation',
+            shuffle=False,
+            seed=self.seed,
+            class_mode="other")
+
+        # Model checkpoint - Saves model weights when validation accuracy improves
+        callbacks = [ModelCheckpoint(self.checkpoint_file, monitor='val_loss', verbose=1, save_best_only=True,
+                                     save_weights_only=True, mode='min')]
+
+        history = self.model.fit_generator(
+            training_generator,
+            steps_per_epoch=len(training_generator.filenames) // batch_size,
+            epochs=epochs,
+            validation_data=validation_generator,
+            validation_steps=len(validation_generator.filenames) // batch_size,
+            callbacks=callbacks,
+            verbose=1)
+
+        if print_graphs:
+            if not os.path.exists(self.graphs_directory):
+                os.makedirs(self.graphs_directory)
+            # Plot training & validation accuracy values
+            plt.plot(history.history['acc'])
+            plt.plot(history.history['val_acc'])
+            plt.title('Model accuracy')
+            plt.ylabel('Accuracy')
+            plt.xlabel('Epoch')
+            plt.legend(['Train', 'Validation'], loc='upper left')
+            plt.savefig(self.graphs_directory + "accuracy.png")
+            plt.clf()
+            plt.cla()
+            plt.close()
+
+            # Plot training & validation loss values
+            plt.plot(history.history['loss'])
+            plt.plot(history.history['val_loss'])
+            plt.title('Model loss')
+            plt.ylabel('Loss')
+            plt.xlabel('Epoch')
+            plt.legend(['Train', 'Validation'], loc='upper left')
+            plt.savefig(self.graphs_directory + "loss.png")
+
+    def predict_on_array(self, image):
+        """Predict the labels for a single screenshot
+
+        Keyword arguments:
+        image -- The numpy array of the image to classify
+        """
+        img = np.expand_dims(image, axis=0)
+        img = preprocess_input(img)
+
+        prediction = self.model.predict(img, batch_size=1)
+        result = dict()
+        result["filename"] = "custom-image"
+        result["custom404"] = prediction[0][0]
+        result["login"] = prediction[0][1]
+        result["homepage"] = prediction[0][2]
+        result["oldlooking"] = prediction[0][3]
+        return result
+
+    def predict(self, path):
+        """Predict the labels for a single file
+
+        Keyword arguments:
+        path -- The path to the file that we'll be evaluating
+        """
+        # Is this a single file, or a directory?
+        screenshots = []
+        if os.path.isfile(path):
+            screenshots = [path]
+        elif os.path.isdir(path):
+            screenshots = os.listdir(path)
+            screenshots = [os.path.join(path, s) for s in screenshots]
+        else:
+            raise FileNotFoundError
+
+        results = []
+        for screenshot in screenshots:
+            # Load the image into memory
+            img = image.load_img(screenshot, target_size=(self.image_width, self.image_height))
+            img = image.img_to_array(img)
+            img = np.expand_dims(img, axis=0)
+            img = preprocess_input(img)
+
+            prediction = self.model.predict(img, batch_size=1)
+            result = dict()
+            result["filename"] = screenshot
+            result["custom404"] = prediction[0][0]
+            result["login"] = prediction[0][1]
+            result["homepage"] = prediction[0][2]
+            result["oldlooking"] = prediction[0][3]
+            results.append(result)
+
+        return results
+
+    def evaluate(self, threshold=0.5):
+        """Evaluate performance against the persistent evaluation data set
+
+        Keyword arguments:
+        threshold -- Value between 0->1. The cutoff where the numerical prediction becomes boolean. Default: 0.5
+        """
+        # Prepare the data
+        data_generator = ImageDataGenerator(
+            preprocessing_function=preprocess_input)
+        evaluation_generator = data_generator.flow_from_dataframe(
+            self.evaluation_labels,
+            directory=self.image_dir,
+            x_col="filename",
+            y_col=DATA_LABELS,
+            target_size=(self.image_width, self.image_height),
+            shuffle=False,
+            batch_size=1,
+            class_mode="other")
+        # If a seed was selected, then also evaluate on the validation set for that seed
+        if not self.random_seed:
+            print("Using validation set...")
+            # Data augmentation
+            data_generator = ImageDataGenerator(
+                preprocessing_function=self.preprocess_training_function,
+                samplewise_center=True,
+                validation_split=0.2)
+            evaluation_generator = data_generator.flow_from_dataframe(
+                self.training_labels,
+                directory=self.image_dir,
+                x_col="filename",
+                y_col=DATA_LABELS,
+                target_size=(self.image_width, self.image_height),
+                batch_size=1,
+                subset='validation',
+                shuffle=False,
+                seed=self.seed,
+                class_mode="other")
+        else:
+            print("Using evaluation set...")
+
+        prediction_manager = PredictionManager()
+
+        try:
+            self.model.predict_generator(
+                evaluation_generator,
+                callbacks=[prediction_manager],
+                verbose=1,
+                steps=len(evaluation_generator))
+        except KeyboardInterrupt:
+            print('\r\nJob interrupted (Ctrl+C): Outputting completed results')
+
+        predictions = prediction_manager.getPredictions()
+
+        # Truncates evaluation generator if test was interrupted
+        evaluation_generator._data = evaluation_generator.data[:predictions.shape[0]]
+
+        self._save_prediction_histograms(predictions)
+        predictions = predictions > threshold
+        ground_truth = evaluation_generator.data
+        stats = classification_report(ground_truth, predictions, target_names=DATA_LABELS, output_dict=True)
+        stats["total_binary_accuracy"] = 1 - hamming_loss(ground_truth, predictions)
+        stats["all_or_nothing_accuracy"] = accuracy_score(ground_truth, predictions)
+        stats["top_10_best"] = self._top_images(evaluation_generator, predictions, best=True)
+        stats["top_10_worst"] = self._top_images(evaluation_generator, predictions, best=False)
+        return stats
+
+    def _top_images(self, labels, predictions, top_k=10, best=False):
+        """Collect top-k best or top-k worst predicted images
+
+        Keyword arguments:
+        labels -- The keras generator that contains the filenames and data labels
+        predictions -- The numpy array of predictions
+        top_k -- Top k elements
+        best -- True/False. Calculate either the best or worst images
+
+        :Return -- Tuple of top-k indicies and top-k filenames
+        """
+        true_labels = np.array(labels.data).astype(float)
+        predictions = predictions.astype(float)
+
+        differences = np.abs(predictions - true_labels).sum(axis=1)
+        indicies = np.argsort(differences, axis=0)
+
+        top_file_list = []
+
+        if not best:
+            # Reverse numpy array
+            indicies = np.flipud(indicies)
+
+        for i in indicies[:top_k]:
+            top_file_list.append(labels.filenames[i])
+
+        return indicies[:top_k], top_file_list
+
+    def _save_prediction_histograms(self, predictions, buckets=50):
+        """Saves a series of histogram screenshots
+
+            Keyword arguments:
+            predictions -- The numpy array of predicted labels, ranging from 0->1.
+            buckets -- The number of buckets to divide the data into. Default: 50
+            :Returns -- Nothing"
+        """
+        figure, axes = plt.subplots(nrows=len(DATA_LABELS))
+        for i, label in enumerate(DATA_LABELS):
+            axes[i].hist(predictions[:, i], buckets, alpha=.75)
+            axes[i].set_xlabel("Prediction")
+            axes[i].set_ylabel("Counts of Predictions")
+            axes[i].set_title(label)
+            axes[i].grid(True)
+        figure.set_size_inches(5, 3*len(DATA_LABELS))
+        figure.tight_layout()
+        if not os.path.exists(self.graphs_directory):
+            os.makedirs(self.graphs_directory)
+        plt.savefig(self.graphs_directory + "label_histograms.png")

eyeballer/visualization.py

@@ -0,0 +1,127 @@
+import numpy as np
+import math
+from keras.preprocessing import image
+from copy import deepcopy
+from matplotlib import pyplot as plt
+from eyeballer.model import DATA_LABELS
+
+
+class HeatMap():
+    def __init__(self, screenshot_path, model, boxsize=28, step=7):
+        self.model = model
+
+        img = image.load_img(screenshot_path, target_size=(self.model.image_width, self.model.image_height))
+        img = image.img_to_array(img)
+
+        self.screenshot = img
+        self.boxsize = boxsize
+        self.step = step
+        self.x = 0
+        self.y = 0
+        self.screenshot_path = screenshot_path
+        self.gamma = 3
+
+    def generate(self, output_file="heatmap.png"):
+        """ Make a single heatmap image and return it """
+        heatmaps = []
+        labels = []
+        results = self.model.predict(self.screenshot_path)
+        results = results[0]
+        for label in DATA_LABELS:
+            boxsize = self.boxsize
+            # Ignore this label if it didn't predict positively (true label)
+            if results[label] > 0.5:
+                worst_score = 1
+                while worst_score > 0.5:
+                    heatmap, worst_score = self._get_heatmap(label, boxsize)
+                    if worst_score > 0.5:
+                        boxsize += 28
+                        print(f"Didn't get a good image for {label}. Trying again with a bigger boxsize: {boxsize}")
+                    else:
+                        heatmaps.append(heatmap)
+                        labels.append(label)
+
+        plt.figure()
+        screenshot_image = image.load_img(self.screenshot_path, target_size=(self.model.image_width, self.model.image_width))
+        _, subplots = plt.subplots(1, len(heatmaps))
+        # This is a little janky, but matplotlib returns a list above if there's multiple subplots, and just a single subplot if there's only one
+        #   it would have been easier if it was always a list. But alas
+        if not heatmaps:
+            print("No heatmap made. The image did not have a True classification for any label")
+            return
+        if len(heatmaps) > 1:
+            for i, heatmap in enumerate(heatmaps):
+                subplots[i].imshow(screenshot_image, cmap='binary', interpolation='none')
+                subplots[i].imshow(heatmap, alpha=0.5, interpolation='none')
+                subplots[i].set_title(labels[i])
+        else:
+            subplots.imshow(screenshot_image, cmap='binary', interpolation='none')
+            subplots.imshow(heatmap, alpha=0.5, interpolation='none')
+            subplots.set_title(labels[0])
+        plt.savefig(output_file)
+        print(f"Heatmap image written to: {output_file}")
+
+    def _get_heatmap(self, label, boxsize):
+        worst_score = 1
+        self.x, self.y = 0, 0
+
+        heatmap = np.ones((self.model.image_width, self.model.image_height))
+        heatmap *= 255
+        while True:
+            # Occlude an image
+            occluded_image, x, y = self._occlude(boxsize)
+            if occluded_image is not None:
+                # Score the new occluded image
+                results = self.model.predict_on_array(occluded_image)
+                score = results[label]  # TODO
+                worst_score = min(worst_score, score)
+                # Scale the score up to pixel values
+                score = math.floor(score * 255)
+                # Anneal the scores as you go outward into the occlusion block
+                new_scores = np.ones((boxsize, boxsize))
+                new_scores *= score
+                new_scores = self._gamma_anneal(new_scores)
+
+                # For each pixel, update only where the new score is lower
+                original_scores = heatmap[x: x+boxsize, y: y+boxsize]
+
+                target_area = np.where(new_scores < original_scores, new_scores, original_scores)
+                heatmap[x: x+boxsize, y: y+boxsize] = target_area
+            else:
+                break
+
+        return heatmap, worst_score
+
+    def _gamma_anneal(self, occlusion_area):
+        """ Get the annealed score based on centeredness of the occlusion area """
+        width, height = occlusion_area.shape
+
+        center_x = (width-1) / 2
+        center_y = (height-1) / 2
+
+        annealed_area = np.zeros(occlusion_area.shape)
+
+        for (x, y), score in np.ndenumerate(occlusion_area):
+            distance = math.sqrt((center_x - x)**2 + (center_y - y)**2)
+            annealed_area[x, y] = min(255, score + (self.gamma * distance))
+
+        return annealed_area
+
+    def _occlude(self, boxsize):
+        ''' Return a single occluded image and its location '''
+        if self.x + boxsize > self.screenshot.shape[0]:
+            return None, None, None
+
+        retImg = np.copy(self.screenshot)
+        retImg[self.x:self.x+boxsize, self.y:self.y+boxsize] = 0.0
+
+        old_i = deepcopy(self.x)
+        old_j = deepcopy(self.y)
+
+        # update indices
+        self.y = self.y + self.step
+        if self.y+boxsize > self.screenshot.shape[1]:  # reached end
+            self.y = 0  # reset j
+            self.x = self.x + self.step  # go to next row
+
+        return retImg, old_i, old_j

setup.py

@@ -9,13 +9,22 @@ setup(
     author='Daniel Petro',
     author_email='',
     description='',
+    entry_points={
+        'console_scripts': [
+            'eyeballer = eyeballer.eyeballer:cli'
+        ]
+    },
     install_requires=[
-        'keras',
-        'sklearn',
+        'Augmentor',
+        'click',
+        'matplotlib',
         'numpy',
         'pandas',
-        'matplotlib',
-        'pillow'
+        'pillow',
+        'sklearn',
+        'tensorflow',
+        'keras @ git+https://github.com/keras-team/keras',
+    ],
+    dependency_links=[
     ],
-    test_requires=[],
 )

tests/EvaluateTest.py

@@ -0,0 +1 @@
+#TODO

tests/PredictTest.py

@@ -0,0 +1,83 @@
+import errno
+import os
+import unittest
+
+from eyeballer.model import EyeballModel, DATA_LABELS
+
+class PredictTest(unittest.TestCase):
+    def setUp(self):
+        weights_file = "latest_weights.h5"
+        if not os.path.isfile(weights_file):
+            print("Error: Symlink the latest weights file to 'latest_weights.h5'")
+            raise FileNotFoundError(
+            errno.ENOENT,
+            os.strerror(errno.ENOENT),
+            weights_file)
+
+        model_kwargs = {
+            "weights_file": weights_file,
+            "print_summary": False,
+            "seed": None
+        }
+        self.model = EyeballModel(**model_kwargs)
+
+    def test_different_seed_predict(self):
+        model_kwargs = {
+            "weights_file": None,
+            "print_summary": False,
+            "seed": 12345678
+        }
+        same_seed_model = EyeballModel(**model_kwargs)
+
+        screenshot = "tests/data/test.png"
+
+        results_one = self.model.predict(screenshot)
+        results_two = same_seed_model.predict(screenshot)
+
+        self.assertNotEqual(results_one, results_two)
+
+    def test_same_seed_predict(self):
+        model_kwargs = {
+            "weights_file": None,
+            "print_summary": False,
+            "seed": 12345678
+        }
+        same_seed_model = EyeballModel(**model_kwargs)
+
+        screenshot = "tests/data/test.png"
+
+        results_one = same_seed_model.predict(screenshot)
+        results_two = same_seed_model.predict(screenshot)
+
+        self.assertEquals(results_one, results_two)
+
+    def get_prediction_by_name(self,results):
+        prediction = None
+        max_val = 0.0
+
+        print(results)
+        for k, v in results[0].items():
+            if not isinstance(v, str) and v > max_val:
+                    prediction = k
+                    max_val = v
+        return prediction
+
+    def test_predict_custom404(self):
+        screenshot = "tests/data/custom404.png"
+        results = self.model.predict(screenshot)
+        self.assertEqual(self.get_prediction_by_name(results), "custom404")
+
+    def test_predict_login(self):
+        screenshot = "tests/data/login.png"
+        results = self.model.predict(screenshot)
+        self.assertEqual(self.get_prediction_by_name(results), "login")
+
+    def test_predict_homepage(self):
+        screenshot = "tests/data/homepage.png"
+        results = self.model.predict(screenshot)
+        self.assertEqual(self.get_prediction_by_name(results), "homepage")
+
+    def test_predict_oldlooking(self):
+        screenshot = "tests/data/oldlooking.png"
+        results = self.model.predict(screenshot)
+        self.assertEqual(self.get_prediction_by_name(results), "oldlooking")

tests/TrainTest.py

@@ -0,0 +1 @@
+#TODO

tests/__init__.py

@@ -0,0 +1 @@
+from tests.PredictTest import *

utils/labelbox_to_labels.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import csv
+import random
+import json
+
+with open("newlabels.csv", "w", newline="") as csvfile:
+    # Open the old labels file
+    with open("labelbox.csv", newline="") as oldfile:
+        # Get the header labels
+        csvreader = csv.DictReader(oldfile)
+        fieldnames = next(csvreader)
+        labelwriter = csv.DictWriter(csvfile, fieldnames=["filename", "login", "custom404", "homepage", "oldlooking", "evaluation"])
+        labelwriter.writeheader()
+
+        # Loop through the file
+        rows = []
+        for row in csvreader:
+            filename = row["External ID"]
+            print(row["Label"])
+            labelstring = row["Label"]
+            if row["Label"] == "Skip":
+                labelstring = '{"imageclassification":[]}'
+            labels = json.loads(labelstring)
+
+            newrow = dict()
+            newrow["filename"] = filename
+            newrow["oldlooking"] = True
+            newrow["login"] = "loginpage" in labels["imageclassification"]
+            newrow["homepage"] = "homepage" in labels["imageclassification"]
+            newrow["custom404"] = "custom404"in labels["imageclassification"]
+
+            newrow["evaluation"] = random.random() > 0.8
+
+            rows.append(newrow)
+        labelwriter.writerows(rows)
+print("Made new labels file: newlabels.csv")

utils/reroll.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import csv
+import random
+
+with open("newlabels.csv", "w", newline="") as csvfile:
+    # Open the old labels file
+    with open("labels.csv", newline="") as oldfile:
+        # Get the header labels
+        csvreader = csv.DictReader(oldfile)
+        fieldnames = next(csvreader)
+        labelwriter = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        labelwriter.writeheader()
+
+        # Loop through the file
+        rows = []
+        for row in csvreader:
+            row["evaluation"] = random.random() > 0.8
+            rows.append(row)
+        labelwriter.writerows(rows)
+print("Made new labels file: newlabels.csv")

utils/resizer.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Converts full-size images to a given size (replacing the originals)
+for f in images_full_size/*
+do
+	convert -resize 224x224 $f $f
+done