Switch maximum string limit to warning. (#1350)

When we hit the maximum match limit on a string we currently abort all scanning,
which can be problematic since it isn't a good reason to stop all scanning, and
is better off as a warning to the user.

This commit makes it so that when the string match limit is hit a warning is
printed and scanning will continue. The warning message indicates that the
results may be invalid. It is hard to know for certain if they are invalid
because it depends entirely upon the condition. If you use the # or @ operator
then your condition _MAY_ be invalid since the total number of matches and
locations is not known anymore. Just checking that the string exists won't be an
invalid state since we have confirmed the existence of _AT LEAST_ the maximum
number of occurrences, which clearly satisfies the existence of a single
instance of your string.

I decided to implement this as a callback so the user of libyara has some choice
on the default behavior. If they want to consider this a "stop all scanning"
error they are free to return CALLBACK_ERROR in their callback.

cli/yara.c

@@ -1044,7 +1044,15 @@ static int callback(
       mutex_unlock(&output_mutex);
     }
 
-    return CALLBACK_CONTINUE;
+      return CALLBACK_CONTINUE;
+
+    case CALLBACK_MSG_TOO_MANY_MATCHES:
+      fprintf(
+          stderr,
+          "Warning: maximum matches for string %s. Results may be invalid.\n",
+          ((YR_STRING*) message_data)->identifier);
+
+      return CALLBACK_CONTINUE;
   }
 
   return CALLBACK_ERROR;

docs/capi.rst

@@ -225,6 +225,7 @@ Possible values for ``message`` are::
   CALLBACK_MSG_SCAN_FINISHED
   CALLBACK_MSG_IMPORT_MODULE
   CALLBACK_MSG_MODULE_IMPORTED
+  CALLBACK_MSG_TOO_MANY_MATCHES
 
 Your callback function will be called once for each rule with either
 a ``CALLBACK_MSG_RULE_MATCHING`` or ``CALLBACK_MSG_RULE_NOT_MATCHING`` message,
@@ -253,6 +254,12 @@ CALLBACK_MSG_MODULE_IMPORTED. When this happens ``message_data`` points to a
 :c:type:`YR_OBJECT_STRUCTURE` structure. This structure contains all the
 information provided by the module about the currently scanned file.
 
+If during the scan a string hits the maximum number of matches your callback
+will be called once with the CALLBACK_MSG_TOO_MANY_MATCHES. When this happens,
+`message_data` is a `YR_STRING*` which points to the string which caused the
+warning. If your callback returns CALLBACK_CONTINUE the string will be disabled
+and scanning will continue, otherwise scanning will be halted.
+
 Lastly, the callback function is also called with the
 ``CALLBACK_MSG_SCAN_FINISHED`` message when the scan is finished. In this case
 ``message_data`` is ``NULL``.

docs/yarapython.rst

@@ -266,6 +266,33 @@ Here is an example:
 
 The passed dictionary will contain the information from the module.
 
+You can also specify a warning callback function when invoking the ``match``
+method.  The provided function will be called for every runtime warning.
+Your callback function should expect two parameters. The first is an integer
+which contains the type of warning and the second is a string with the warning
+message. Your callback should return ``CALLBACK_CONTINUE`` to proceed with the
+scan or ``CALLBACK_ABORT`` to stop.
+
+Possible values for the type are::
+
+  CALLBACK_TOO_MANY_MATCHES
+
+Here is an example:
+
+.. code-block:: python
+
+  import yara
+
+  def warning_callback(warning_type, message):
+    if warning_type == yara.CALLBACK_TOO_MANY_MATCHES:
+        print(message)
+    return yara.CALLBACK_CONTINUE
+
+  matches = rules.match('/foo/bar/my_file', warning_callback=warning_callback)
+
+If you do not use a warning callback a warning message will be sent to the
+normal python warning system for you and scanning will continue.
+
 
 You may also find that the default sizes for the stack for the matching engine in
 yara or the default size for the maximum number of strings per rule is too low. In

libyara/include/yara/limits.h

@@ -109,7 +109,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 // Maximum number of matches allowed for a string. If more matches are found
-// the scan will fail with ERROR_TOO_MANY_MATCHES.
+// the scan will have a CALLBACK_MSG_TOO_MANY_MATCHES.
 #ifndef YR_MAX_STRING_MATCHES
 #define YR_MAX_STRING_MATCHES 1000000
 #endif

libyara/include/yara/rules.h

@@ -34,6 +34,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <yara/types.h>
 #include <yara/utils.h>
 
+
+#define CALLBACK_MSG_RULE_MATCHING              1
+#define CALLBACK_MSG_RULE_NOT_MATCHING          2
+#define CALLBACK_MSG_SCAN_FINISHED              3
+#define CALLBACK_MSG_IMPORT_MODULE              4
+#define CALLBACK_MSG_MODULE_IMPORTED            5
+#define CALLBACK_MSG_TOO_MANY_MATCHES           6
+
 #define CALLBACK_MSG_RULE_MATCHING     1
 #define CALLBACK_MSG_RULE_NOT_MATCHING 2
 #define CALLBACK_MSG_SCAN_FINISHED     3

libyara/scan.c

@@ -38,6 +38,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <yara/re.h>
 #include <yara/scan.h>
 #include <yara/stopwatch.h>
+#include <yara/rules.h>
 #include <yara/types.h>
 #include <yara/utils.h>
 
@@ -990,6 +991,7 @@ int yr_scan_verify_match(
       offset);
 
   YR_STRING* string = ac_match->string;
+  YR_CALLBACK_FUNC callback = context->callback;
 
   int result;
 
@@ -999,6 +1001,25 @@ int yr_scan_verify_match(
   if (STRING_IS_DISABLED(string))
     return ERROR_SUCCESS;
 
+  if (context->matches[string->idx].count == YR_MAX_STRING_MATCHES)
+  {
+    result = callback(
+        context,
+        CALLBACK_MSG_TOO_MANY_MATCHES,
+        (void*) string,
+        context->user_data);
+
+    if (result == CALLBACK_CONTINUE)
+    {
+      string->flags |= STRING_FLAGS_DISABLED;
+      return ERROR_SUCCESS;
+    }
+    else if (result == CALLBACK_ABORT || result == CALLBACK_ERROR)
+      return ERROR_TOO_MANY_MATCHES;
+    else
+      return ERROR_INTERNAL_FATAL_ERROR;
+  }
+
   if (context->flags & SCAN_FLAGS_FAST_MODE && STRING_IS_SINGLE_MATCH(string) &&
       context->matches[string->idx].head != NULL)
     return ERROR_SUCCESS;

tests/test-api.c

@@ -26,6 +26,7 @@ void test_disabled_rules()
 
   counters.rules_not_matching = 0;
   counters.rules_matching = 0;
+  counters.rules_warning = 0;
 
   yr_initialize();
 
@@ -281,6 +282,7 @@ void test_save_load_rules()
 
   counters.rules_not_matching = 0;
   counters.rules_matching = 0;
+  counters.rules_warning = 0;
 
   char* rules_str = "rule t {condition: bool_var and str_var == \"foobar\"}";
 
@@ -446,6 +448,7 @@ void test_scanner()
 
   counters.rules_not_matching = 0;
   counters.rules_matching = 0;
+  counters.rules_warning = 0;
 
   // Set the callback and the correct variable values for the rule to match.
   yr_scanner_set_callback(scanner1, count, &counters);
@@ -480,6 +483,7 @@ void test_scanner()
 
   counters.rules_matching = 0;
   counters.rules_not_matching = 0;
+  counters.rules_warning = 0;
 
   yr_scanner_set_flags(scanner1, SCAN_FLAGS_REPORT_RULES_MATCHING);
   yr_scanner_set_callback(scanner1, count, &counters);
@@ -495,6 +499,7 @@ void test_scanner()
 
   counters.rules_matching = 0;
   counters.rules_not_matching = 0;
+  counters.rules_warning = 0;
 
   yr_scanner_set_flags(scanner2, SCAN_FLAGS_REPORT_RULES_NOT_MATCHING);
   yr_scanner_set_callback(scanner2, count, &counters);
@@ -783,6 +788,64 @@ void test_issue_920()
   yr_finalize();
 }
 
+
+void test_runtime_warnings() {
+  // This rule should never match since it will hit the maximum number of
+  // matches (see YR_MAX_STRING_MATCHES) and a warning will be issued, and any
+  // further matches no longer count.
+  const char* rules_str = "rule test { \
+    strings: \
+      $x = \"X\" \
+    condition: \
+      #x > 1000000 \
+    }";
+
+  YR_COMPILER* compiler = NULL;
+  YR_RULES* rules = NULL;
+  struct COUNTERS counters;
+
+  counters.rules_not_matching = 0;
+  counters.rules_matching = 0;
+  counters.rules_warning = 0;
+
+  yr_initialize();
+
+  if (yr_compiler_create(&compiler) != ERROR_SUCCESS) {
+    perror("yr_compiler_create");
+    exit(EXIT_FAILURE);
+  }
+
+  if (yr_compiler_add_string(compiler, rules_str, NULL) != ERROR_SUCCESS) {
+    yr_compiler_destroy(compiler);
+    perror("yr_compiler_add_string");
+    exit(EXIT_FAILURE);
+  }
+
+  if (yr_compiler_get_rules(compiler, &rules) != ERROR_SUCCESS)
+  {
+    yr_compiler_destroy(compiler);
+    perror("yr_compiler_get_rules");
+    exit(EXIT_FAILURE);
+  }
+
+  yr_compiler_destroy(compiler);
+
+  if (yr_rules_scan_file(rules, "tests/data/x.txt", 0, count, &counters, 0) != ERROR_SUCCESS) {
+    yr_rules_destroy(rules);
+    perror("yr_rules_scan_file");
+    exit(EXIT_FAILURE);
+  }
+
+  // There should only be a single warning issued since the string is disabled
+  // after the callback returns CALLBACK_CONTINUE.
+  assert_true_expr(counters.rules_warning == 1);
+  assert_true_expr(counters.rules_matching == 0);
+  assert_true_expr(counters.rules_not_matching == 1);
+
+  yr_rules_destroy(rules);
+  yr_finalize();
+}
+
 int main(int argc, char** argv)
 {
   chdir_if_env_top_srcdir();
@@ -798,6 +861,7 @@ int main(int argc, char** argv)
   test_rules_stats();
   test_issue_834();
   test_issue_920();
+  test_runtime_warnings();
 
   return 0;
 }

tests/util.c

@@ -253,6 +253,10 @@ int count(
 {
   switch (message)
   {
+  case CALLBACK_MSG_TOO_MANY_MATCHES:
+    (*(struct COUNTERS*) user_data).rules_warning++;
+    break;
+
   case CALLBACK_MSG_RULE_MATCHING:
     (*(struct COUNTERS*) user_data).rules_matching++;
     break;

tests/util.h

@@ -58,6 +58,7 @@ struct COUNTERS
 {
   int rules_matching;
   int rules_not_matching;
+  int rules_warning;
 };
 
 int compile_rule(