Drop filesize parameter from torperf failures graph.

Implements tpo/metrics/website#40001.

src/main/R/rserver/rserve-init.R

@@ -611,36 +611,26 @@ plot_torperf <- function(start_p, end_p, server_p, filesize_p, path_p) {
 }
 
 prepare_torperf_failures <- function(start_p = NULL, end_p = NULL,
-    server_p = NULL, filesize_p = NULL) {
-  read_csv(file = paste(stats_dir, "torperf-1.1.csv", sep = ""),
+    server_p = NULL) {
+  read_csv(file = paste(stats_dir, "onionperf-failures.csv", sep = ""),
       col_types = cols(
         date = col_date(format = ""),
-        filesize = col_double(),
         source = col_character(),
         server = col_character(),
-        q1 = col_skip(),
-        md = col_skip(),
-        q3 = col_skip(),
         timeouts = col_double(),
         failures = col_double(),
         requests = col_double())) %>%
     filter(if (!is.null(start_p)) date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p)) date <= as.Date(end_p) else TRUE) %>%
-    filter(if (!is.null(filesize_p))
-        filesize == ifelse(filesize_p == "50kb", 50 * 1024,
-        ifelse(filesize_p == "1mb", 1024 * 1024, 5 * 1024 * 1024)) else
-        TRUE) %>%
     filter(if (!is.null(server_p)) server == server_p else TRUE) %>%
     filter(requests > 0) %>%
-    transmute(date, filesize, source, server, timeouts = timeouts / requests,
+    transmute(date, source, server, timeouts = timeouts / requests,
         failures = failures / requests)
 }
 
-plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p,
-    path_p) {
-  prepare_torperf_failures(start_p, end_p, server_p, filesize_p) %>%
-    filter(source != "") %>%
-    gather(variable, value, -c(date, filesize, source, server)) %>%
+plot_torperf_failures <- function(start_p, end_p, server_p, path_p) {
+  prepare_torperf_failures(start_p, end_p, server_p) %>%
+    gather(variable, value, -c(date, source, server)) %>%
     mutate(variable = factor(variable, levels = c("timeouts", "failures"),
       labels = c("Timeouts", "Failures"))) %>%
     ggplot(aes(x = date, y = value, colour = source)) +
@@ -650,10 +640,7 @@ plot_torperf_failures <- function(start_p, end_p, server_p, filesize_p,
     scale_y_continuous(name = "", labels = percent, limits = c(0, NA)) +
     scale_colour_hue(name = "Source") +
     facet_grid(variable ~ .) +
-    ggtitle(paste("Timeouts and failures of",
-        ifelse(filesize_p == "50kb", "50 KiB",
-        ifelse(filesize_p == "1mb", "1 MiB", "5 MiB")),
-        "requests to", server_p, "server")) +
+    ggtitle(paste("Timeouts and failures of requests to", server_p, "server")) +
     labs(caption = copyright_notice) +
     theme(legend.position = "top")
   ggsave(filename = path_p, width = 8, height = 5, dpi = 150)

src/main/java/org/torproject/metrics/stats/onionperf/Main.java

@@ -49,8 +49,8 @@ public class Main {
     writeStatistics(
         new File(baseDir, "stats/onionperf-including-partials.csv").toPath(),
         queryOnionperfIncludingPartials(connection));
-    writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(),
-        queryOnionPerf(connection));
+    writeStatistics(new File(baseDir, "stats/onionperf-failures.csv").toPath(),
+        queryOnionperfFailures(connection));
     writeStatistics(new File(baseDir, "stats/buildtimes.csv").toPath(),
         queryBuildTimes(connection));
     writeStatistics(new File(baseDir, "stats/latencies.csv").toPath(),
@@ -281,26 +281,21 @@ public class Main {
     return statistics;
   }
 
-  static List<String> queryOnionPerf(Connection connection)
+  static List<String> queryOnionperfFailures(Connection connection)
       throws SQLException {
     logger.info("Querying timeout and failure statistics from database.");
     List<String> statistics = new ArrayList<>();
-    statistics
-        .add("date,filesize,source,server,q1,md,q3,timeouts,failures,requests");
+    statistics.add("date,source,server,timeouts,failures,requests");
     Statement st = connection.createStatement();
-    String queryString = "SELECT date, filesize, source, server, q1, md, q3, "
-        + "timeouts, failures, requests FROM onionperf";
+    String queryString = "SELECT date, source, server, timeouts, failures, "
+        + "requests FROM onionperf_failures";
     DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
     try (ResultSet rs = st.executeQuery(queryString)) {
       while (rs.next()) {
-        statistics.add(String.format("%s,%d,%s,%s,%.0f,%.0f,%.0f,%d,%d,%d",
+        statistics.add(String.format("%s,%s,%s,%d,%d,%d",
             dateFormat.format(rs.getDate("date")),
-            rs.getInt("filesize"),
             getStringFromResultSet(rs, "source"),
             getStringFromResultSet(rs, "server"),
-            getDoubleFromResultSet(rs, "q1"),
-            getDoubleFromResultSet(rs, "md"),
-            getDoubleFromResultSet(rs, "q3"),
             rs.getInt("timeouts"),
             rs.getInt("failures"),
             rs.getInt("requests")));

src/main/resources/web/json/metrics.json

@@ -291,8 +291,7 @@
     "parameters": [
       "start",
       "end",
-      "server",
-      "filesize"
+      "server"
     ]
   },
   {

src/main/resources/web/jsps/stats.jsp

@@ -57,6 +57,7 @@ https://metrics.torproject.org/identifier.csv
 <li><b>December 18, 2019:</b> Added <a href="#bridgedb-transport">BridgeDB requests by requested transport</a> and <a href="#bridgedb-distributor">BridgeDB requests by distributor</a> graphs.</li>
 <li><b>May 24, 2020:</b> Included version 3 onion service measurements in <a href="#torperf">Time to download files over Tor</a>, <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a>, <a href="#onionperf-buildtimes">Circuit build times</a>, <a href="#onionperf-latencies">Circuit round-trip latencies</a> graphs.</li>
 <li><b>May 24, 2020:</b> Included partial downloads of larger file sizes in <a href="#torperf">Time to download files over Tor</a> graph.</li>
+<li><b>June 25, 2020:</b> Removed filesize parameter from <a href="#torperf-failures">Timeouts and failures of downloading files over Tor</a> graphs.</li>
 </ul>
 
 </div>
@@ -576,14 +577,12 @@ Performance <a href="#performance" name="performance" class="anchor">#</a></h2>
 <li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li>
 <li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li>
 <li><b>server:</b> Either <b>"public"</b> for requests to a server on the public internet, or <b>"onion"</b> for requests to an onion server.</li>
-<li><b>filesize:</b> Size of the downloaded file in bytes, with pre-defined possible values: <b>"50kb"</b>, <b>"1mb"</b>, or <b>"5mb"</b>.</li>
 </ul>
 
 <h4>Columns</h4>
 
 <ul>
 <li><b>date:</b> UTC date (YYYY-MM-DD) when download performance was measured.</li>
-<li><b>filesize:</b> Size of the downloaded file in bytes.</li>
 <li><b>source:</b> Name of the OnionPerf or Torperf service performing measurements.</li>
 <li><b>server:</b> Either <b>"public"</b> if the request was made to a server on the public internet, or <b>"onion"</b> if the request was made to an onion server.</li>
 <li><b>timeouts:</b> Fraction of requests that timed out when attempting to download the static file over Tor.</li>

src/main/sql/onionperf/init-onionperf.sql

@@ -110,26 +110,11 @@ SELECT date,
 FROM grouped
 ORDER BY date, filesize, source, server;
 
-CREATE OR REPLACE VIEW onionperf AS
-SELECT date,
-  filesize,
-  source,
-  server,
-  CASE WHEN q IS NULL THEN NULL ELSE q[1] END AS q1,
-  CASE WHEN q IS NULL THEN NULL ELSE q[2] END AS md,
-  CASE WHEN q IS NULL THEN NULL ELSE q[3] END AS q3,
-  timeouts,
-  failures,
-  requests
-FROM (
+CREATE OR REPLACE VIEW onionperf_failures AS
 SELECT DATE(start) AS date,
-  filesize,
   source,
   CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion'
     ELSE 'public' END AS server,
-  CASE WHEN COUNT(*) > 0 THEN
-    PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete)
-    ELSE NULL END AS q,
   COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END)
     AS timeouts,
   COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
@@ -137,25 +122,8 @@ SELECT DATE(start) AS date,
   COUNT(*) AS requests
 FROM measurements
 WHERE DATE(start) < current_date - 1
-GROUP BY date, filesize, source, server
-UNION
-SELECT DATE(start) AS date,
-  filesize,
-  '' AS source,
-  CASE WHEN endpointremote LIKE '%.onion:%' THEN 'onion'
-    ELSE 'public' END AS server,
-  CASE WHEN COUNT(*) > 0 THEN
-    PERCENTILE_CONT(ARRAY[0.25,0.5,0.75]) WITHIN GROUP(ORDER BY datacomplete)
-    ELSE NULL END AS q,
-  COUNT(CASE WHEN didtimeout OR datacomplete < 1 THEN 1 ELSE NULL END)
-    AS timeouts,
-  COUNT(CASE WHEN NOT didtimeout AND datacomplete >= 1
-    AND readbytes < filesize THEN 1 ELSE NULL END) AS failures,
-  COUNT(*) AS requests
-FROM measurements
-WHERE DATE(start) < current_date - 1
-GROUP BY date, filesize, 3, server) sub
-ORDER BY date, filesize, source, server;
+GROUP BY date, source, server
+ORDER BY date, source, server;
 
 CREATE OR REPLACE VIEW buildtimes_stats AS
 SELECT date,