Parse bridge statuses.

src/ArchiveReader.java

@@ -9,8 +9,8 @@ public class ArchiveReader {
   public ArchiveReader(ConsensusParser cp, ServerDescriptorParser sdp,
       ExtraInfoParser eip, String archivesDir, Set<String> directoryKeys)
       throws IOException {
-    System.out.print("Reading all files in directory " + archivesDir
-        + "/ ...");
+    System.out.print("Parsing all files in directory " + archivesDir
+        + "/...");
     Stack<File> filesInInputDir = new Stack<File>();
     filesInInputDir.add(new File(archivesDir));
     while (!filesInInputDir.isEmpty()) {

src/BridgeReader.java

@@ -4,8 +4,11 @@ import java.util.*;
 import org.apache.commons.codec.digest.*;
 
 public class BridgeReader {
-  public BridgeReader(BridgeStatsFileHandler bsfh, String bridgesDir,
+  public BridgeReader(ConsensusStatsFileHandler csfh,
+      BridgeStatsFileHandler bsfh, String bridgesDir,
       SortedSet<String> countries) throws IOException, ParseException {
+    System.out.print("Parsing all files in directory " + bridgesDir
+        + "/...");
     SimpleDateFormat timeFormat = new SimpleDateFormat(
         "yyyy-MM-dd HH:mm:ss");
     timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -27,7 +30,20 @@ public class BridgeReader {
       boolean skip = false;
       String line = null;
       while ((line = br.readLine()) != null) {
-        if (line.startsWith("extra-info ")) {
+        if (line.startsWith("r ")) {
+          // parse bridge status; TODO possibly move to own class
+          int runningBridges = 0;
+          while ((line = br.readLine()) != null) {
+            if (line.startsWith("s ") && line.contains(" Running")) {
+              runningBridges++;
+            }
+          }
+          String fn = pop.getName();
+          String date = fn.substring(0, 4) + "-" + fn.substring(4, 6)
+              + "-" + fn.substring(6, 8) + " " + fn.substring(9, 11)
+              + ":" + fn.substring(11, 13) + ":" + fn.substring(13, 15);
+          csfh.addBridgeConsensusResults(date, runningBridges);
+        } else if (line.startsWith("extra-info ")) {
           hashedIdentity = needToHash ? DigestUtils.shaHex(
               line.split(" ")[2]).toUpperCase() : line.split(" ")[2];
           skip = bsfh.isKnownRelay(hashedIdentity);
@@ -66,5 +82,6 @@ public class BridgeReader {
       }
       br.close();
     }
+    System.out.println("done");
   }
 }

src/ConsensusStatsFileHandler.java

@@ -7,13 +7,14 @@ import java.util.*;
 public class ConsensusStatsFileHandler {
   private String statsDir;
   private File consensusStatsRawFile;
+  private File bridgeConsensusStatsRawFile;
   private File consensusStatsFile;
   private SortedMap<String, String> consensusResults;
+  private SortedMap<String, String> bridgeConsensusResults;
   public ConsensusStatsFileHandler(String statsDir) throws IOException {
     this.statsDir = statsDir;
     this.consensusResults = new TreeMap<String, String>();
     this.consensusStatsRawFile = new File(statsDir + "/consensus-stats-raw");
-    this.consensusStatsFile = new File(statsDir + "/consensus-stats");
     if (this.consensusStatsRawFile.exists()) {
       System.out.print("Reading existing file " + statsDir
           + "/consensus-stats-raw... ");
@@ -26,37 +27,52 @@ public class ConsensusStatsFileHandler {
       System.out.println("done");
       br.close();
     }
+    this.bridgeConsensusResults = new TreeMap<String, String>();
+    this.bridgeConsensusStatsRawFile = new File(statsDir
+        + "/bridge-consensus-stats-raw");
+    if (this.bridgeConsensusStatsRawFile.exists()) {
+      System.out.print("Reading existing file " + statsDir
+          + "/bridge-consensus-stats-raw... ");
+      BufferedReader br = new BufferedReader(new FileReader(
+          this.bridgeConsensusStatsRawFile));
+      String line = null;
+      while ((line = br.readLine()) != null) {
+        bridgeConsensusResults.put(line.split(",")[0], line);
+      }
+      System.out.println("done");
+      br.close();
+    }
+    this.consensusStatsFile = new File(statsDir + "/consensus-stats");
   }
   public void addConsensusResults(String validAfter, int exit, int fast,
       int guard, int running, int stable) {
     consensusResults.put(validAfter, validAfter + "," + exit + "," + fast
         + "," + guard + "," + running + "," + stable);
   }
+  public void addBridgeConsensusResults(String published, int running) {
+    bridgeConsensusResults.put(published, published + "," + running);
+  }
   public void writeFile() throws IOException {
-    System.out.print("Writing files " + this.statsDir
-        + "/consensus-stats-raw and " + this.statsDir
-        + "/consensus-stats... ");
+    System.out.print("Writing file " + this.statsDir
+        + "/consensus-stats-raw... ");
     BufferedWriter bwConsensusStatsRaw = new BufferedWriter(
         new FileWriter(this.consensusStatsRawFile));
-    BufferedWriter bwConsensusStats = new BufferedWriter(
-        new FileWriter(this.consensusStatsFile));
-    bwConsensusStats.append("date,exit,fast,guard,running,stable\n");
     String tempDate = null;
     int exitDay = 0, fastDay = 0, guardDay = 0, runningDay = 0,
         stableDay = 0, consensusesDay = 0;
     Iterator<String> it = consensusResults.values().iterator();
     boolean haveWrittenFinalLine = false;
+    SortedMap<String, String> csAggr = new TreeMap<String, String>();
     while (it.hasNext() || !haveWrittenFinalLine) {
-      String next = it.hasNext() ? it.next()
-          : null;
+      String next = it.hasNext() ? it.next() : null;
       if (tempDate != null && (next == null ||
           !next.substring(0, 10).equals(tempDate))) {
-        bwConsensusStats.append(tempDate + ","
+        csAggr.put(tempDate, tempDate + ","
             + (exitDay / consensusesDay) + ","
             + (fastDay / consensusesDay) + ","
             + (guardDay / consensusesDay) + ","
             + (runningDay / consensusesDay) + ","
-            + (stableDay / consensusesDay) + "\n");
+            + (stableDay / consensusesDay));
         exitDay = 0;
         fastDay = 0;
         guardDay = 0;
@@ -80,6 +96,54 @@ public class ConsensusStatsFileHandler {
       }
     }
     bwConsensusStatsRaw.close();
+    System.out.print("done\nWriting file " + this.statsDir
+        + "/bridge-consensus-stats-raw... ");
+    BufferedWriter bwBridgeConsensusStatsRaw = new BufferedWriter(
+        new FileWriter(this.bridgeConsensusStatsRawFile));
+    tempDate = null;
+    int brunningDay = 0, bridgeStatusesDay = 0;
+    it = bridgeConsensusResults.values().iterator();
+    haveWrittenFinalLine = false;
+    SortedMap<String, String> bcsAggr = new TreeMap<String, String>();
+    while (it.hasNext() || !haveWrittenFinalLine) {
+      String next = it.hasNext() ? it.next() : null;
+      if (tempDate != null && (next == null ||
+          !next.substring(0, 10).equals(tempDate))) {
+        bcsAggr.put(tempDate, "" + (brunningDay / bridgeStatusesDay) + "\n");
+        brunningDay = 0;
+        bridgeStatusesDay = 0;
+        if (next == null) {
+          haveWrittenFinalLine = true;
+        }
+      }
+      if (next != null) {
+        bwBridgeConsensusStatsRaw.append(next + "\n");
+        tempDate = next.substring(0, 10);
+        bridgeStatusesDay++;
+        brunningDay += Integer.parseInt(next.split(",")[1]);
+      }
+    }
+    bwBridgeConsensusStatsRaw.close();
+    System.out.print("done\nWriting file " + this.statsDir
+        + "/consensus-stats... ");
+    BufferedWriter bwConsensusStats = new BufferedWriter(
+        new FileWriter(this.consensusStatsFile));
+    bwConsensusStats.append("date,exit,fast,guard,running,stable,brunning\n");
+    SortedSet<String> allDates = new TreeSet<String>();
+    allDates.addAll(csAggr.keySet());
+    allDates.addAll(bcsAggr.keySet());
+    for (String date : allDates) {
+      if (csAggr.containsKey(date)) {
+        bwConsensusStats.append(csAggr.get(date));
+      } else {
+        bwConsensusStats.append(date + ",NA,NA,NA,NA,NA");
+      }
+      if (bcsAggr.containsKey(date)) {
+        bwConsensusStats.append("," + bcsAggr.get(date));
+      } else {
+        bwConsensusStats.append(",NA\n");
+      }
+    }
     bwConsensusStats.close();
     System.out.println("done");
   }

src/Main.java

@@ -54,11 +54,13 @@ public class Main {
     ExtraInfoParser eip = new ExtraInfoParser(dsfh, countries,
         directories);
 
-    // Read files in archives/ directory
+    // Read files in archives/ and bridges/ directory
     if (!downloadOnly) {
 // TODO prevent overlapping runs by cron and manually!!
       ArchiveReader ar = new ArchiveReader(cp, sdp, eip, "archives",
           directories.keySet());
+      BridgeReader br = new BridgeReader(csfh, bsfh, "bridges",
+          countries);
     }
 
     // Download current descriptors
@@ -72,9 +74,6 @@ public class Main {
           directories);
     }
 
-    // Read files in bridges/ directory
-    BridgeReader br = new BridgeReader(bsfh, "bridges", countries);
-
     // Parse torperf files
     // TODO maybe convert them in a format that is easier to process for R than the current one?
     // TorperfParser tp = new TorperfParser(tsfh);