Include microdescriptors when syncing.

CHANGELOG.md

@@ -4,6 +4,9 @@
   - Clean up descriptors written to the `out/` directory by deleting
     files that are older than seven weeks.
   - Correctly index files that are moved away and back.
+  - Include microdescriptors when syncing from another CollecTor
+    instance.
+  - Update to metrics-lib 2.15.0.
 
 
 # Changes in version 1.16.1 - 2020-08-16

build.xml

@@ -12,7 +12,7 @@
   <property name="release.version" value="1.16.1-dev" />
   <property name="project-main-class" value="org.torproject.metrics.collector.Main" />
   <property name="name" value="collector"/>
-  <property name="metricslibversion" value="2.14.0" />
+  <property name="metricslibversion" value="2.15.0" />
   <property name="jarincludes" value="collector.properties logback.xml" />
 
   <patternset id="runtime" >

src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java

@@ -0,0 +1,36 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.Microdescriptor;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import java.nio.file.Paths;
+
+public class MicrodescriptorPersistence
+    extends DescriptorPersistence<Microdescriptor> {
+
+  private static final String RELAY_DESCRIPTORS = "relay-descriptors";
+
+  public MicrodescriptorPersistence(Microdescriptor descriptor, long received,
+      String year, String month) {
+    super(descriptor, Annotation.Microdescriptor.bytes());
+    calculatePaths(received, year, month);
+  }
+
+  private void calculatePaths(long received, String year, String month) {
+    String file = PersistenceUtils.dateTime(received);
+    this.recentPath = Paths.get(
+        RELAY_DESCRIPTORS, MICRODESCS, "micro",
+        file + "-micro-" + year + "-" + month).toString();
+    String digest = desc.getDigestSha256Hex();
+    this.storagePath = Paths.get(
+        RELAY_DESCRIPTORS,
+        MICRODESC, year, month, "micro",
+        digest.substring(0,1),
+        digest.substring(1,2),
+        digest).toString();
+  }
+}
+

src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java

@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.Microdescriptor;
 import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -109,6 +110,8 @@ public class ArchiveWriter extends CollecTorMain {
     this.mapPathDescriptors.put(
         "recent/relay-descriptors/microdescs/consensus-microdesc",
         RelayNetworkStatusConsensus.class);
+    this.mapPathDescriptors.put("recent/relay-descriptors/microdescs/micro/",
+        Microdescriptor.class);
     this.mapPathDescriptors.put("recent/relay-descriptors/server-descriptors",
         RelayServerDescriptor.class);
     this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",
@@ -801,15 +804,17 @@ public class ArchiveWriter extends CollecTorMain {
      * file written in the first call.  However, this method must be
      * called twice to store the same microdescriptor in two different
      * valid-after months. */
-    SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+    SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy-MM");
+    String[] yearMonth = descriptorFormat.format(validAfter).split("-");
     File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
-        descriptorFormat.format(validAfter), MICRO,
+        yearMonth[0], yearMonth[1], MICRO,
         microdescriptorDigest.substring(0, 1),
         microdescriptorDigest.substring(1, 2),
         microdescriptorDigest).toFile();
     boolean tarballFileExistedBefore = tarballFile.exists();
     File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
-        MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile();
+        MICRODESCS, MICRO, this.rsyncCatString + "-micro-" + yearMonth[0] + "-"
+        + yearMonth[1] + ".tmp").toFile();
     File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
     boolean[] append = new boolean[] { false, true };
     if (this.store(Annotation.Microdescriptor.bytes(), data, outputFiles,

src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java

@@ -11,6 +11,7 @@ import org.torproject.descriptor.BridgeServerDescriptor;
 import org.torproject.descriptor.BridgedbMetrics;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.Microdescriptor;
 import org.torproject.descriptor.RelayExtraInfoDescriptor;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -32,6 +33,7 @@ import org.torproject.metrics.collector.persist.DescriptorPersistence;
 import org.torproject.metrics.collector.persist.ExitlistPersistence;
 import org.torproject.metrics.collector.persist.ExtraInfoPersistence;
 import org.torproject.metrics.collector.persist.MicroConsensusPersistence;
+import org.torproject.metrics.collector.persist.MicrodescriptorPersistence;
 import org.torproject.metrics.collector.persist.OnionPerfPersistence;
 import org.torproject.metrics.collector.persist.PersistenceUtils;
 import org.torproject.metrics.collector.persist.ServerDescriptorPersistence;
@@ -118,6 +120,14 @@ public class SyncPersistence {
           descPersist = new ExtraInfoPersistence(
               (RelayExtraInfoDescriptor) desc, received);
           break;
+        case "Microdescriptor":
+          if (filename.contains("-micro-")) {
+            String[] yearMonth = filename.substring(
+                filename.indexOf("-micro-") + 7).split("-");
+            descPersist = new MicrodescriptorPersistence((Microdescriptor) desc,
+                received, yearMonth[0], yearMonth[1]);
+          }
+          break;
         case "BridgeNetworkStatus": // need to infer authId from filename
           String[] filenameParts = filename.split(DASH);
           if (filenameParts.length < 3) {