Download using index.json; implements task-19791.

Added tests and adapted coverage checks.

.gitignore

@@ -7,4 +7,4 @@ descriptor-*.tar.gz
 .classpath
 .project
 build.properties
-
+*~

build.xml

@@ -280,8 +280,21 @@
         <include name="**/*.java" />
       </fileset>
     </cobertura-report>
-    <cobertura-check branchrate="0" totallinerate="57" totalbranchrate="50" >
-      <regex pattern="org.torproject.descriptor.benchmark.*" branchrate="0" linerate="0"/>
+    <cobertura-check totallinerate="58" totalbranchrate="50" >
+      <regex pattern="org.torproject.descriptor.benchmark.*"
+             linerate="0" branchrate="0"/>
+      <regex pattern="org.torproject.descriptor.index"
+             linerate="97" branchrate="62"/>
+      <regex pattern="org.torproject.descriptor.DescriptorSourceFactory"
+             linerate="100" branchrate="77"/>
+      <regex pattern="org.torproject.descriptor.index.DescriptorIndexCollector"
+             linerate="92" branchrate="61"/>
+      <regex pattern="org.torproject.descriptor.index.IndexNode"
+             linerate="100" branchrate="61"/>
+      <regex pattern="org.torproject.descriptor.index.FileNode"
+             linerate="100" branchrate="100"/>
+      <regex pattern="org.torproject.descriptor.index.DirectoryNode"
+             linerate="100" branchrate="100"/>
     </cobertura-check>
   </target>
 

src/main/java/org/torproject/descriptor/index/DescriptorIndexCollector.java

@@ -0,0 +1,136 @@
+/* Copyright 2015--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import org.torproject.descriptor.DescriptorCollector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * Download files from a CollecTor instance based on the remote
+ * instance's index.json.
+ *
+ * @since 1.4.0
+ */
+public class DescriptorIndexCollector implements DescriptorCollector {
+
+  private static Logger log = LoggerFactory
+      .getLogger(DescriptorIndexCollector.class);
+
+  /**
+   * The parameter usage differs from the interface:
+   * <code>collecTorIndexUrl</code> is expected to contain the URL
+   * for an index JSON file (plain or compressed).
+   */
+  @Override
+  public void collectDescriptors(String collecTorIndexUrl,
+      String[] remoteDirectories, long minLastModified,
+      File localDirectory, boolean deleteExtraneousLocalFiles) {
+    if (minLastModified < 0) {
+      throw new IllegalArgumentException("A negative minimum "
+          + "last-modified time is not permitted.");
+    }
+    if (null == localDirectory || localDirectory.isFile()) {
+      throw new IllegalArgumentException("Local directory already exists "
+          + "and is not a directory.");
+    }
+    SortedMap<String, Long> localFiles = statLocalDirectory(localDirectory);
+    SortedMap<String, FileNode> remoteFiles = null;
+    IndexNode index = null;
+    try {
+      index = IndexNode.fetchIndex(collecTorIndexUrl);
+      remoteFiles = index.retrieveFilesIn(remoteDirectories);
+    } catch (Exception ioe) {
+      throw new RuntimeException("Cannot fetch index. ", ioe);
+    }
+    this.fetchRemoteFiles(index.path, remoteFiles, minLastModified,
+        localDirectory, localFiles);
+    if (deleteExtraneousLocalFiles) {
+      this.deleteExtraneousLocalFiles(remoteFiles, localDirectory, localFiles);
+    }
+  }
+
+  void fetchRemoteFiles(String baseUrl, SortedMap<String, FileNode> remotes,
+      long minLastModified, File localDir, SortedMap<String, Long> locals) {
+    for (Map.Entry<String, FileNode> entry : remotes.entrySet()) {
+      String filepathname = entry.getKey();
+      String filename = entry.getValue().path;
+      File filepath = new File(localDir,
+          filepathname.replace(filename, ""));
+      long lastModifiedMillis = entry.getValue().lastModifiedMillis();
+      if (lastModifiedMillis < minLastModified
+          || (locals.containsKey(filepathname)
+              && locals.get(filepathname) >= lastModifiedMillis)) {
+        continue;
+      }
+      if (!filepath.exists() && !filepath.mkdirs()) {
+        throw new RuntimeException("Cannot create dir: " + filepath);
+      }
+      File destinationFile = new File(filepath, filename);
+      File tempDestinationFile = new File(filepath, "." + filename);
+      try (InputStream is = new URL(baseUrl + "/" + filepathname)
+          .openStream()) {
+        Files.copy(is, tempDestinationFile.toPath());
+        if (tempDestinationFile.length() == entry.getValue().size) {
+          tempDestinationFile.renameTo(destinationFile);
+          destinationFile.setLastModified(lastModifiedMillis);
+        } else {
+          log.warn("File sizes don't match. Expected {},  but received {}.",
+              entry.getValue().size, tempDestinationFile.length());
+        }
+      } catch (IOException e) {
+        log.error("Cannot fetch remote file: {}", filename, e);
+      }
+    }
+  }
+
+  static void deleteExtraneousLocalFiles(
+      SortedMap<String, FileNode> remoteFiles,
+      File localDir, SortedMap<String, Long> locals) {
+    for (String localPath : locals.keySet()) {
+      if (!remoteFiles.containsKey(localPath)) {
+        new File(localDir, localPath).delete();
+      }
+    }
+  }
+
+  static SortedMap<String, Long> statLocalDirectory(
+      final File localDir) {
+    final SortedMap<String, Long> locals = new TreeMap<>();
+    if (!localDir.exists()) {
+      return locals;
+    }
+    try {
+      Files.walkFileTree(localDir.toPath(),
+          new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path path, BasicFileAttributes bfa)
+                throws IOException {
+              locals.put(path.toFile().getAbsolutePath()
+                  .replace(localDir.getAbsolutePath() + "/", ""),
+                      path.toFile().lastModified());
+              return FileVisitResult.CONTINUE;
+            }
+        });
+    } catch (IOException ioe) {
+      log.error("Cannot stat local directory.", ioe);
+    }
+    return locals;
+  }
+}
+

src/main/java/org/torproject/descriptor/index/DirectoryNode.java

@@ -0,0 +1,52 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import com.google.gson.annotations.Expose;
+
+import java.util.SortedSet;
+
+/**
+ * A directory node has a file set and a set of subdirectories.
+ *
+ * @since 1.4.0
+ */
+public class DirectoryNode implements Comparable<DirectoryNode> {
+
+  /** Path (i.e. directory name) is exposed in JSON. */
+  @Expose
+  public final String path;
+
+  /** The file list is exposed in JSON. Sorted according to path. */
+  @Expose
+  public final SortedSet<FileNode> files;
+
+  /** The directory list is exposed in JSON. Sorted according to path. */
+  @Expose
+  public final SortedSet<DirectoryNode> directories;
+
+  /** A directory for the JSON structure. */
+  public DirectoryNode(String path, SortedSet<FileNode> files,
+                SortedSet<DirectoryNode> directories) {
+    this.path = path;
+    this.files = files;
+    this.directories = directories;
+  }
+
+  /**
+   * This compareTo is not compatible with equals or hash!
+   * It simply ensures a path-sorted JSON output.
+   */
+  @Override
+  public int compareTo(DirectoryNode other) {
+    return this.path.compareTo(other.path);
+  }
+
+  /** For debugging purposes. */
+  @Override
+  public String toString() {
+    return "Dn: " + path + " fns: " + files + " dirs: " + directories;
+  }
+}
+

src/main/java/org/torproject/descriptor/index/FileNode.java

@@ -0,0 +1,84 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
+import java.util.TimeZone;
+
+/**
+ * A FileNode provides the file's name, size, and modified time.
+ *
+ * @since 1.4.0
+ */
+public class FileNode implements Comparable<FileNode> {
+
+  private static Logger log = LoggerFactory.getLogger(FileNode.class);
+
+  /** Path (i.e. file name) is exposed in JSON. */
+  @Expose
+  public final String path;
+
+  /** The file size is exposed in JSON. */
+  @Expose
+  public final long size;
+
+  /** The last modified date-time string is exposed in JSON. */
+  @Expose
+  @SerializedName("last_modified")
+  public final String lastModified;
+
+  private long lastModifiedMillis;
+
+  /**
+   * A FileNode needs a path, i.e. the file name, the file size, and
+   * the last modified date-time string.
+   */
+  public FileNode(String path, long size, String lastModified) {
+    this.path = path;
+    this.size = size;
+    this.lastModified = lastModified;
+  }
+
+  /**
+   * This compareTo is not compatible with equals or hash!
+   * It simply ensures a path-sorted Gson output.
+   */
+  @Override
+  public int compareTo(FileNode other) {
+    return this.path.compareTo(other.path);
+  }
+
+  /** Lazily returns the last modified time in millis. */
+  public long lastModifiedMillis() {
+    if (this.lastModifiedMillis == 0) {
+      DateFormat dateTimeFormat =
+          new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US);
+      dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      try {
+        lastModifiedMillis = dateTimeFormat.parse(this.lastModified).getTime();
+      } catch (ParseException ex) {
+        log.warn("Cannot parse date-time. Setting lastModifiedMillis to -1L.",
+            ex);
+        this.lastModifiedMillis = -1L;
+      }
+    }
+    return this.lastModifiedMillis;
+  }
+
+  /** For debugging purposes. */
+  @Override
+  public String toString() {
+    return "Fn: " + path;
+  }
+}
+

src/main/java/org/torproject/descriptor/index/FileType.java

@@ -0,0 +1,49 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * A file type enum provides all compression functionality.
+ *
+ * @since 1.4.0
+ */
+public enum FileType {
+  BZ2(BZip2CompressorInputStream.class, BZip2CompressorOutputStream.class),
+  GZ(GzipCompressorInputStream.class, GzipCompressorOutputStream.class),
+  JSON(BufferedInputStream.class, BufferedOutputStream.class),
+  XZ(XZCompressorInputStream.class, XZCompressorOutputStream.class);
+
+  private final Class<? extends InputStream> inClass;
+  private final Class<? extends OutputStream> outClass;
+
+  FileType(Class<? extends InputStream> in, Class<? extends OutputStream> out) {
+    this.inClass = in;
+    this.outClass = out;
+  }
+
+  /** Return the appropriate input stream. */
+  public InputStream inputStream(InputStream is) throws Exception {
+    return this.inClass.getConstructor(new Class[]{InputStream.class})
+        .newInstance(is);
+  }
+
+  /** Return the appropriate output stream. */
+  public OutputStream outputStream(OutputStream os) throws Exception {
+    return this.outClass.getConstructor(new Class[]{OutputStream.class})
+        .newInstance(os);
+  }
+}
+

src/main/java/org/torproject/descriptor/index/IndexNode.java

@@ -0,0 +1,166 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/**
+ * An index node is the top-level node in the JSON structure.
+ * It provides some utility methods for reading
+ * and searching (in a limited way) it's sub-structure.
+ *
+ * @since 1.4.0
+ */
+public class IndexNode {
+
+  private static Logger log = LoggerFactory.getLogger(IndexNode.class);
+
+  /** An empty node, which is not added to JSON output. */
+  public static final IndexNode emptyNode = new IndexNode("", "",
+      new TreeSet<FileNode>(), new TreeSet<DirectoryNode>());
+
+  /** The created date-time is exposed in JSON as 'index_created' field. */
+  @Expose
+  @SerializedName("index_created")
+  public final String created;
+
+  /** Path (i.e. base url) is exposed in JSON. */
+  @Expose
+  public final String path;
+
+  /** The directory list is exposed in JSON. Sorted according to path. */
+  @Expose
+  public final SortedSet<DirectoryNode> directories;
+
+  /** The file list is exposed in JSON. Sorted according to path. */
+  @Expose
+  public final SortedSet<FileNode> files;
+
+  /** An index node is the top-level node in the JSON structure. */
+  public IndexNode(String created, String path,
+            SortedSet<FileNode> files,
+            SortedSet<DirectoryNode> directories) {
+    this.path = path;
+    this.files = files;
+    this.directories = directories;
+    this.created = created;
+  }
+
+  /**
+   * Reads JSON from given URL String.
+   * Returns an empty IndexNode in case of an error.
+   */
+  public static IndexNode fetchIndex(String urlString) throws Exception {
+    String ending
+        = urlString.substring(urlString.lastIndexOf(".") + 1).toUpperCase();
+    try (InputStream is = FileType.valueOf(ending)
+        .inputStream(new URL(urlString).openStream())) {
+      return fetchIndex(is);
+    }
+  }
+
+  /**
+   * Reads JSON from given InputStream.
+   * Returns an empty IndexNode in case of an error.
+   */
+  public static IndexNode fetchIndex(InputStream is) throws IOException {
+    Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation()
+        .create();
+    try (Reader reader = new InputStreamReader(is)) {
+      return gson.fromJson(reader, IndexNode.class);
+    }
+  }
+
+  /** Return a map of file paths for the given directories. */
+  public SortedMap<String, FileNode> retrieveFilesIn(String ... remoteDirs) {
+    SortedMap<String, FileNode> map = new TreeMap<>();
+    for (String remote : remoteDirs) {
+      if (null == remote || remote.isEmpty()) {
+        continue;
+      }
+      String[] dirs = remote.replaceAll("/", " ").trim().split(" ");
+      DirectoryNode currentDir = findPathIn(dirs[0], this.directories);
+      if (null == currentDir) {
+        continue;
+      }
+      String currentPath = dirs[0] + "/";
+      for (int k = 1; k < dirs.length; k++) {
+        DirectoryNode dn = findPathIn(dirs[k], currentDir.directories);
+        if (null == dn) {
+          break;
+        } else {
+          currentPath += dirs[k] + "/";
+          currentDir = dn;
+        }
+      }
+      if (null == currentDir.files) {
+        continue;
+      }
+      for (FileNode file : currentDir.files) {
+        if (file.lastModifiedMillis() > 0) { // only add valid files
+          map.put(currentPath + file.path, file);
+        }
+      }
+    }
+    return map;
+  }
+
+  /** Returns the directory nodes with the given path, but no file nodes. */
+  public static DirectoryNode findPathIn(String path,
+      SortedSet<DirectoryNode> dirs) {
+    if (null != dirs) {
+      for (DirectoryNode dn : dirs) {
+        if (dn.path.equals(path)) {
+          return dn;
+        }
+      }
+    }
+    return null;
+  }
+
+  /** Write JSON representation of the given index node to the given path. */
+  public static void writeIndex(Path outPath, IndexNode indexNode)
+      throws Exception {
+    String ending = outPath.toString()
+        .substring(outPath.toString().lastIndexOf(".") + 1).toUpperCase();
+    try (OutputStream os = FileType.valueOf(ending)
+         .outputStream(Files.newOutputStream(outPath))) {
+      os.write(makeJsonString(indexNode).getBytes());
+    }
+  }
+
+  /** Write JSON representation of the given index node to a string. */
+  public static String makeJsonString(IndexNode indexNode) {
+    Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation()
+        .create();
+    return gson.toJson(indexNode);
+  }
+
+  /** For debugging purposes. */
+  @Override
+  public String toString() {
+    return "index: " + path + ", created " + created
+        + ",\nfns: " + files + ",\ndirs: " + directories;
+  }
+}
+

src/main/java/org/torproject/descriptor/index/package-info.java

@@ -0,0 +1,17 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+/**
+ * <h1>This package is still in alpha stage.</h1>
+ * <p>The public interface might still change in unexpected ways.</p> 
+ *
+ * <p>Interfaces and essential classes for obtaining and processing
+ * CollecTor's index.json file.</p>
+ *
+ * <p>Interfaces and classes make the content of index.json available.</p>
+ *
+ *
+ * @since 1.4.0
+ */
+package org.torproject.descriptor.index;
+

src/test/java/org/torproject/descriptor/index/DescriptorIndexCollectorTest.java

@@ -0,0 +1,227 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.torproject.descriptor.DescriptorCollector;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+public class DescriptorIndexCollectorTest {
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Rule
+  public TemporaryFolder tmpf = new TemporaryFolder();
+
+  @Test()
+  public void testNormalCollecting() throws Exception {
+    // create local file structure
+    File localFolder = tmpf.newFolder();
+    makeStructure(localFolder, "1");
+
+    // create remote file structure
+    File remoteDirectory = tmpf.newFolder();
+    makeStructure(remoteDirectory, "2");
+
+    File indexFile = newIndexFile("testindex.json",
+        remoteDirectory.toURL().toString());
+    checkContains(false,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/y2","a/b/x2");
+
+    DescriptorCollector dc = new DescriptorIndexCollector();
+    dc.collectDescriptors(indexFile.toURL().toString(),
+        new String[]{"a/b", "a"}, 1451606400_000L, localFolder, false);
+    checkContains(true,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/x1", "a/b/y1", "a/b/y2","a/b/x2", "a/b/c/w1", "a/b/c/z1");
+    checkContains(false,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/c/w2","a/b/c/z2");
+  }
+
+  private void makeStructure(File folder, String suffix) throws IOException {
+    File dir = makeDirs(folder.toString(), "a", "b");
+    makeFiles(dir, "x" + suffix, "y" + suffix);
+    File subdir = makeDirs(dir.toString(), "c");
+    makeFiles(subdir, "w" + suffix, "z" + suffix);
+    SortedMap<String, Long> local = DescriptorIndexCollector
+        .statLocalDirectory(folder);
+    assertEquals("found " + local, 4, local.size());
+  }
+
+  private File makeDirs(String first, String ... dirs) throws IOException {
+    File dir = Files.createDirectories(Paths.get(first, dirs)).toFile();
+    assertTrue(dir.isDirectory());
+    return dir;
+  }
+
+  private void makeFiles(File dir, String ... files) throws IOException {
+    for (String file : files) {
+      assertTrue(new File(dir, file).createNewFile());
+    }
+  }
+
+  private void checkContains(boolean should, String listing,
+      String ... vals) {
+    for (String part : vals) {
+      if (should) {
+        assertTrue("missing " + part + " in " + listing,
+            listing.contains(part));
+      } else {
+        assertFalse("Shouldn't be: " + part + " in " + listing,
+            listing.contains(part));
+      }
+    }
+  }
+
+  private File newIndexFile(String name, String remoteDirectory)
+      throws Exception {
+    SortedSet<FileNode> fm = new TreeSet<>();
+    fm.add(new FileNode("w2", 100L, "2100-01-01 01:01"));
+    fm.add(new FileNode("z2", 2L, "1900-01-01 01:02"));
+    SortedSet<DirectoryNode> dm = new TreeSet<>();
+    dm.add(new DirectoryNode("c", fm, null));
+    fm = new TreeSet<>();
+    fm.add(new FileNode("x2", 2L, "2100-01-01 01:01"));
+    fm.add(new FileNode("y2", 2L, "2100-01-01 01:02"));
+    DirectoryNode dnb = new DirectoryNode("b", fm, dm);
+    dm = new TreeSet<>();
+    dm.add(dnb);
+    DirectoryNode dna = new DirectoryNode("a", null, dm);
+    dm = new TreeSet<>();
+    dm.add(dna);
+    IndexNode in = new IndexNode("2016-01-01 01:01",
+        remoteDirectory, null, dm);
+    File indexFile = tmpf.newFile(name);
+    in.writeIndex(indexFile.toPath(), in);
+    return indexFile;
+  }
+
+  @Test()
+  public void testNormalCollectingWithDeletion() throws Exception {
+    File localFolder = tmpf.newFolder();
+    makeStructure(localFolder, "1");
+
+    File remoteDirectory = tmpf.newFolder();
+    makeStructure(remoteDirectory, "2");
+
+    File indexFile = newIndexFile("testindexDelete.json",
+        remoteDirectory.toURL().toString());
+    checkContains(false,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/y2","a/b/x2");
+
+    new DescriptorIndexCollector()
+        .collectDescriptors(indexFile.toURL().toString(),
+            new String[]{"a/b", "a/b/c"}, 1451606400_000L, localFolder, true);
+    checkContains(true,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/y2","a/b/x2");
+    checkContains(false,
+        DescriptorIndexCollector.statLocalDirectory(localFolder).toString(),
+        "a/b/x1", "a/b/y1", "a/b/c/w1", "a/b/c/z1", "a/b/c/w2", "a/b/c/z2");
+  }
+
+  @Test()
+  public void testNormalStatLocalDirectory() throws IOException {
+    // create local file structure
+    File dir = tmpf.newFolder();
+    File ab = makeDirs(dir.toString(), "a", "b");
+    SortedMap<String, Long> res = DescriptorIndexCollector
+        .statLocalDirectory(dir);
+    assertTrue("found " + res, res.isEmpty());
+    makeFiles(ab, "x");
+    res = DescriptorIndexCollector.statLocalDirectory(dir);
+    assertFalse("found " + res, res.isEmpty());
+    assertEquals("found " + res, 1, res.size());
+    assertNotNull("found " + res, res.get("a/b/x"));
+    File subdir = makeDirs(ab.toString(), "c");
+    makeFiles(subdir, "y");
+    res = DescriptorIndexCollector.statLocalDirectory(dir);
+    assertFalse("found " + res, res.isEmpty());
+    assertEquals("found " + res, 2, res.size());
+    assertNotNull("found " + res, res.get("a/b/x"));
+    assertNotNull("found " + res, res.get("a/b/c/y"));
+    res = DescriptorIndexCollector.statLocalDirectory(new File(subdir, "y"));
+    assertFalse("found " + res, res.isEmpty());
+    assertEquals("found " + res, 1, res.size());
+  }
+
+  @Test()
+  public void testWrongInputStatLocalDirectory() throws IOException {
+    File dir = makeDirs(tmpf.newFolder().toString(), "a", "b");
+    SortedMap<String, Long> res = DescriptorIndexCollector
+        .statLocalDirectory(new File(dir, "not-there"));
+    assertTrue("found " + res, res.isEmpty());
+    dir.setReadable(false);
+    res = DescriptorIndexCollector.statLocalDirectory(dir);
+    assertTrue("found " + res, res.isEmpty());
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testMinimalArgs() throws IOException {
+    File fakeDir = tmpf.newFolder("fantasy-dir");
+    new DescriptorIndexCollector()
+        .collectDescriptors(null, new String[]{}, 100L, fakeDir, true);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testIllegalMillis() {
+    new DescriptorIndexCollector()
+        .collectDescriptors("", new String[]{}, -3L, null, false);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testIllegalDirectory() throws IOException {
+    File fakeDir = tmpf.newFile("fantasy-dir");
+    new DescriptorIndexCollector().collectDescriptors(
+        null, new String[]{}, 100L, fakeDir, false);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNullDirectory() throws IOException {
+    new DescriptorIndexCollector().collectDescriptors(
+        null, new String[]{}, 100L, null, false);
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testExistingFile() throws IOException {
+    File fakeDir = tmpf.newFile("fantasy-dir");
+    new DescriptorIndexCollector()
+        .collectDescriptors(null, null, 100L, fakeDir, false);
+  }
+
+  @Test()
+  public void testExistingDir() throws IOException {
+    File dir = tmpf.newFolder();
+    dir.setWritable(false);
+    SortedMap<String, FileNode> fm = new TreeMap<>();
+    fm.put("readonly", new FileNode("w", 2L, "2100-01-01 01:01"));
+    thrown.expect(RuntimeException.class);
+    thrown.expectMessage("Cannot create dir: " + dir.toString()
+        + "/readonly");
+    new DescriptorIndexCollector()
+        .fetchRemoteFiles(null, fm, 100L, dir, new TreeMap<String, Long>());
+  }
+}
+

src/test/java/org/torproject/descriptor/index/DirectoryNodeTest.java

@@ -0,0 +1,43 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class DirectoryNodeTest {
+
+  @Test()
+  public void testCompare() {
+    DirectoryNode dn1 = new DirectoryNode("a1", null, null);
+    DirectoryNode dn2 = new DirectoryNode("a2", null,
+        new TreeSet<DirectoryNode>());
+    assertEquals(-1, dn1.compareTo(dn2));
+    DirectoryNode dn3 = new DirectoryNode("a1", new TreeSet<FileNode>(),
+        new TreeSet<DirectoryNode>());
+    assertEquals(0, dn1.compareTo(dn3));
+    assertEquals(1, dn2.compareTo(dn3));
+  }
+
+  @Test()
+  public void testFind() {
+    FileNode fnx = new FileNode("x", 0L, "2000-01-01 01:01");
+    SortedSet<FileNode> fm = new TreeSet<>();
+    fm.add(fnx);
+    DirectoryNode dnb = new DirectoryNode("b", fm, null);
+    SortedSet<DirectoryNode> dm1 = new TreeSet<>();
+    dm1.add(dnb);
+    DirectoryNode dna = new DirectoryNode("a", null, dm1);
+    SortedSet<DirectoryNode> dm2 = new TreeSet<>();
+    dm2.add(dna);
+    assertNull(IndexNode.findPathIn("b", dm2));
+    assertEquals(dnb, IndexNode.findPathIn("b", dm1));
+  }
+}
+

src/test/java/org/torproject/descriptor/index/FileNodeTest.java

@@ -0,0 +1,22 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class FileNodeTest {
+
+  @Test()
+  public void testCompare() {
+    FileNode fn1 = new FileNode("a1", 1L, "2016-01-01 01:01");
+    FileNode fn2 = new FileNode("a2", 1L, "2016-01-01 02:02");
+    assertEquals(-1, fn1.compareTo(fn2));
+    FileNode fn3 = new FileNode("a1", 100L, "2016-01-01 03:03");
+    assertEquals(0, fn1.compareTo(fn3));
+    assertEquals(1, fn2.compareTo(fn3));
+  }
+}
+

src/test/java/org/torproject/descriptor/index/IndexNodeTest.java

@@ -0,0 +1,141 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.index;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+import java.util.SortedMap;
+
+public class IndexNodeTest {
+
+  @Rule
+  public TemporaryFolder tmpf = new TemporaryFolder();
+
+  @Test()
+  public void testSimpleIndexRead() throws Exception {
+    URL indexUrl = getClass().getClassLoader().getResource("index1.json");
+    IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+    verifyIndex1(index);
+  }
+
+  /* toString is only used for debugging. Simply ensure that paths,
+   * file names, and urls are readable. */
+  @Test()
+  public void testToString() throws Exception {
+    URL indexUrl = getClass().getClassLoader().getResource("index1.json");
+    IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+    assertTrue(index.toString().contains("archive"));
+    assertTrue(index.toString().contains("file-one.tar.xz"));
+    assertTrue(index.toString().contains("file-two.tar.xz"));
+    assertTrue(index.toString().contains("https://some.collector.url"));
+  }
+
+  private void verifyIndex1(IndexNode index) {
+    assertEquals("https://some.collector.url", index.path);
+    assertEquals("2016-01-01 00:01", index.created);
+    assertEquals("archive", index.directories.first().path);
+    assertEquals("path-one",
+        index.directories.first().directories.first().path);
+    assertEquals("file-one.tar.xz",
+        index.directories.first().directories.first().files.first().path);
+    assertEquals(624_156L,
+        index.directories.first().directories.first().files.first().size);
+    assertEquals("file-two.tar.xz",
+        index.directories.first().directories.first().files.last().path);
+  }
+
+  @Test()
+  public void testCompressedIndexRead() throws Exception {
+    for (String fileName : new String[] {"index1.json.xz", "index1.json.bz2",
+        "index1.json.gz"}) {
+      URL indexUrl = getClass().getClassLoader().getResource(fileName);
+      IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+      verifyIndex1(index);
+    }
+  }
+
+  @Test()
+  public void testIndexWrite() throws Exception {
+    for (String fileName : new String[] {
+        "test.json", "test.json.bz2", "test.json.gz", "test.json.xz"}) {
+      URL indexUrl = getClass().getClassLoader().getResource(fileName);
+      IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+      Path writtenIndex = tmpf.newFile("new" + fileName).toPath();
+      IndexNode.writeIndex(writtenIndex, index);
+      assertTrue("Verifying " + writtenIndex, Files.size(writtenIndex) > 20);
+      compareContents(Paths.get(indexUrl.toURI()), writtenIndex);
+    }
+  }
+
+  private void compareContents(Path oldPath, Path newPath) throws IOException {
+    String oldJson = new String(Files.readAllBytes(oldPath));
+    String newJson = new String(Files.readAllBytes(newPath));
+    assertEquals("Comparing to " + oldPath, oldJson, newJson);
+  }
+
+  @Test()
+  public void testRetrieveFiles() throws Exception {
+    URL indexUrl = getClass().getClassLoader().getResource("index2.json");
+    IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+    assertTrue(index.retrieveFilesIn(new String[]{"b2"}).isEmpty());
+    assertTrue(index.retrieveFilesIn(new String[]{"a1"}).isEmpty());
+    SortedMap<String, FileNode> oneFile
+        = index.retrieveFilesIn(new String[]{"a1/p2"});
+    assertFalse(oneFile.isEmpty());
+    assertEquals(1330787700_000L,
+        oneFile.get("a1/p2/file3").lastModifiedMillis());
+    SortedMap<String, FileNode> twoFiles
+        = index.retrieveFilesIn(new String[]{"y", "a1/x", "a1/p1"});
+    assertEquals(2, twoFiles.size());
+    assertEquals(1328192040_000L,
+        twoFiles.get("a1/p1/file2").lastModifiedMillis());
+    SortedMap<String, FileNode> someFile
+        = index.retrieveFilesIn(new String[]{"a1"});
+    assertTrue(someFile.isEmpty());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testUnknownCompression() throws Exception {
+    URL indexUrl = getClass()
+        .getClassLoader().getResource("unknown.compression");
+    IndexNode.fetchIndex(indexUrl.toString());
+  }
+
+  @Test(expected = com.google.gson.JsonSyntaxException.class)
+  public void testWrongJson() throws Exception {
+    URL indexUrl = getClass().getClassLoader().getResource("index1.json.gz");
+    IndexNode.fetchIndex(indexUrl.openStream());
+  }
+
+  @Test()
+  public void testRetrieveEmpty() throws Exception {
+    URL indexUrl = getClass().getClassLoader().getResource("index1.json");
+    IndexNode index = IndexNode.fetchIndex(indexUrl.toString());
+    Map<String, FileNode> map = index.retrieveFilesIn(new String[]{});
+    assertTrue("map was " + map, map.isEmpty());
+    map = index.retrieveFilesIn(new String[]{});
+    assertTrue("map was " + map, map.isEmpty());
+    map = index.retrieveFilesIn(new String[]{"/", null, ""});
+    assertTrue("map was " + map, map.isEmpty());
+    indexUrl = getClass().getClassLoader().getResource("index3.json");
+    index = IndexNode.fetchIndex(indexUrl.toString());
+    map = index.retrieveFilesIn(new String[]{"a1/p1"});
+    assertTrue("map was " + map, map.isEmpty());
+    map = index.retrieveFilesIn(new String[]{"a1/p3"});
+    assertTrue("map was " + map, map.isEmpty());
+  }
+}
+

src/test/resources/index1.json

@@ -0,0 +1 @@
+{"index_created":"2016-01-01 00:01","path":"https://some.collector.url","directories":[{"path":"archive","directories":[{"path":"path-one","files":[{"path":"file-one.tar.xz","size":624156,"last_modified":"2012-05-30 19:41"},{"path":"file-two.tar.xz","size":1010648,"last_modified":"2012-05-30 19:41"}]},{"path":"path-two","files":[{"path":"file-three.tar.xz","size":624156,"last_modified":"2012-05-30 19:41"}]}]}]}

src/test/resources/index2.json

@@ -0,0 +1 @@
+{"index_created":"2016-02-02 00:02","path":"https://some.collector.url","directories":[{"path":"a1","directories":[{"path":"p1","files":[{"path":"file1","size":624156,"last_modified":"2012-01-01 13:13"},{"path":"file2","size":1010648,"last_modified":"2012-02-02 14:14"}]},{"path":"p2","files":[{"path":"file3","size":624156,"last_modified":"2012-03-03 15:15"}]}]}]}

src/test/resources/index3.json

@@ -0,0 +1 @@
+{"index_created":"2016-02-02 00:02","path":"https://some.collector.url","directories":[{"path":"a1","directories":[{"path":"p1","files":[{"path":"file1","size":624156,"last_modified":"2012-01-01 77:xy"}]},{"path":"p2"}]}]}

src/test/resources/indexNormal.json

@@ -0,0 +1 @@
+{"index_created":"2016-02-02 00:02","path":"https://some.collector.url","directories":[{"path":"a1","directories":[{"path":"p1","files":[{"path":"file1","size":624156,"last_modified":"2012-01-01 77:xy"}]},{"path":"p2"}]}]}