torproject/collector
1
0
Fork 0
mirror of https://github.com/torproject/collector.git synced 2025-02-20 17:42:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make local paths configurable.

Browse Source
This commit is contained in:
Karsten Loesing 2010-04-13 10:50:12 +02:00
parent c06be6d499
commit 50835aebb4
6 changed files with 140 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
config
View File

@ -3,9 +3,16 @@
## Read cached-* files from a local Tor data directory
#ImportCachedRelayDescriptors 0
#
## Relative path to Tor data directory to read cached-* files from (can be
## specified multiple times)
#CachedRelayDescriptorsDirectory cacheddesc/
#
## Import directory archives from disk, if available
#ImportDirectoryArchives 0
#
## Relative path to directory to import directory archives from
#DirectoryArchivesDirectory archives/
#
## Keep a history of imported directory archive files to know which files
## have been imported before. This history can be useful when importing
## from a changing source to avoid importing descriptors over and over
@ -22,12 +29,21 @@
## Import sanitized bridges from disk, if available
#ImportSanitizedBridges 0
#
## Relative path to directory to import sanitized bridges from
#SanitizedBridgesDirectory bridges/
#
## Import bridge snapshots from disk, if available
#ImportBridgeSnapshots 0
#
## Relative path to directory to import bridge descriptor snapshots from
#BridgeSnapshotsDirectory bridge-directories/
#
## Import local Maxmind GeoIP databases
#ImportGeoIPDatabases 0
#
## Relative path to directory to import GeoIP databases from
#GeoIPDatabasesDirectory geoipdb/
#
## Download (commercial) Maxmind GeoIP database for sanitizing bridge
## descriptors
#DownloadGeoIPDatabase 0
@ -38,6 +54,9 @@
## Import torperf data, if available, and write stats to disk
#ImportWriteTorperfStats 0
#
## Relate path to directory to import torperf results from
#TorperfDirectory torperf/
#
## Download and process GetTor stats
#DownloadProcessGetTorStats 0
#
@ -52,6 +71,9 @@
## Write directory archives to disk
#WriteDirectoryArchives 0
#
## Relative path to directory to write directory archives to
#DirectoryArchivesOutputDirectory directory-archive/
#
## Write relay descriptors to a database for later evaluation
#WriteRelayDescriptorDatabase 0
#
@ -61,6 +83,9 @@
## Write sanitized bridges to disk
#WriteSanitizedBridges 0
#
## Relate path to directory to write sanitized bridges to
#SanitizedBridgesWriteDirectory sanitized-bridges/
#
## Write consensus stats to disk
#WriteConsensusStats 0
#

34
src/ArchiveWriter.java
View File

@ -7,8 +7,10 @@ import org.apache.commons.codec.binary.*;
public class ArchiveWriter {
private Logger logger;
public ArchiveWriter() {
private String outputDirectory;
public ArchiveWriter(String outputDirectory) {
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
this.outputDirectory = outputDirectory;
}
private void store(byte[] data, String filename) {
@ -32,7 +34,7 @@ public class ArchiveWriter {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String filename = "directory-archive/consensus/"
String filename = outputDirectory + "/consensus/"
+ printFormat.format(new Date(validAfter)) + "-consensus";
this.store(data, filename);
}
@ -42,7 +44,7 @@ public class ArchiveWriter {
SimpleDateFormat printFormat = new SimpleDateFormat(
"yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String filename = "directory-archive/vote/"
String filename = outputDirectory + "/vote/"
+ printFormat.format(new Date(validAfter)) + "-vote-"
+ fingerprint + "-" + digest;
this.store(data, filename);
@ -52,7 +54,7 @@ public class ArchiveWriter {
long published) {
SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String filename = "directory-archive/server-descriptor/"
String filename = outputDirectory + "/server-descriptor/"
+ printFormat.format(new Date(published))
+ digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ digest;
@ -63,7 +65,7 @@ public class ArchiveWriter {
String extraInfoDigest, long published) {
SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
String filename = "directory-archive/extra-info/"
String filename = outputDirectory + "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
@ -84,7 +86,7 @@ public class ArchiveWriter {
} else if (pop.length() > 0) {
String absPath = pop.getAbsolutePath().replaceAll(":", "-");
String relPath = absPath.substring(absPath.indexOf(
"directory-archive/"));
outputDirectory + "/"));
files.add(relPath);
}
}
@ -98,13 +100,13 @@ public class ArchiveWriter {
public void dumpStats() {
try {
SortedSet<String> votes = getFileNames(
new File("directory-archive/vote"));
new File(outputDirectory + "/vote"));
SortedSet<String> serverDescs = getFileNames(
new File("directory-archive/server-descriptor"));
new File(outputDirectory + "/server-descriptor"));
SortedSet<String> extraInfos = getFileNames(
new File("directory-archive/extra-info"));
new File(outputDirectory + "/extra-info"));
SortedSet<String> consensuses = getFileNames(
new File("directory-archive/consensus"));
new File(outputDirectory + "/consensus"));
SimpleDateFormat validAfterFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@ -134,7 +136,7 @@ public class ArchiveWriter {
validAfterTime = line.substring("valid-after ".length());
long validAfter = validAfterFormat.parse(
validAfterTime).getTime();
votePrefix = "directory-archive/vote/"
votePrefix = outputDirectory + "/vote/"
+ consensusVoteFormat.format(new Date(validAfter))
+ "-vote-";
} else if (line.startsWith("dir-source ")) {
@ -162,7 +164,8 @@ public class ArchiveWriter {
long published = validAfterFormat.parse(
line3.split(" ")[4] + " "
+ line3.split(" ")[5]).getTime();
String filename = "directory-archive/server-descriptor/"
String filename = outputDirectory
+ "/server-descriptor/"
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
@ -177,7 +180,8 @@ public class ArchiveWriter {
String extraInfoDigest = line2.startsWith("opt ") ?
line2.split(" ")[2].toLowerCase() :
line2.split(" ")[1].toLowerCase();
String filename2 = "directory-archive/extra-info/"
String filename2 = outputDirectory
+ "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"
@ -209,7 +213,7 @@ public class ArchiveWriter {
line.split(" ")[3] + "=")).toLowerCase();
long published = validAfterFormat.parse(
line.split(" ")[4] + " " + line.split(" ")[5]).getTime();
String filename = "directory-archive/server-descriptor/"
String filename = outputDirectory + "/server-descriptor/"
+ descriptorFormat.format(new Date(published))
+ digest.substring(0, 1) + "/"
+ digest.substring(1, 2) + "/" + digest;
@ -224,7 +228,7 @@ public class ArchiveWriter {
String extraInfoDigest = line2.startsWith("opt ") ?
line2.split(" ")[2].toLowerCase() :
line2.split(" ")[1].toLowerCase();
String filename2 = "directory-archive/extra-info/"
String filename2 = outputDirectory + "/extra-info/"
+ descriptorFormat.format(new Date(published))
+ extraInfoDigest.substring(0, 1) + "/"
+ extraInfoDigest.substring(1, 2) + "/"

26
src/CachedRelayDescriptorReader.java
View File

@ -8,12 +8,19 @@ import java.util.logging.*;
* into directory structure in directory-archive/.
*/
public class CachedRelayDescriptorReader {
public CachedRelayDescriptorReader(RelayDescriptorParser rdp) {
public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
List<String> inputDirectories) {
Logger logger = Logger.getLogger(
CachedRelayDescriptorReader.class.getName());
File cachedDescDir = new File("cacheddesc");
if (cachedDescDir.exists()) {
logger.fine("Reading cacheddesc/ directory.");
for (String inputDirectory : inputDirectories) {
File cachedDescDir = new File(inputDirectory);
if (!cachedDescDir.exists()) {
logger.warning("Directory " + cachedDescDir.getAbsolutePath()
+ " does not exist. Skipping.");
continue;
}
logger.fine("Reading " + cachedDescDir.getAbsolutePath()
+ " directory.");
for (File f : cachedDescDir.listFiles()) {
try {
// descriptors may contain non-ASCII chars; read as bytes to
@ -86,14 +93,15 @@ public class CachedRelayDescriptorReader {
rdp.parse(descBytes);
}
}
logger.fine("Finished reading cacheddesc/ directory.");
logger.fine("Finished reading "
+ cachedDescDir.getAbsolutePath() + " directory.");
}
} catch (IOException e) {
logger.log(Level.WARNING, "Failed reading cacheddesc/ "
+ "directory.", e);
logger.log(Level.WARNING, "Failed reading "
+ cachedDescDir.getAbsolutePath() + " directory.", e);
} catch (ParseException e) {
logger.log(Level.WARNING, "Failed reading cacheddesc/ "
+ "directory.", e);
logger.log(Level.WARNING, "Failed reading "
+ cachedDescDir.getAbsolutePath() + " directory.", e);
}
}
}

55
src/Configuration.java
View File

@ -23,16 +23,25 @@ public class Configuration {
private List<String> relayPlatforms = new ArrayList<String>(
Arrays.asList("Linux,Windows,Darwin,FreeBSD".split(",")));
private boolean writeDirectoryArchives = false;
private String directoryArchivesOutputDirectory = "directory-archive/";
private boolean importCachedRelayDescriptors = false;
//this.cachedRelayDescriptorDirectory.add
private List<String> cachedRelayDescriptorsDirectory =
new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
private boolean importDirectoryArchives = false;
private String directoryArchivesDirectory = "archives/";
private boolean keepDirectoryArchiveImportHistory = false;
private boolean writeRelayDescriptorDatabase = false;
private String relayDescriptorDatabaseJdbc =
"jdbc:postgresql://localhost/tordir?user=ernie&password=password";
private boolean writeSanitizedBridges = false;
private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
private boolean importSanitizedBridges = false;
private String sanitizedBridgesDirectory = "bridges/";
private boolean importBridgeSnapshots = false;
private String bridgeSnapshotsDirectory = "bridge-directories/";
private boolean importWriteTorperfStats = false;
private String torperfDirectory = "torperf/";
private boolean downloadRelayDescriptors = false;
private List<String> downloadFromDirectoryAuthorities = Arrays.asList(
"86.59.21.38,194.109.206.212,80.190.246.100:8180".split(","));
@ -41,6 +50,7 @@ public class Configuration {
+ "~gettor/gettor_stats.txt";
private boolean downloadExitList = false;
private boolean importGeoIPDatabases = false;
private String geoIPDatabasesDirectory = "geoipdb/";
private boolean downloadGeoIPDatabase = false;
private String maxmindLicenseKey = "";
public Configuration() {
@ -60,6 +70,7 @@ public class Configuration {
return;
}
String line = null;
boolean containsCachedRelayDescriptorsDirectory = false;
try {
BufferedReader br = new BufferedReader(new FileReader(configFile));
while ((line = br.readLine()) != null) {
@ -99,12 +110,22 @@ public class Configuration {
} else if (line.startsWith("WriteDirectoryArchives")) {
this.writeDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
this.directoryArchivesOutputDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportCachedRelayDescriptors")) {
this.importCachedRelayDescriptors = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
if (!containsCachedRelayDescriptorsDirectory) {
this.cachedRelayDescriptorsDirectory.clear();
containsCachedRelayDescriptorsDirectory = true;
}
this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
} else if (line.startsWith("ImportDirectoryArchives")) {
this.importDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("DirectoryArchivesDirectory")) {
this.directoryArchivesDirectory = line.split(" ")[1];
} else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
this.keepDirectoryArchiveImportHistory = Integer.parseInt(
line.split(" ")[1]) != 0;
@ -116,15 +137,23 @@ public class Configuration {
} else if (line.startsWith("WriteSanitizedBridges")) {
this.writeSanitizedBridges = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportSanitizedBridges")) {
this.importSanitizedBridges = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("SanitizedBridgesDirectory")) {
this.sanitizedBridgesDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportBridgeSnapshots")) {
this.importBridgeSnapshots = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("BridgeSnapshotsDirectory")) {
this.bridgeSnapshotsDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportWriteTorperfStats")) {
this.importWriteTorperfStats = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("TorperfDirectory")) {
this.torperfDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadRelayDescriptors")) {
this.downloadRelayDescriptors = Integer.parseInt(
line.split(" ")[1]) != 0;
@ -155,6 +184,8 @@ public class Configuration {
} else if (line.startsWith("ImportGeoIPDatabases")) {
this.importGeoIPDatabases = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("GeoIPDatabasesDirectory")) {
this.geoIPDatabasesDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadGeoIPDatabase")) {
this.downloadGeoIPDatabase = Integer.parseInt(
line.split(" ")[1]) != 0;
@ -264,12 +295,21 @@ public class Configuration {
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
}
public String getDirectoryArchivesOutputDirectory() {
return this.directoryArchivesOutputDirectory;
}
public boolean getImportCachedRelayDescriptors() {
return this.importCachedRelayDescriptors;
}
public List<String> getCachedRelayDescriptorDirectory() {
return this.cachedRelayDescriptorsDirectory;
}
public boolean getImportDirectoryArchives() {
return this.importDirectoryArchives;
}
public String getDirectoryArchivesDirectory() {
return this.directoryArchivesDirectory;
}
public boolean getKeepDirectoryArchiveImportHistory() {
return this.keepDirectoryArchiveImportHistory;
}
@ -282,15 +322,27 @@ public class Configuration {
public boolean getWriteSanitizedBridges() {
return this.writeSanitizedBridges;
}
public String getSanitizedBridgesWriteDirectory() {
return this.sanitizedBridgesWriteDirectory;
}
public boolean getImportSanitizedBridges() {
return this.importSanitizedBridges;
}
public String getSanitizedBridgesDirectory() {
return this.sanitizedBridgesDirectory;
}
public boolean getImportBridgeSnapshots() {
return this.importBridgeSnapshots;
}
public String getBridgeSnapshotsDirectory() {
return this.bridgeSnapshotsDirectory;
}
public boolean getImportWriteTorperfStats() {
return this.importWriteTorperfStats;
}
public String getTorperfDirectory() {
return this.torperfDirectory;
}
public boolean getDownloadRelayDescriptors() {
return this.downloadRelayDescriptors;
}
@ -309,6 +361,9 @@ public class Configuration {
public boolean getImportGeoIPDatabases() {
return this.importGeoIPDatabases;
}
public String getGeoIPDatabasesDirectory() {
return this.geoIPDatabasesDirectory;
}
public boolean getDownloadGeoIPDatabase() {
return this.downloadGeoIPDatabase;
}

15
src/GeoIPDatabaseManager.java
View File

@ -68,6 +68,8 @@ public class GeoIPDatabaseManager {
*/
private String lastDownloadedTime;
private String geoipDir;
/**
* Logger for this class.
*/
@ -79,9 +81,10 @@ public class GeoIPDatabaseManager {
* Initializes this class by reading in the database versions known so
* far.
*/
public GeoIPDatabaseManager() {
public GeoIPDatabaseManager(String geoipDir) {
/* Initialize instance variables. */
this.geoipDir = geoipDir;
this.combinedDatabaseFile = new File("stats/geoip-database");
this.combinedDatabase = new TreeMap<Long, DatabaseEntry>();
this.allDatabases = new ArrayList<String>();
@ -192,7 +195,7 @@ public class GeoIPDatabaseManager {
while ((entry = zis.getNextEntry()) != null) {
if (!entry.isDirectory() &&
entry.getName().endsWith("GeoIP-108.csv")) {
String filename = "geoipdb/GeoIP-108_" + date + ".csv";
String filename = geoipDir + "/GeoIP-108_" + date + ".csv";
OutputStream out = new BufferedOutputStream(
new FileOutputStream(filename));
byte[] buffer = new byte[1024];
@ -212,11 +215,11 @@ public class GeoIPDatabaseManager {
}
/**
* Imports the GeoIP databases from <code>directory</code> to include
* them in the combined GeoIP database.
* Imports the GeoIP databases to include them in the combined GeoIP
* database.
*/
public void importGeoIPDatabaseFromDisk(String directory) {
File databaseDirectory = new File(directory);
public void importGeoIPDatabaseFromDisk() {
File databaseDirectory = new File(this.geoipDir);
if (!databaseDirectory.exists()) {
return;
}

24
src/Main.java
View File

@ -40,7 +40,8 @@ public class Main {
// Prepare writing relay descriptor archive to disk
ArchiveWriter aw = config.getWriteDirectoryArchives() ?
new ArchiveWriter() : null;
new ArchiveWriter(config.getDirectoryArchivesOutputDirectory())
: null;
// Prepare writing relay descriptors to database
RelayDescriptorDatabaseImporter rddi =
@ -78,10 +79,11 @@ public class Main {
rdp.setRelayDescriptorDownloader(rdd);
}
if (config.getImportCachedRelayDescriptors()) {
new CachedRelayDescriptorReader(rdp);
new CachedRelayDescriptorReader(rdp,
config.getCachedRelayDescriptorDirectory());
}
if (config.getImportDirectoryArchives()) {
new ArchiveReader(rdp, "archives",
new ArchiveReader(rdp, config.getDirectoryArchivesDirectory(),
config.getKeepDirectoryArchiveImportHistory());
}
if (rdd != null) {
@ -105,18 +107,20 @@ public class Main {
}
// Import/download GeoIP databases
GeoIPDatabaseManager gd = new GeoIPDatabaseManager();
GeoIPDatabaseManager gd = new GeoIPDatabaseManager(
config.getGeoIPDatabasesDirectory());
if (config.getDownloadGeoIPDatabase()) {
gd.downloadGeoIPDatabase(config.getMaxmindLicenseKey());
}
if (config.getImportGeoIPDatabases()) {
gd.importGeoIPDatabaseFromDisk("geoipdb/");
gd.importGeoIPDatabaseFromDisk();
gd.writeCombinedDatabase();
}
// Prepare sanitized bridge descriptor writer
SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
new SanitizedBridgesWriter(gd, "sanitized-bridges") : null;
new SanitizedBridgesWriter(gd,
config.getSanitizedBridgesWriteDirectory()) : null;
// Prepare bridge descriptor parser
BridgeDescriptorParser bdp = config.getWriteConsensusStats() ||
@ -125,10 +129,12 @@ public class Main {
// Import bridge descriptors
if (bdp != null && config.getImportSanitizedBridges()) {
new SanitizedBridgesReader(bdp, "bridges", countries);
new SanitizedBridgesReader(bdp,
config.getSanitizedBridgesDirectory(), countries);
}
if (bdp != null && config.getImportBridgeSnapshots()) {
new BridgeSnapshotReader(bdp, "bridge-directories", countries);
new BridgeSnapshotReader(bdp, config.getBridgeSnapshotsDirectory(),
countries);
}
// Finish writing sanitized bridge descriptors to disk
@ -149,7 +155,7 @@ public class Main {
// Import and process torperf stats
if (config.getImportWriteTorperfStats()) {
new TorperfProcessor("torperf");
new TorperfProcessor(config.getTorperfDirectory());
}
// Download and process GetTor stats