torproject/collector
1
0
Fork 0
mirror of https://github.com/torproject/collector.git synced 2024-11-23 01:19:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove dependency on metrics-lib's log package (2/4).

Browse Source 
- Remove unused code.
This commit is contained in:
Karsten Loesing 2019-11-23 17:43:45 +01:00
parent 859476ecae
commit ea1b1b4f6a
6 changed files with 4 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java
View File

@ -7,9 +7,6 @@ import org.torproject.descriptor.WebServerAccessLog;
import org.torproject.metrics.collector.webstats.FileType;
import org.torproject.metrics.collector.webstats.InternalWebServerAccessLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.time.format.DateTimeFormatter;
@ -19,8 +16,6 @@ public class WebServerAccessLogPersistence
public static final String SEP = InternalWebServerAccessLog.SEP;
public static final FileType COMPRESSION = FileType.XZ;
private static final Logger log
= LoggerFactory.getLogger(WebServerAccessLogPersistence.class);
private DateTimeFormatter yearPattern = DateTimeFormatter.ofPattern("yyyy");
private DateTimeFormatter monthPattern = DateTimeFormatter.ofPattern("MM");

35
src/main/java/org/torproject/metrics/collector/webstats/InternalLogDescriptor.java
View File

@ -3,7 +3,6 @@
package org.torproject.metrics.collector.webstats;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.LogDescriptor;
/**
@ -16,23 +15,6 @@ public interface InternalLogDescriptor extends LogDescriptor {
/** Logfile name parts separator. */
String SEP = "_";
/**
* Validate log lines.
*
* @since 2.2.0
*/
void validate() throws DescriptorParseException;
/**
* Set the {@code Validator} that will perform the validation on log
* lines.
*
* <p>Usually set by the implementing class.</p>
*
* @since 2.2.0
*/
void setValidator(Validator validator);
/**
* Set the descriptor's bytes.
*
@ -42,22 +24,5 @@ public interface InternalLogDescriptor extends LogDescriptor {
/** Return the descriptor's preferred compression. */
String getCompressionType();
/**
* Provides a single function for validating a single log line.
*
* @since 2.2.0
*/
interface Validator {
/**
* Verifies a log line.
*
* @since 2.2.0
*/
boolean validate(String line);
}
}

53
src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java
View File

@ -3,25 +3,17 @@
package org.torproject.metrics.collector.webstats;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.LogDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Base class for log descriptors.
@ -34,11 +26,6 @@ public abstract class LogDescriptorImpl
/** The log's file name should contain this string. */
public static final String MARKER = ".log";
private static final int unrecognizedLinesLimit = 3;
private static final Logger log
= LoggerFactory.getLogger(LogDescriptorImpl.class);
private static Pattern filenamePattern = Pattern.compile(
"(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)");
@ -51,8 +38,6 @@ public abstract class LogDescriptorImpl
private List<String> unrecognizedLines = new ArrayList<>();
private Validator validator = (String line) -> true;
/**
* This constructor performs basic operations on the given bytes.
*
@ -63,8 +48,7 @@ public abstract class LogDescriptorImpl
* @since 2.2.0
*/
protected LogDescriptorImpl(byte[] logBytes, File descriptorFile,
String logName, FileType defaultCompression)
throws DescriptorParseException {
String logName) throws DescriptorParseException {
this.logBytes = logBytes;
this.descriptorFile = descriptorFile;
try {
@ -75,7 +59,7 @@ public abstract class LogDescriptorImpl
}
this.fileType = FileType.findType(mat.group(1).toUpperCase());
if (FileType.PLAIN == this.fileType) {
this.fileType = defaultCompression;
this.fileType = FileType.XZ;
this.logBytes = this.fileType.compress(this.logBytes);
}
} catch (Exception ex) {
@ -94,39 +78,6 @@ public abstract class LogDescriptorImpl
}
}
@Override
public void validate() throws DescriptorParseException {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(decompressedByteStream()))) {
this.unrecognizedLines.addAll(br.lines().parallel().filter((line)
-> null != line && !line.isEmpty() && !validator.validate(line))
.limit(unrecognizedLinesLimit).collect(Collectors.toList()));
} catch (Exception ex) {
throw new DescriptorParseException("Cannot validate log lines.", ex);
}
}
/**
* Assemble a LogDescriptor.
*
* @since 2.2.0
*/
public static List<Descriptor> parse(byte[] logBytes,
File descriptorFile, String logName) throws DescriptorParseException {
if (logName.contains(InternalWebServerAccessLog.MARKER)) {
return Arrays.asList(new Descriptor[]{
new WebServerAccessLogImpl(logBytes, descriptorFile, logName)});
} else {
throw new DescriptorParseException("Cannot parse file " + logName
+ " from file " + descriptorFile.getName());
}
}
@Override
public void setValidator(Validator validator) {
this.validator = validator;
}
@Override
public String getCompressionType() {
return this.fileType.name().toLowerCase();

26
src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java
View File

@ -15,7 +15,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.time.LocalDate;
import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Stream;
/**
* Processes the given path and stores metadata for log files.
@ -74,30 +73,5 @@ public class LogFileMap
k -> new TreeMap<>());
physicalHosts.put(metadata.date, metadata);
}
/**
* Takes the given metadata and returns the LogMetadata for the entry
* of the next day.
*/
public Optional<LogMetadata> nextDayLogFor(LogMetadata metadata) {
TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts
= this.get(metadata.virtualHost);
if (null == virtualHosts) {
return Optional.empty();
}
TreeMap<LocalDate, LogMetadata> physicalHosts
= virtualHosts.get(metadata.physicalHost);
if (null == physicalHosts) {
return Optional.empty();
}
return Optional.ofNullable(physicalHosts.get(metadata.date.plusDays(1)));
}
/** Returns a stream of all contained log metadata. */
public Stream<LogMetadata> metadataStream() {
return this.values().stream()
.flatMap((virtualHosts) -> virtualHosts.values().stream())
.flatMap((physicalHosts) -> physicalHosts.values().stream());
}
}

2
src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
View File

@ -147,7 +147,7 @@ public class SanitizeWeblogs extends CollecTorMain {
WebServerAccessLogPersistence walp
= new WebServerAccessLogPersistence(
new WebServerAccessLogImpl(toCompressedBytes(retainedLines),
new File(name), name, false));
new File(name), name));
log.debug("Storing {}.", name);
walp.storeOut(this.outputPathName);
walp.storeRecent(this.recentPathName);

38
src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java
View File

@ -6,9 +6,6 @@ package org.torproject.metrics.collector.webstats;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.WebServerAccessLog;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
@ -32,9 +29,6 @@ import java.util.stream.Stream;
public class WebServerAccessLogImpl extends LogDescriptorImpl
implements InternalWebServerAccessLog, WebServerAccessLog {
private static final Logger log
= LoggerFactory.getLogger(WebServerAccessLogImpl.class);
/** The log's name should include this string. */
public static final String MARKER = InternalWebServerAccessLog.MARKER;
@ -49,8 +43,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
private final LocalDate logDate;
private boolean validate = true;
/**
* Creates a WebServerAccessLog from the given bytes and filename.
*
@ -68,30 +60,7 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
*/
protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName)
throws DescriptorParseException {
this(logBytes, file, logName, FileType.XZ);
}
/** For internal use only. */
public WebServerAccessLogImpl(byte[] bytes, String filename,
boolean validate) throws DescriptorParseException {
this(bytes, null, filename, FileType.XZ, validate);
}
/** For internal use only. */
public WebServerAccessLogImpl(byte[] bytes, File sourceFile, String filename,
boolean validate) throws DescriptorParseException {
this(bytes, sourceFile, filename, FileType.XZ, validate);
}
private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
FileType defaultCompression) throws DescriptorParseException {
this(logBytes, file, logName, defaultCompression, true);
}
private WebServerAccessLogImpl(byte[] logBytes, File file, String logName,
FileType defaultCompression, boolean validate)
throws DescriptorParseException {
super(logBytes, file, logName, defaultCompression);
super(logBytes, file, logName);
try {
String fn = Paths.get(logName).getFileName().toString();
Matcher mat = filenamePattern.matcher(fn);
@ -108,11 +77,6 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl
}
String ymd = mat.group(3);
this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE);
this.setValidator((line)
-> WebServerAccessLogLine.makeLine(line).isValid());
if (validate) {
this.validate();
}
} catch (DescriptorParseException dpe) {
throw dpe; // escalate
} catch (Exception pe) {