Skip to content
Commits on Source (7)
Show whitespace changes
Inline Side-by-side
GoogleChartsExample/plotFiles.html
View file @ ab74d8f4
......@@ -188,7 +188,6 @@
var names = [];
var histos = [];
var fileCount = 0;
fileDisplayArea.innerText = "file selected...\n";
......@@ -375,7 +374,7 @@ Percentile range:
<script type="text/javascript">
function showValue(newValue) {
var x = Math.pow(10, newValue);
percentile = 100.0 - (100.0 / x);
var percentile = 100.0 - (100.0 / x);
document.getElementById("percentileRange").innerHTML=percentile + "%";
maxPercentile = x;
drawChart();
......
README.md
View file @ ab74d8f4
......@@ -2,15 +2,19 @@ HdrHistogram
----------------------------------------------
[![Gitter](https://img.shields.io/gitter/room/gitterHQ/gitter.svg)](https://gitter.im/HdrHistogram/HdrHistogram?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Build Status](https://travis-ci.org/HdrHistogram/HdrHistogram.svg?branch=master)](https://travis-ci.org/HdrHistogram/HdrHistogram)
[![Javadoc](http://javadoc-emblem.rhcloud.com/doc/org.hdrhistogram/HdrHistogram/badge.svg)](http://www.javadoc.io/doc/org.hdrhistogram/HdrHistogram)
[![Javadocs](http://www.javadoc.io/badge/org.hdrhistogram/HdrHistogram.svg)](http://www.javadoc.io/doc/org.hdrhistogram/HdrHistogram)
[![Total alerts](https://img.shields.io/lgtm/alerts/g/HdrHistogram/HdrHistogram.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HdrHistogram/HdrHistogram/alerts/)
[![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/HdrHistogram/HdrHistogram.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/HdrHistogram/HdrHistogram/context:java)
----------------------------------------------------------------------------
HdrHistogram: A High Dynamic Range (HDR) Histogram
This respository currently includes Java and C# implementations of
HdrHistogram, C, Python, Erlang, and Go ports can be found in other
respositories. All of which share common concepts and data
representation capabilities.
This respository currently includes a Java implementation of
HdrHistogram. C, C#/.NET, Python, Javascript, Rust, Erlang, and Go ports
can be found in other respositories. All of which share common concepts
and data representation capabilities. Look at repositories under the
[HdrHistogram organization](https://github.com/HdrHistogram) for various
implementations and useful tools.
Note: The below is an excerpt from a Histogram JavaDoc. While much
of it generally applies to other language implementations as well,
......
debian/changelog
View file @ ab74d8f4
hdrhistogram (2.1.11-1) unstable; urgency=medium
* New upstream release
* Standards-Version updated to 4.3.0
* Switch to debhelper level 11
* Use salsa.debian.org Vcs-* URLs
-- Emmanuel Bourg <ebourg@apache.org> Sun, 20 Jan 2019 00:12:04 +0100
hdrhistogram (2.1.10-1) unstable; urgency=medium
* New upstream release (Closes: #875367)
......
debian/control
View file @ ab74d8f4
......@@ -4,16 +4,16 @@ Priority: optional
Maintainer: Debian Java Maintainers <pkg-java-maintainers@lists.alioth.debian.org>
Uploaders: Emmanuel Bourg <ebourg@apache.org>
Build-Depends:
debhelper (>= 10),
debhelper (>= 11),
default-jdk,
junit4,
libmaven-bundle-plugin-java,
libmaven-dependency-plugin-java,
libreplacer-java,
maven-debian-helper (>= 2.1)
Standards-Version: 4.1.0
Vcs-Git: https://anonscm.debian.org/git/pkg-java/hdrhistogram.git
Vcs-Browser: https://anonscm.debian.org/cgit/pkg-java/hdrhistogram.git
Standards-Version: 4.3.0
Vcs-Git: https://salsa.debian.org/java-team/hdrhistogram.git
Vcs-Browser: https://salsa.debian.org/java-team/hdrhistogram
Homepage: http://hdrhistogram.github.io/HdrHistogram/
Package: libhdrhistogram-java
......
debian/maven.ignoreRules
View file @ ab74d8f4
......@@ -3,3 +3,4 @@ org.apache.maven.plugins maven-javadoc-plugin * * * *
org.apache.maven.plugins maven-release-plugin * * * *
org.apache.maven.plugins maven-source-plugin * * * *
org.apache.maven.plugins maven-surefire-plugin * * * *
org.sonatype.plugins nexus-staging-maven-plugin * * * *
debian/rules
View file @ ab74d8f4
......@@ -2,6 +2,3 @@
%:
dh $@ --buildsystem=maven
get-orig-source:
uscan --download-current-version --force-download --no-symlink
debian/watch
View file @ ab74d8f4
version=3
version=4
opts="repack,compression=xz" \
https://github.com/HdrHistogram/HdrHistogram/tags .*/(?:.*?)([\d\.]+)\.tar\.gz
pom.xml
View file @ ab74d8f4
......@@ -2,15 +2,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
<version>2.1.10</version>
<version>2.1.11</version>
<name>HdrHistogram</name>
......@@ -52,7 +46,7 @@
<url>scm:git:git://github.com/HdrHistogram/HdrHistogram.git</url>
<connection>scm:git:git://github.com/HdrHistogram/HdrHistogram.git</connection>
<developerConnection>scm:git:git@github.com:HdrHistogram/HdrHistogram.git</developerConnection>
<tag>HdrHistogram-2.1.10</tag>
<tag>HdrHistogram-2.1.11</tag>
</scm>
......@@ -127,7 +121,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<version>3.8.0</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
......@@ -145,9 +139,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>2.5</version>
<version>2.5.3</version>
<configuration>
<arguments>-Dgpg.passphrase=${gpg.passphrase}</arguments>
<autoVersionSubmodules>true</autoVersionSubmodules>
<useReleaseProfile>false</useReleaseProfile>
<releaseProfiles>release</releaseProfiles>
<goals>deploy</goals>
</configuration>
</plugin>
<plugin>
......@@ -203,12 +200,44 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.7</version>
<extensions>true</extensions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release-sign-artifacts</id>
<id>deploy</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>release</id>
<activation>
<property>
<name>performRelease</name>
......@@ -220,10 +249,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.4</version>
<configuration>
<passphrase>${gpg.passphrase}</passphrase>
</configuration>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
......
src/main/java/org/HdrHistogram/AbstractHistogram.java
View file @ ab74d8f4
......@@ -70,10 +70,12 @@ abstract class AbstractHistogramBase extends EncodableHistogram {
return doubleToIntegerValueConversionRatio;
}
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
void nonConcurrentSetIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
this.integerToDoubleValueConversionRatio = integerToDoubleValueConversionRatio;
this.doubleToIntegerValueConversionRatio = 1.0/integerToDoubleValueConversionRatio;
}
abstract void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio);
}
/**
......@@ -95,7 +97,7 @@ abstract class AbstractHistogramBase extends EncodableHistogram {
* See package description for {@link org.HdrHistogram} for details.
*
*/
public abstract class AbstractHistogram extends AbstractHistogramBase implements Serializable {
public abstract class AbstractHistogram extends AbstractHistogramBase implements ValueRecorder, Serializable {
// "Hot" accessed fields (used in the the value recording code path) are bunched here, such
// that they will have a good chance of ending up in the same cache line as the totalCounts and
......@@ -422,6 +424,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
* @param value The value to be recorded
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValue(final long value) throws ArrayIndexOutOfBoundsException {
recordSingleValue(value);
}
......@@ -433,6 +436,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
* @param count The number of occurrences of this value to record
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValueWithCount(final long value, final long count) throws ArrayIndexOutOfBoundsException {
recordCountAtValue(count, value);
}
......@@ -458,6 +462,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
* than expectedIntervalBetweenValueSamples
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValueWithExpectedInterval(final long value, final long expectedIntervalBetweenValueSamples)
throws ArrayIndexOutOfBoundsException {
recordSingleValueWithExpectedInterval(value, expectedIntervalBetweenValueSamples);
......@@ -529,7 +534,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
private void handleRecordException(final long count, final long value, Exception ex) {
if (!autoResize) {
throw new ArrayIndexOutOfBoundsException("value outside of histogram covered range. Caused by: " + ex);
throw new ArrayIndexOutOfBoundsException("value " + value + " outside of histogram covered range. Caused by: " + ex);
}
resize(value);
int countsIndex = countsArrayIndex(value);
......@@ -574,6 +579,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
/**
* Reset the contents and stats of this histogram
*/
@Override
public void reset() {
clearCounts();
resetMaxValue(0);
......@@ -831,7 +837,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
long maxValueBeforeShift = maxValueUpdater.getAndSet(this, 0);
long minNonZeroValueBeforeShift = minNonZeroValueUpdater.getAndSet(this, Long.MAX_VALUE);
boolean lowestHalfBucketPopulated = (minNonZeroValueBeforeShift < subBucketHalfCount);
boolean lowestHalfBucketPopulated = (minNonZeroValueBeforeShift < (subBucketHalfCount << unitMagnitude));
// Perform the shift:
shiftNormalizingIndexByOffset(shiftAmount, lowestHalfBucketPopulated, newIntegerToDoubleValueConversionRatio);
......@@ -848,19 +854,27 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
// Save and clear the 0 value count:
long zeroValueCount = getCountAtIndex(0);
setCountAtIndex(0, 0);
int preShiftZeroIndex = normalizeIndex(0, getNormalizingIndexOffset(), countsArrayLength);
setNormalizingIndexOffset(getNormalizingIndexOffset() + shiftAmount);
// Deal with lower half bucket if needed:
if (lowestHalfBucketPopulated) {
shiftLowestHalfBucketContentsLeft(shiftAmount);
if (shiftAmount <= 0) {
// Shifts with lowest half bucket populated can only be to the left.
// Any right shift logic calling this should have already verified that
// the lowest half bucket is not populated.
throw new ArrayIndexOutOfBoundsException(
"Attempt to right-shift with already-recorded value counts that would underflow and lose precision");
}
shiftLowestHalfBucketContentsLeft(shiftAmount, preShiftZeroIndex);
}
// Restore the 0 value count:
setCountAtIndex(0, zeroValueCount);
}
private void shiftLowestHalfBucketContentsLeft(int shiftAmount) {
private void shiftLowestHalfBucketContentsLeft(int shiftAmount, int preShiftZeroIndex) {
final int numberOfBinaryOrdersOfMagnitude = shiftAmount >> subBucketHalfCountMagnitude;
// The lowest half-bucket (not including the 0 value) is special: unlike all other half
......@@ -881,9 +895,9 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
for (int fromIndex = 1; fromIndex < subBucketHalfCount; fromIndex++) {
long toValue = valueFromIndex(fromIndex) << numberOfBinaryOrdersOfMagnitude;
int toIndex = countsArrayIndex(toValue);
long countAtFromIndex = getCountAtNormalizedIndex(fromIndex);
long countAtFromIndex = getCountAtNormalizedIndex(fromIndex + preShiftZeroIndex);
setCountAtIndex(toIndex, countAtFromIndex);
setCountAtNormalizedIndex(fromIndex, 0);
setCountAtNormalizedIndex(fromIndex + preShiftZeroIndex, 0);
}
// Note that the above loop only creates O(N) work for histograms that have values in
......@@ -1008,11 +1022,27 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
if (getMinNonZeroValue() != that.getMinNonZeroValue()) {
return false;
}
// 2 histograms may be equal but have different underlying array sizes. This can happen for instance due to
// resizing.
if (countsArrayLength == that.countsArrayLength) {
for (int i = 0; i < countsArrayLength; i++) {
if (getCountAtIndex(i) != that.getCountAtIndex(i)) {
return false;
}
}
}
else
{
// Comparing the values is valid here because we have already confirmed the histograms have the same total
// count. It would not be correct otherwise.
for (HistogramIterationValue value : this.recordedValues()) {
long countAtValueIteratedTo = value.getCountAtValueIteratedTo();
long valueIteratedTo = value.getValueIteratedTo();
if (that.getCountAtValue(valueIteratedTo) != countAtValueIteratedTo) {
return false;
}
}
}
return true;
}
......@@ -1283,7 +1313,7 @@ public abstract class AbstractHistogram extends AbstractHistogramBase implements
while (recordedValuesIterator.hasNext()) {
HistogramIterationValue iterationValue = recordedValuesIterator.next();
totalValue += medianEquivalentValue(iterationValue.getValueIteratedTo())
* iterationValue.getCountAtValueIteratedTo();
* (double) iterationValue.getCountAtValueIteratedTo();
}
return (totalValue * 1.0) / getTotalCount();
}
......
src/main/java/org/HdrHistogram/AbstractHistogramLogReader.java deleted 100644 → 0
View file @ f436f7dd
/**
* Written by Gil Tene of Azul Systems, and released to the public domain,
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
*
* @author Gil Tene
*/
package org.HdrHistogram;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.Scanner;
import java.util.zip.DataFormatException;
class AbstractHistogramLogReader {
protected final Scanner scanner;
private double startTimeSec = 0.0;
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified file name.
* @param inputFileName The name of the file to read from
* @throws java.io.FileNotFoundException when unable to find inputFileName
*/
public AbstractHistogramLogReader(final String inputFileName) throws FileNotFoundException {
scanner = new Scanner(new File(inputFileName));
initScanner();
}
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified InputStream.
* @param inputStream The InputStream to read from
*/
public AbstractHistogramLogReader(final InputStream inputStream) {
scanner = new Scanner(inputStream);
initScanner();
}
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified file.
* @param inputFile The File to read from
* @throws java.io.FileNotFoundException when unable to find inputFile
*/
public AbstractHistogramLogReader(final File inputFile) throws FileNotFoundException {
scanner = new Scanner(inputFile);
initScanner();
}
private void initScanner() {
scanner.useLocale(Locale.US);
scanner.useDelimiter("[ ,\\r\\n]");
}
/**
* get the latest start time found in the file so far (or 0.0),
* per the log file format explained above. Assuming the "#[StartTime:" comment
* line precedes the actual intervals recorded in the file, getStartTimeSec() can
* be safely used after each interval is read to determine's the offset of that
* interval's timestamp from the epoch.
* @return latest Start Time found in the file (or 0.0 if non found)
*/
public double getStartTimeSec() {
return startTimeSec;
}
protected void setStartTimeSec(double startTimeSec) {
this.startTimeSec = startTimeSec;
}
/**
* Read the next interval histogram from the log, if interval falls within a time range.
* <p>
* Returns a histogram object if an interval line was found with an
* associated start timestamp value that falls between startTimeSec and
* endTimeSec, or null if no such interval line is found. Note that
* the range is assumed to be in seconds relative to the actual
* timestamp value found in each interval line in the log, and not
* in absolute time.
* <p>
* Timestamps are assumed to appear in order in the log file, and as such
* this method will return a null upon encountering a timestamp larger than
* rangeEndTimeSec.
* <p>
* The histogram returned will have it's timestamp set to the absolute
* timestamp calculated from adding the interval's indicated timestamp
* value to the latest [optional] start time found in the log.
* <p>
* Upon encountering any unexpected format errors in reading the next
* interval from the file, this method will return a null.
* @param startTimeSec The (non-absolute time) start of the expected
* time range, in seconds.
* @param endTimeSec The (non-absolute time) end of the expected time
* range, in seconds.
* @return a histogram, or a null if no appropriate interval found
*/
public EncodableHistogram nextIntervalHistogram(final Double startTimeSec,
final Double endTimeSec) {
return nextIntervalHistogram(startTimeSec, endTimeSec, false);
}
/**
* Read the next interval histogram from the log, if interval falls within an absolute time range
* <p>
* Returns a histogram object if an interval line was found with an
* associated absolute start timestamp value that falls between
* absoluteStartTimeSec and absoluteEndTimeSec, or null if no such
* interval line is found.
* <p>
* Timestamps are assumed to appear in order in the log file, and as such
* this method will return a null upon encountering a timestamp larger than
* rangeEndTimeSec.
* <p>
* The histogram returned will have it's timestamp set to the absolute
* timestamp calculated from adding the interval's indicated timestamp
* value to the latest [optional] start time found in the log.
* <p>
* Absolute timestamps are calculated by adding the timestamp found
* with the recorded interval to the [latest, optional] start time
* found in the log. The start time is indicated in the log with
* a "#[StartTime: " followed by the start time in seconds.
* <p>
* Upon encountering any unexpected format errors in reading the next
* interval from the file, this method will return a null.
* @param absoluteStartTimeSec The (absolute time) start of the expected
* time range, in seconds.
* @param absoluteEndTimeSec The (absolute time) end of the expected
* time range, in seconds.
* @return A histogram, or a null if no appropriate interval found
*/
public EncodableHistogram nextAbsoluteIntervalHistogram(final Double absoluteStartTimeSec,
final Double absoluteEndTimeSec) {
return nextIntervalHistogram(absoluteStartTimeSec, absoluteEndTimeSec, true);
}
/**
* Read the next interval histogram from the log. Returns a Histogram object if
* an interval line was found, or null if not.
* <p>Upon encountering any unexpected format errors in reading the next interval
* from the file, this method will return a null.
* @return a DecodedInterval, or a null if no appropriate interval found
*/
public EncodableHistogram nextIntervalHistogram() {
return nextIntervalHistogram(0.0, Long.MAX_VALUE * 1.0, true);
}
private EncodableHistogram nextIntervalHistogram(final Double rangeStartTimeSec,
final Double rangeEndTimeSec, boolean absolute) {
while (scanner.hasNextLine()) {
try {
if (scanner.hasNext("\\#.*")) {
// comment line
if (scanner.hasNext("#\\[StartTime:")) {
scanner.next("#\\[StartTime:");
if (scanner.hasNextDouble()) {
setStartTimeSec(scanner.nextDouble()); // start time represented as seconds since epoch
}
}
scanner.nextLine();
continue;
}
if (scanner.hasNext("\"StartTimestamp\".*")) {
// Legend line
scanner.nextLine();
continue;
}
// Decode: startTimestamp, intervalLength, maxTime, histogramPayload
final double offsetStartTimeStampSec = scanner.nextDouble(); // Timestamp start is expect to be in seconds
final double absoluteStartTimeStampSec = getStartTimeSec() + offsetStartTimeStampSec;
final double intervalLengthSec = scanner.nextDouble(); // Timestamp length is expect to be in seconds
final double offsetEndTimeStampSec = offsetStartTimeStampSec + intervalLengthSec;
final double absoluteEndTimeStampSec = getStartTimeSec() + offsetEndTimeStampSec;
final double startTimeStampToCheckRangeOn = absolute ? absoluteStartTimeStampSec : offsetStartTimeStampSec;
if (startTimeStampToCheckRangeOn < rangeStartTimeSec) {
scanner.nextLine();
continue;
}
if (startTimeStampToCheckRangeOn > rangeEndTimeSec) {
return null;
}
scanner.nextDouble(); // Skip maxTime field, as max time can be deduced from the histogram.
final String compressedPayloadString = scanner.next();
final ByteBuffer buffer = ByteBuffer.wrap(
Base64Helper.parseBase64Binary(compressedPayloadString));
EncodableHistogram histogram = Histogram.decodeFromCompressedByteBuffer(buffer, 0);
histogram.setStartTimeStamp((long) (absoluteStartTimeStampSec * 1000.0));
histogram.setEndTimeStamp((long) (absoluteEndTimeStampSec * 1000.0));
return histogram;
} catch (java.util.NoSuchElementException ex) {
return null;
} catch (DataFormatException ex) {
return null;
}
}
return null;
}
}
src/main/java/org/HdrHistogram/Base64Helper.java
View file @ ab74d8f4
......@@ -10,15 +10,20 @@ package org.HdrHistogram;
import java.lang.reflect.Method;
/**
* Base64Helper exists to bridge gaps between Java SE platform support of Base64 encoding and decoding.
* Base64Helper exists to bridge inconsistencies in Java SE support of Base64 encoding and decoding.
* Earlier Java SE platforms (up to and including Java SE 8) supported base64 encode/decode via the
* javax.xml.bind.DatatypeConverter class, which was deprecated and eventually removed in Java SE 9.
* Later Java SE platforms Java SE 8 and later support supported base64 encode/decode via the
* Later Java SE platforms (Java SE 8 and later) support base64 encode/decode via the
* java.util.Base64 class (first introduced in Java SE 8, and not available on e.g. Java SE 6 or 7).
*
* This makes it "hard" to write a single piece of source code that deals with base64 encodings and
* will compile and run on e.g. Java SE 7 AND Java SE 9. And such common source is a common need for
* libraries. This class is intended to encapsulate this "hard"-ness and hide the ugly pretzle-twising
* needed under the covers.
*
* Base64Helper provides a common API that works across Java SE 6..9 (and beyond hopefully), and
* uses late binding (Reflection) to avoid javac-compile-time dependencies on a specific Java SE
* version (e.g. beyond 6 or before 9).
* uses late binding (Reflection) internally to avoid javac-compile-time dependencies on a specific
* Java SE version (e.g. beyond 7 or before 9).
*
*/
public class Base64Helper {
......
src/main/java/org/HdrHistogram/ConcurrentHistogram.java
View file @ ab74d8f4
......@@ -211,6 +211,8 @@ public class ConcurrentHistogram extends Histogram {
assert (countsArrayLength == activeCounts.length());
assert (countsArrayLength == inactiveCounts.length());
assert (activeCounts.getNormalizingIndexOffset() == inactiveCounts.getNormalizingIndexOffset());
if (normalizingIndexOffset == activeCounts.getNormalizingIndexOffset()) {
return; // Nothing to do.
}
......@@ -225,7 +227,7 @@ public class ConcurrentHistogram extends Histogram {
// Handle the inactive lowest half bucket:
if ((shiftedAmount > 0) && lowestHalfBucketPopulated) {
shiftLowestInactiveHalfBucketContentsLeft(shiftedAmount);
shiftLowestInactiveHalfBucketContentsLeft(shiftedAmount, zeroIndex);
}
// Restore the inactive 0 value count:
......@@ -251,7 +253,7 @@ public class ConcurrentHistogram extends Histogram {
// Handle the newly inactive lowest half bucket:
if ((shiftedAmount > 0) && lowestHalfBucketPopulated) {
shiftLowestInactiveHalfBucketContentsLeft(shiftedAmount);
shiftLowestInactiveHalfBucketContentsLeft(shiftedAmount, zeroIndex);
}
// Restore the newly inactive 0 value count:
......@@ -275,7 +277,7 @@ public class ConcurrentHistogram extends Histogram {
}
}
private void shiftLowestInactiveHalfBucketContentsLeft(final int shiftAmount) {
private void shiftLowestInactiveHalfBucketContentsLeft(final int shiftAmount, final int preShiftZeroIndex) {
final int numberOfBinaryOrdersOfMagnitude = shiftAmount >> subBucketHalfCountMagnitude;
// The lowest inactive half-bucket (not including the 0 value) is special: unlike all other half
......@@ -298,9 +300,9 @@ public class ConcurrentHistogram extends Histogram {
int toIndex = countsArrayIndex(toValue);
int normalizedToIndex =
normalizeIndex(toIndex, inactiveCounts.getNormalizingIndexOffset(), inactiveCounts.length());
long countAtFromIndex = inactiveCounts.get(fromIndex);
long countAtFromIndex = inactiveCounts.get(fromIndex + preShiftZeroIndex);
inactiveCounts.lazySet(normalizedToIndex, countAtFromIndex);
inactiveCounts.lazySet(fromIndex, 0);
inactiveCounts.lazySet(fromIndex + preShiftZeroIndex, 0);
}
// Note that the above loop only creates O(N) work for histograms that have values in
......@@ -344,34 +346,25 @@ public class ConcurrentHistogram extends Histogram {
return;
}
int oldNormalizedZeroIndex =
normalizeIndex(0, inactiveCounts.getNormalizingIndexOffset(), inactiveCounts.length());
// Resize the current inactiveCounts:
AtomicLongArray oldInactiveCounts = inactiveCounts;
inactiveCounts =
// Allocate both counts arrays here, so if one allocation fails, neither will "take":
AtomicLongArrayWithNormalizingOffset newInactiveCounts1 =
new AtomicLongArrayWithNormalizingOffset(
newArrayLength,
inactiveCounts.getNormalizingIndexOffset()
);
AtomicLongArrayWithNormalizingOffset newInactiveCounts2 =
new AtomicLongArrayWithNormalizingOffset(
newArrayLength,
activeCounts.getNormalizingIndexOffset()
);
// Resize the current inactiveCounts:
AtomicLongArrayWithNormalizingOffset oldInactiveCounts = inactiveCounts;
inactiveCounts = newInactiveCounts1;
// Copy inactive contents to newly sized inactiveCounts:
for (int i = 0 ; i < oldInactiveCounts.length(); i++) {
inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
}
if (oldNormalizedZeroIndex != 0) {
// We need to shift the stuff from the zero index and up to the end of the array:
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
int src, dst;
for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex;
src < oldNormalizedZeroIndex + lengthToCopy;
src++, dst++) {
inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
}
for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
inactiveCounts.lazySet(dst, 0);
}
}
copyInactiveCountsContentsOnResize(oldInactiveCounts, countsDelta);
// switch active and inactive:
AtomicLongArrayWithNormalizingOffset tmp = activeCounts;
......@@ -382,29 +375,10 @@ public class ConcurrentHistogram extends Histogram {
// Resize the newly inactiveCounts:
oldInactiveCounts = inactiveCounts;
inactiveCounts =
new AtomicLongArrayWithNormalizingOffset(
newArrayLength,
inactiveCounts.getNormalizingIndexOffset()
);
inactiveCounts = newInactiveCounts2;
// Copy inactive contents to newly sized inactiveCounts:
for (int i = 0 ; i < oldInactiveCounts.length(); i++) {
inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
}
if (oldNormalizedZeroIndex != 0) {
// We need to shift the stuff from the zero index and up to the end of the array:
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
int lengthToCopy = (newArrayLength - countsDelta) - oldNormalizedZeroIndex;
int src, dst;
for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex;
src < oldNormalizedZeroIndex + lengthToCopy;
src++, dst++) {
inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
}
for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
inactiveCounts.lazySet(dst, 0);
}
}
copyInactiveCountsContentsOnResize(oldInactiveCounts, countsDelta);
// switch active and inactive again:
tmp = activeCounts;
......@@ -427,6 +401,34 @@ public class ConcurrentHistogram extends Histogram {
}
}
private void copyInactiveCountsContentsOnResize(
AtomicLongArrayWithNormalizingOffset oldInactiveCounts, int countsDelta) {
int oldNormalizedZeroIndex =
normalizeIndex(0,
oldInactiveCounts.getNormalizingIndexOffset(),
oldInactiveCounts.length());
// Copy old inactive contents to (current) newly sized inactiveCounts:
for (int i = 0; i < oldInactiveCounts.length(); i++) {
inactiveCounts.lazySet(i, oldInactiveCounts.get(i));
}
if (oldNormalizedZeroIndex != 0) {
// We need to shift the stuff from the zero index and up to the end of the array:
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
int lengthToCopy = (inactiveCounts.length() - countsDelta) - oldNormalizedZeroIndex;
int src, dst;
for (src = oldNormalizedZeroIndex, dst = newNormalizedZeroIndex;
src < oldNormalizedZeroIndex + lengthToCopy;
src++, dst++) {
inactiveCounts.lazySet(dst, oldInactiveCounts.get(src));
}
for (dst = oldNormalizedZeroIndex; dst < newNormalizedZeroIndex; dst++) {
inactiveCounts.lazySet(dst, 0);
}
}
}
@Override
public void setAutoResize(final boolean autoResize) {
this.autoResize = true;
......
src/main/java/org/HdrHistogram/DoubleHistogram.java
View file @ ab74d8f4
......@@ -50,7 +50,7 @@ import java.util.zip.Deflater;
* <p>
* See package description for {@link org.HdrHistogram} for details.
*/
public class DoubleHistogram extends EncodableHistogram implements Serializable {
public class DoubleHistogram extends EncodableHistogram implements DoubleValueRecorder, Serializable {
private static final double highestAllowedValueEver; // A value that will keep us from multiplying into infinity.
private long configuredHighestToLowestValueRatio;
......@@ -287,8 +287,9 @@ public class DoubleHistogram extends EncodableHistogram implements Serializable
* Record a value in the histogram
*
* @param value The value to be recorded
* @throws ArrayIndexOutOfBoundsException (may throw) if value is cannot be covered by the histogram's range
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
@Override
public void recordValue(final double value) throws ArrayIndexOutOfBoundsException {
recordSingleValue(value);
}
......@@ -298,8 +299,9 @@ public class DoubleHistogram extends EncodableHistogram implements Serializable
*
* @param value The value to be recorded
* @param count The number of occurrences of this value to record
* @throws ArrayIndexOutOfBoundsException (may throw) if value is cannot be covered by the histogram's range
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
@Override
public void recordValueWithCount(final double value, final long count) throws ArrayIndexOutOfBoundsException {
recordCountAtValue(count, value);
}
......@@ -323,8 +325,9 @@ public class DoubleHistogram extends EncodableHistogram implements Serializable
* @param expectedIntervalBetweenValueSamples If expectedIntervalBetweenValueSamples is larger than 0, add
* auto-generated value records as appropriate if value is larger
* than expectedIntervalBetweenValueSamples
* @throws ArrayIndexOutOfBoundsException (may throw) if value is cannot be covered by the histogram's range
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
@Override
public void recordValueWithExpectedInterval(final double value, final double expectedIntervalBetweenValueSamples)
throws ArrayIndexOutOfBoundsException {
recordValueWithCountAndExpectedInterval(value, 1, expectedIntervalBetweenValueSamples);
......@@ -573,8 +576,11 @@ public class DoubleHistogram extends EncodableHistogram implements Serializable
/**
* Reset the contents and stats of this histogram
*/
@Override
public void reset() {
integerValuesHistogram.clearCounts();
integerValuesHistogram.reset();
double initialLowestValueInAutoRange = Math.pow(2.0, 800);
init(configuredHighestToLowestValueRatio, initialLowestValueInAutoRange, integerValuesHistogram);
}
//
......
src/main/java/org/HdrHistogram/DoubleRecorder.java
View file @ ab74d8f4
......@@ -22,10 +22,23 @@ import java.util.concurrent.atomic.AtomicLong;
* {@link DoubleRecorder#recordValueWithExpectedInterval} calls.
* Recording calls are wait-free on architectures that support atomic increment operations, and
* are lock-free on architectures that do not.
*
* <p>
* A common pattern for using a {@link DoubleRecorder} looks like this:
* <br><pre><code>
* DoubleRecorder recorder = new DoubleRecorder(2); // Two decimal point accuracy
* DoubleHistogram intervalHistogram = null;
* ...
* [start of some loop construct that periodically wants to grab an interval histogram]
* ...
* // Get interval histogram, recycling previous interval histogram:
* intervalHistogram = recorder.getIntervalHistogram(intervalHistogram);
* histogramLogWriter.outputIntervalHistogram(intervalHistogram);
* ...
* [end of loop construct]
* </code></pre>
*/
public class DoubleRecorder {
public class DoubleRecorder implements DoubleValueRecorder {
private static AtomicLong instanceIdSequencer = new AtomicLong(1);
private final long instanceId = instanceIdSequencer.getAndIncrement();
......@@ -70,6 +83,7 @@ public class DoubleRecorder {
* @param value the value to record
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValue(final double value) {
long criticalValueAtEnter = recordingPhaser.writerCriticalSectionEnter();
try {
......@@ -86,6 +100,7 @@ public class DoubleRecorder {
* @param count The number of occurrences of this value to record
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValueWithCount(final double value, final long count) throws ArrayIndexOutOfBoundsException {
long criticalValueAtEnter = recordingPhaser.writerCriticalSectionEnter();
try {
......@@ -111,6 +126,7 @@ public class DoubleRecorder {
* than expectedIntervalBetweenValueSamples
* @throws ArrayIndexOutOfBoundsException (may throw) if value is exceeds highestTrackableValue
*/
@Override
public void recordValueWithExpectedInterval(final double value, final double expectedIntervalBetweenValueSamples)
throws ArrayIndexOutOfBoundsException {
long criticalValueAtEnter = recordingPhaser.writerCriticalSectionEnter();
......@@ -157,13 +173,49 @@ public class DoubleRecorder {
* getIntervalHistogram(histogramToRecycle)} will reset the value counts, and start accumulating value
* counts for the next interval
*
* @param histogramToRecycle a previously returned interval histogram that may be recycled to avoid allocation and
* @param histogramToRecycle a previously returned interval histogram (from this instance of
* {@link DoubleRecorder}) that may be recycled to avoid allocation and
* copy operations.
* @return a histogram containing the value counts accumulated since the last interval histogram was taken.
*/
public synchronized DoubleHistogram getIntervalHistogram(DoubleHistogram histogramToRecycle) {
return getIntervalHistogram(histogramToRecycle, true);
}
/**
* Get an interval histogram, which will include a stable, consistent view of all value counts
* accumulated since the last interval histogram was taken.
* <p>
* {@link DoubleRecorder#getIntervalHistogram(DoubleHistogram histogramToRecycle)
* getIntervalHistogram(histogramToRecycle)}
* accepts a previously returned interval histogram that can be recycled internally to avoid allocation
* and content copying operations, and is therefore significantly more efficient for repeated use than
* {@link DoubleRecorder#getIntervalHistogram()} and
* {@link DoubleRecorder#getIntervalHistogramInto getIntervalHistogramInto()}. The provided
* {@code histogramToRecycle} must
* be either be null or an interval histogram returned by a previous call to
* {@link DoubleRecorder#getIntervalHistogram(DoubleHistogram histogramToRecycle)
* getIntervalHistogram(histogramToRecycle)} or
* {@link DoubleRecorder#getIntervalHistogram()}.
* <p>
* NOTE: The caller is responsible for not recycling the same returned interval histogram more than once. If
* the same interval histogram instance is recycled more than once, behavior is undefined.
* <p>
* Calling {@link DoubleRecorder#getIntervalHistogram(DoubleHistogram histogramToRecycle)
* getIntervalHistogram(histogramToRecycle)} will reset the value counts, and start accumulating value
* counts for the next interval
*
* @param histogramToRecycle a previously returned interval histogram that may be recycled to avoid allocation and
* copy operations.
* @param enforeContainingInstance if true, will only allow recycling of histograms previously returned from this
* instance of {@link DoubleRecorder}. If false, will allow recycling histograms
* previously returned by other instances of {@link DoubleRecorder}.
* @return a histogram containing the value counts accumulated since the last interval histogram was taken.
*/
public synchronized DoubleHistogram getIntervalHistogram(DoubleHistogram histogramToRecycle,
boolean enforeContainingInstance) {
// Verify that replacement histogram can validly be used as an inactive histogram replacement:
validateFitAsReplacementHistogram(histogramToRecycle);
validateFitAsReplacementHistogram(histogramToRecycle, enforeContainingInstance);
inactiveHistogram = (InternalConcurrentDoubleHistogram) histogramToRecycle;
performIntervalSample();
DoubleHistogram sampledHistogram = inactiveHistogram;
......@@ -188,6 +240,7 @@ public class DoubleRecorder {
/**
* Reset any value counts accumulated thus far.
*/
@Override
public synchronized void reset() {
// the currently inactive histogram is reset each time we flip. So flipping twice resets both:
performIntervalSample();
......@@ -245,15 +298,17 @@ public class DoubleRecorder {
}
}
void validateFitAsReplacementHistogram(DoubleHistogram replacementHistogram) {
private void validateFitAsReplacementHistogram(DoubleHistogram replacementHistogram,
boolean enforeContainingInstance) {
boolean bad = true;
if (replacementHistogram == null) {
bad = false;
} else if ((replacementHistogram instanceof InternalConcurrentDoubleHistogram)
&&
((!enforeContainingInstance) ||
(((InternalConcurrentDoubleHistogram) replacementHistogram).containingInstanceId ==
activeHistogram.containingInstanceId)
) {
)) {
bad = false;
}
......
src/main/java/org/HdrHistogram/DoubleValueRecorder.java 0 → 100644
View file @ ab74d8f4
package org.HdrHistogram;
public interface DoubleValueRecorder {
/**
* Record a value
*
* @param value The value to be recorded
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
void recordValue(double value) throws ArrayIndexOutOfBoundsException;
/**
* Record a value (adding to the value's current count)
*
* @param value The value to be recorded
* @param count The number of occurrences of this value to record
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
void recordValueWithCount(double value, long count) throws ArrayIndexOutOfBoundsException;
/**
* Record a value.
* <p>
* To compensate for the loss of sampled values when a recorded value is larger than the expected
* interval between value samples, will auto-generate an additional series of decreasingly-smaller
* (down to the expectedIntervalBetweenValueSamples) value records.
* <p>
* Note: This is a at-recording correction method, as opposed to the post-recording correction method provided
* by {@link DoubleHistogram#copyCorrectedForCoordinatedOmission(double)}.
* The two methods are mutually exclusive, and only one of the two should be be used on a given data set to correct
* for the same coordinated omission issue.
*
* @param value The value to record
* @param expectedIntervalBetweenValueSamples If expectedIntervalBetweenValueSamples is larger than 0, add
* auto-generated value records as appropriate if value is larger
* than expectedIntervalBetweenValueSamples
* @throws ArrayIndexOutOfBoundsException (may throw) if value cannot be covered by the histogram's range
*/
void recordValueWithExpectedInterval(double value, double expectedIntervalBetweenValueSamples)
throws ArrayIndexOutOfBoundsException;
/**
* Reset the contents and collected stats
*/
void reset();
}
src/main/java/org/HdrHistogram/Histogram.java
View file @ ab74d8f4
......@@ -87,6 +87,11 @@ public class Histogram extends AbstractHistogram {
this.normalizingIndexOffset = normalizingIndexOffset;
}
@Override
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
}
@Override
void shiftNormalizingIndexByOffset(int offsetToAdd,
boolean lowestHalfBucketPopulated,
......
src/main/java/org/HdrHistogram/HistogramLogProcessor.java
View file @ ab74d8f4
......@@ -51,36 +51,37 @@ import java.util.*;
*/
public class HistogramLogProcessor extends Thread {
public static final String versionString = "Histogram Log Processor version " + Version.version;
static final String versionString = "Histogram Log Processor version " + Version.version;
private final HistogramLogProcessorConfiguration config;
private HistogramLogReader logReader;
private static class HistogramLogProcessorConfiguration {
public boolean verbose = false;
public String outputFileName = null;
public String inputFileName = null;
public String tag = null;
boolean verbose = false;
String outputFileName = null;
String inputFileName = null;
String tag = null;
public double rangeStartTimeSec = 0.0;
public double rangeEndTimeSec = Double.MAX_VALUE;
double rangeStartTimeSec = 0.0;
double rangeEndTimeSec = Double.MAX_VALUE;
public boolean logFormatCsv = false;
public boolean listTags = false;
public boolean allTags = false;
boolean logFormatCsv = false;
boolean listTags = false;
boolean allTags = false;
public boolean movingWindow = false;
public double movingWindowPercentileToReport = 99.0;
public long movingWindowLengthInMsec = 60000; // 1 minute
boolean movingWindow = false;
double movingWindowPercentileToReport = 99.0;
long movingWindowLengthInMsec = 60000; // 1 minute
public int percentilesOutputTicksPerHalf = 5;
public Double outputValueUnitRatio = 1000000.0; // default to msec units for output.
int percentilesOutputTicksPerHalf = 5;
Double outputValueUnitRatio = 1000000.0; // default to msec units for output.
public boolean error = false;
public String errorMessage = "";
double expectedIntervalForCoordinatedOmissionCorrection = 0.0;
public HistogramLogProcessorConfiguration(final String[] args) {
String errorMessage = "";
HistogramLogProcessorConfiguration(final String[] args) {
boolean askedForHelp= false;
try {
for (int i = 0; i < args.length; ++i) {
......@@ -93,25 +94,28 @@ public class HistogramLogProcessor extends Thread {
} else if (args[i].equals("-alltags")) {
allTags = true;
} else if (args[i].equals("-i")) {
inputFileName = args[++i];
inputFileName = args[++i]; // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-tag")) {
tag = args[++i];
tag = args[++i]; // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-mwp")) {
movingWindowPercentileToReport = Double.parseDouble(args[++i]);
movingWindowPercentileToReport = Double.parseDouble(args[++i]); // lgtm [java/index-out-of-bounds]
movingWindow = true;
} else if (args[i].equals("-mwpl")) {
movingWindowLengthInMsec = Long.parseLong(args[++i]);
movingWindowLengthInMsec = Long.parseLong(args[++i]); // lgtm [java/index-out-of-bounds]
movingWindow = true;
} else if (args[i].equals("-start")) {
rangeStartTimeSec = Double.parseDouble(args[++i]);
rangeStartTimeSec = Double.parseDouble(args[++i]); // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-end")) {
rangeEndTimeSec = Double.parseDouble(args[++i]);
rangeEndTimeSec = Double.parseDouble(args[++i]); // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-o")) {
outputFileName = args[++i];
outputFileName = args[++i]; // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-percentilesOutputTicksPerHalf")) {
percentilesOutputTicksPerHalf = Integer.parseInt(args[++i]);
percentilesOutputTicksPerHalf = Integer.parseInt(args[++i]); // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-outputValueUnitRatio")) {
outputValueUnitRatio = Double.parseDouble(args[++i]);
outputValueUnitRatio = Double.parseDouble(args[++i]); // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-correctLogWithKnownCoordinatedOmission")) {
expectedIntervalForCoordinatedOmissionCorrection =
Double.parseDouble(args[++i]); // lgtm [java/index-out-of-bounds]
} else if (args[i].equals("-h")) {
askedForHelp = true;
throw new Exception("Help: " + args[i]);
......@@ -119,9 +123,7 @@ public class HistogramLogProcessor extends Thread {
throw new Exception("Invalid args: " + args[i]);
}
}
} catch (Exception e) {
error = true;
errorMessage = "Error: " + versionString + " launched with the following args:\n";
for (String arg : args) {
......@@ -135,7 +137,7 @@ public class HistogramLogProcessor extends Thread {
final String validArgs =
"\"[-csv] [-v] [-i inputFileName] [-o outputFileName] [-tag tag] " +
"[-start rangeStartTimeSec] [-end rangeEndTimeSec] " +
"[-outputValueUnitRatio r] [-listtags]";
"[-outputValueUnitRatio r] [-correctLogWithKnownCoordinatedOmission i] [-listtags]";
System.err.println("valid arguments = " + validArgs);
......@@ -150,6 +152,12 @@ public class HistogramLogProcessor extends Thread {
" [-end rangeEndTimeSec] The end time for the range in the file, in seconds (default is infinite)\n" +
" [-outputValueUnitRatio r] The scaling factor by which to divide histogram recorded values units\n" +
" in output. [default = 1000000.0 (1 msec in nsec)]\n" +
" [-correctLogWithKnownCoordinatedOmission i] When the supplied expected interval i is than 0, performs coordinated\n" +
" omission corection on the input log's interval histograms by adding\n" +
" missing values as appropriate based on the supplied expected interval\n" +
" value i (in wahtever units the log histograms were recorded with). This\n" +
" feature should only be used when the input log is known to have been\n" +
" recorded with coordinated ommisions, and when an expected interval is known.\n" +
" [-listtags] list all tags found on histogram lines the input file."
);
System.exit(1);
......@@ -172,12 +180,33 @@ public class HistogramLogProcessor extends Thread {
startTime, (new Date((long) (startTime * 1000))).toString());
}
int lineNumber = 0;
EncodableHistogram copyCorrectedForCoordinatedOmission(final EncodableHistogram inputHistogram) {
EncodableHistogram histogram = inputHistogram;
if (histogram instanceof DoubleHistogram) {
if (config.expectedIntervalForCoordinatedOmissionCorrection > 0.0) {
histogram = ((DoubleHistogram) histogram).copyCorrectedForCoordinatedOmission(
config.expectedIntervalForCoordinatedOmissionCorrection);
}
} else if (histogram instanceof Histogram) {
long expectedInterval = (long) config.expectedIntervalForCoordinatedOmissionCorrection;
if (expectedInterval > 0) {
histogram = ((Histogram) histogram).copyCorrectedForCoordinatedOmission(expectedInterval);
}
}
return histogram;
}
private int lineNumber = 0;
private EncodableHistogram getIntervalHistogram() {
EncodableHistogram histogram = null;
try {
histogram = logReader.nextIntervalHistogram(config.rangeStartTimeSec, config.rangeEndTimeSec);
if (config.expectedIntervalForCoordinatedOmissionCorrection > 0.0) {
// Apply Coordinated Omission correction to log histograms when arguments indicate that
// such correction is desired, and an expected interval is provided.
histogram = copyCorrectedForCoordinatedOmission(histogram);
}
} catch (RuntimeException ex) {
System.err.println("Log file parsing error at line number " + lineNumber +
": line appears to be malformed.");
......@@ -213,14 +242,11 @@ public class HistogramLogProcessor extends Thread {
PrintStream timeIntervalLog = null;
PrintStream movingWindowLog = null;
PrintStream histogramPercentileLog = System.out;
Double firstStartTime = 0.0;
double firstStartTime = 0.0;
boolean timeIntervalLogLegendWritten = false;
boolean movingWindowLogLegendWritten = false;
// EncodableHistogram[] movingWindow = new EncodableHistogram[config.movingWindowIntervalCount];
EncodableHistogram movingWindowSumHistogram = null;
Queue<EncodableHistogram> movingWindowQueue = new LinkedList<EncodableHistogram>();
int movingWindowIndex = 0;
if (config.listTags) {
Set<String> tags = new TreeSet<String>();
......@@ -283,35 +309,36 @@ public class HistogramLogProcessor extends Thread {
}
EncodableHistogram intervalHistogram = getIntervalHistogram(config.tag);
boolean logUsesDoubleHistograms = (intervalHistogram instanceof DoubleHistogram);
Histogram accumulatedRegularHistogram = null;
DoubleHistogram accumulatedDoubleHistogram = null;
Histogram accumulatedRegularHistogram = logUsesDoubleHistograms ?
new Histogram(3) :
((Histogram) intervalHistogram).copy();
accumulatedRegularHistogram.reset();
accumulatedRegularHistogram.setAutoResize(true);
if (intervalHistogram != null) {
// Shape the accumulated histogram like the histograms in the log file (but clear their contents):
if (intervalHistogram instanceof DoubleHistogram) {
accumulatedDoubleHistogram = ((DoubleHistogram) intervalHistogram).copy();
DoubleHistogram accumulatedDoubleHistogram = logUsesDoubleHistograms ?
((DoubleHistogram) intervalHistogram).copy() :
new DoubleHistogram(3);
accumulatedDoubleHistogram.reset();
accumulatedDoubleHistogram.setAutoResize(true);
movingWindowSumHistogram = new DoubleHistogram(3);
} else {
accumulatedRegularHistogram = ((Histogram) intervalHistogram).copy();
accumulatedRegularHistogram.reset();
accumulatedRegularHistogram.setAutoResize(true);
movingWindowSumHistogram = new Histogram(3);
}
}
EncodableHistogram movingWindowSumHistogram = logUsesDoubleHistograms ?
new DoubleHistogram(3) :
new Histogram(3);
while (intervalHistogram != null) {
// handle accumulated histogram:
if (intervalHistogram instanceof DoubleHistogram) {
if (accumulatedDoubleHistogram == null) {
if (!logUsesDoubleHistograms) {
throw new IllegalStateException("Encountered a DoubleHistogram line in a log of Histograms.");
}
accumulatedDoubleHistogram.add((DoubleHistogram) intervalHistogram);
} else {
if (accumulatedRegularHistogram == null) {
if (logUsesDoubleHistograms) {
throw new IllegalStateException("Encountered a Histogram line in a log of DoubleHistograms.");
}
accumulatedRegularHistogram.add((Histogram) intervalHistogram);
......@@ -366,7 +393,7 @@ public class HistogramLogProcessor extends Thread {
}
}
if (intervalHistogram instanceof DoubleHistogram) {
if (logUsesDoubleHistograms) {
timeIntervalLog.format(Locale.US, logFormat,
((intervalHistogram.getEndTimeStamp() / 1000.0) - logReader.getStartTimeSec()),
// values recorded during the last reporting interval
......@@ -436,21 +463,21 @@ public class HistogramLogProcessor extends Thread {
intervalHistogram = getIntervalHistogram(config.tag);
}
if (accumulatedDoubleHistogram != null) {
if (logUsesDoubleHistograms) {
accumulatedDoubleHistogram.outputPercentileDistribution(histogramPercentileLog,
config.percentilesOutputTicksPerHalf, config.outputValueUnitRatio, config.logFormatCsv);
} else {
if (accumulatedRegularHistogram == null) {
// If there were no histograms in the log file, we still need an empty histogram for the
// one line output (shape/range doesn't matter because it is empty):
accumulatedRegularHistogram = new Histogram(1000000L, 2);
}
accumulatedRegularHistogram.outputPercentileDistribution(histogramPercentileLog,
config.percentilesOutputTicksPerHalf, config.outputValueUnitRatio, config.logFormatCsv);
}
} finally {
if (config.outputFileName != null) {
if (timeIntervalLog != null) {
timeIntervalLog.close();
}
if (movingWindowLog != null) {
movingWindowLog.close();
}
if (histogramPercentileLog != System.out) {
histogramPercentileLog.close();
}
}
......@@ -465,8 +492,15 @@ public class HistogramLogProcessor extends Thread {
* [-i logFileName] File name of Histogram Log to process (default is standard input)
* [-o outputFileName] File name to output to (default is standard output)
* (will replace occurrences of %pid and %date with appropriate information)
* [-tag tag] The tag (default no tag) of the histogram lines to be processed\n
* [-start rangeStartTimeSec] The start time for the range in the file, in seconds (default 0.0)
* [-end rangeEndTimeSec] The end time for the range in the file, in seconds (default is infinite)
* [-correctLogWithKnownCoordinatedOmission expectedInterval] When the supplied expected interval i is than 0, performs coordinated
* omission corection on the input log's interval histograms by adding
* missing values as appropriate based on the supplied expected interval
* value i (in wahtever units the log histograms were recorded with). This
* feature should only be used when the input log is known to have been
* recorded with coordinated ommisions, and when an expected interval is known.
* [-outputValueUnitRatio r] The scaling factor by which to divide histogram recorded values units
* in output. [default = 1000000.0 (1 msec in nsec)]"
* </pre>
......
src/main/java/org/HdrHistogram/HistogramLogReader.java
View file @ ab74d8f4
......@@ -7,12 +7,7 @@
package org.HdrHistogram;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.Scanner;
import java.io.*;
import java.util.zip.DataFormatException;
/**
......@@ -56,22 +51,125 @@ import java.util.zip.DataFormatException;
* that may be added to timestamps in the file to determine an absolute
* timestamp (e.g. since the epoch) for each interval.
*/
public class HistogramLogReader {
public class HistogramLogReader implements Closeable {
private final HistogramLogScanner scanner;
private final HistogramLogScanner.EventHandler handler = new HistogramLogScanner.EventHandler()
{
@Override
public boolean onComment(String comment)
{
return false;
}
@Override
public boolean onBaseTime(double secondsSinceEpoch)
{
baseTimeSec = secondsSinceEpoch; // base time represented as seconds since epoch
observedBaseTime = true;
return false;
}
@Override
public boolean onStartTime(double secondsSinceEpoch)
{
startTimeSec = secondsSinceEpoch; // start time represented as seconds since epoch
observedStartTime = true;
return false;
}
@Override
public boolean onHistogram(String tag, double timestamp, double length,
HistogramLogScanner.EncodableHistogramSupplier lazyReader) {
final double logTimeStampInSec = timestamp; // Timestamp is expected to be in seconds
if (!observedStartTime) {
// No explicit start time noted. Use 1st observed time:
startTimeSec = logTimeStampInSec;
observedStartTime = true;
}
if (!observedBaseTime) {
// No explicit base time noted. Deduce from 1st observed time (compared to start time):
if (logTimeStampInSec < startTimeSec - (365 * 24 * 3600.0)) {
// Criteria Note: if log timestamp is more than a year in the past (compared to
// StartTime), we assume that timestamps in the log are not absolute
baseTimeSec = startTimeSec;
} else {
// Timestamps are absolute
baseTimeSec = 0.0;
}
observedBaseTime = true;
}
final double absoluteStartTimeStampSec = logTimeStampInSec + baseTimeSec;
final double offsetStartTimeStampSec = absoluteStartTimeStampSec - startTimeSec;
final double intervalLengthSec = length; // Timestamp length is expect to be in seconds
final double absoluteEndTimeStampSec = absoluteStartTimeStampSec + intervalLengthSec;
final double startTimeStampToCheckRangeOn = absolute ? absoluteStartTimeStampSec : offsetStartTimeStampSec;
if (startTimeStampToCheckRangeOn < rangeStartTimeSec) {
// keep on trucking
return false;
}
if (startTimeStampToCheckRangeOn > rangeEndTimeSec) {
// after limit we stop on each line
return true;
}
EncodableHistogram histogram;
try
{
histogram = lazyReader.read();
}
catch (DataFormatException e)
{
// stop after exception
return true;
}
histogram.setStartTimeStamp((long) (absoluteStartTimeStampSec * 1000.0));
histogram.setEndTimeStamp((long) (absoluteEndTimeStampSec * 1000.0));
histogram.setTag(tag);
nextHistogram = histogram;
return true;
}
@Override
public boolean onException(Throwable t) {
// We ignore NoSuchElementException, but stop processing.
// Next call to nextIntervalHistogram may return null.
if (t instanceof java.util.NoSuchElementException){
return true;
}
// rethrow
if (t instanceof RuntimeException)
throw (RuntimeException)t;
else
throw new RuntimeException(t);
}
};
private final Scanner scanner;
private double startTimeSec = 0.0;
private boolean observedStartTime = false;
private double baseTimeSec = 0.0;
private boolean observedBaseTime = false;
// scanner handling state
private boolean absolute;
private double rangeStartTimeSec;
private double rangeEndTimeSec;
private EncodableHistogram nextHistogram;
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified file name.
* @param inputFileName The name of the file to read from
* @throws java.io.FileNotFoundException when unable to find inputFileName
*/
public HistogramLogReader(final String inputFileName) throws FileNotFoundException {
scanner = new Scanner(new File(inputFileName));
initScanner();
scanner = new HistogramLogScanner(new File(inputFileName));
}
/**
......@@ -79,8 +177,7 @@ public class HistogramLogReader {
* @param inputStream The InputStream to read from
*/
public HistogramLogReader(final InputStream inputStream) {
scanner = new Scanner(inputStream);
initScanner();
scanner = new HistogramLogScanner(inputStream);
}
/**
......@@ -89,14 +186,7 @@ public class HistogramLogReader {
* @throws java.io.FileNotFoundException when unable to find inputFile
*/
public HistogramLogReader(final File inputFile) throws FileNotFoundException {
scanner = new Scanner(inputFile);
initScanner();
}
private void initScanner() {
scanner.useLocale(Locale.US);
scanner.useDelimiter("[, \\r\\n]");
scanner = new HistogramLogScanner(inputFile);
}
/**
......@@ -195,105 +285,26 @@ public class HistogramLogReader {
private EncodableHistogram nextIntervalHistogram(final double rangeStartTimeSec,
final double rangeEndTimeSec, boolean absolute) {
while (scanner.hasNextLine()) {
try {
if (scanner.hasNext("\\#.*")) {
// comment line.
// Look for explicit start time or base time notes in comments:
if (scanner.hasNext("#\\[StartTime:")) {
scanner.next("#\\[StartTime:");
if (scanner.hasNextDouble()) {
startTimeSec = scanner.nextDouble(); // start time represented as seconds since epoch
observedStartTime = true;
}
} else if (scanner.hasNext("#\\[BaseTime:")) {
scanner.next("#\\[BaseTime:");
if (scanner.hasNextDouble()) {
baseTimeSec = scanner.nextDouble(); // base time represented as seconds since epoch
observedBaseTime = true;
}
}
continue;
}
if (scanner.hasNext("\"StartTimestamp\".*")) {
// Legend line
continue;
}
String tagString = null;
if (scanner.hasNext("Tag\\=.*")) {
tagString = scanner.next("Tag\\=.*").substring(4);
}
// Decode: startTimestamp, intervalLength, maxTime, histogramPayload
final double logTimeStampInSec = scanner.nextDouble(); // Timestamp is expected to be in seconds
if (!observedStartTime) {
// No explicit start time noted. Use 1st observed time:
startTimeSec = logTimeStampInSec;
observedStartTime = true;
}
if (!observedBaseTime) {
// No explicit base time noted. Deduce from 1st observed time (compared to start time):
if (logTimeStampInSec < startTimeSec - (365 * 24 * 3600.0)) {
// Criteria Note: if log timestamp is more than a year in the past (compared to
// StartTime), we assume that timestamps in the log are not absolute
baseTimeSec = startTimeSec;
} else {
// Timestamps are absolute
baseTimeSec = 0.0;
}
observedBaseTime = true;
}
final double absoluteStartTimeStampSec = logTimeStampInSec + baseTimeSec;
final double offsetStartTimeStampSec = absoluteStartTimeStampSec - startTimeSec;
final double intervalLengthSec = scanner.nextDouble(); // Timestamp length is expect to be in seconds
final double absoluteEndTimeStampSec = absoluteStartTimeStampSec + intervalLengthSec;
final double startTimeStampToCheckRangeOn = absolute ? absoluteStartTimeStampSec : offsetStartTimeStampSec;
if (startTimeStampToCheckRangeOn < rangeStartTimeSec) {
continue;
}
if (startTimeStampToCheckRangeOn > rangeEndTimeSec) {
return null;
}
scanner.nextDouble(); // Skip maxTime field, as max time can be deduced from the histogram.
final String compressedPayloadString = scanner.next();
final ByteBuffer buffer = ByteBuffer.wrap(
Base64Helper.parseBase64Binary(compressedPayloadString));
EncodableHistogram histogram = EncodableHistogram.decodeFromCompressedByteBuffer(buffer, 0);
histogram.setStartTimeStamp((long) (absoluteStartTimeStampSec * 1000.0));
histogram.setEndTimeStamp((long) (absoluteEndTimeStampSec * 1000.0));
histogram.setTag(tagString);
this.rangeStartTimeSec = rangeStartTimeSec;
this.rangeEndTimeSec = rangeEndTimeSec;
this.absolute = absolute;
scanner.process(handler);
EncodableHistogram histogram = this.nextHistogram;
nextHistogram = null;
return histogram;
} catch (java.util.NoSuchElementException ex) {
return null;
} catch (DataFormatException ex) {
return null;
} finally {
scanner.nextLine(); // Move to next line.
}
}
return null;
}
/**
* Indicates whther or not additional intervals may exist in the log
* @return ture if additional intervals may exist in the log
* Indicates whether or not additional intervals may exist in the log
* @return true if additional intervals may exist in the log
*/
public boolean hasNext() {
return scanner.hasNextLine();
}
@Override
public void close()
{
scanner.close();
}
}
src/main/java/org/HdrHistogram/HistogramLogScanner.java 0 → 100644
View file @ ab74d8f4
/**
* Written by Gil Tene of Azul Systems, and released to the public domain,
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
*
* @author Gil Tene
*/
package org.HdrHistogram;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.Scanner;
import java.util.zip.DataFormatException;
public class HistogramLogScanner implements Closeable {
// can't use lambdas, and anyway we need to let the handler take the exception
public interface EncodableHistogramSupplier
{
EncodableHistogram read() throws DataFormatException;
}
/**
* Handles log events, return true to stop processing.
*/
public interface EventHandler
{
boolean onComment(String comment);
boolean onBaseTime(double secondsSinceEpoch);
boolean onStartTime(double secondsSinceEpoch);
/**
* A lazy reader is provided to allow fast skipping of bulk of work where tag or timestamp are to be used as
* a basis for filtering the {@link EncodableHistogram} anyway. The reader is to be called only once.
*
* @param tag histogram tag or null if none exist
* @param timestamp logged timestamp
* @param length logged interval length
* @param lazyReader to be called if the histogram needs to be deserialized, given the tag/timestamp etc.
* @return
*/
boolean onHistogram(String tag, double timestamp, double length, EncodableHistogramSupplier lazyReader);
boolean onException(Throwable t);
}
private static class LazyHistogramReader implements EncodableHistogramSupplier {
private final Scanner scanner;
private boolean gotIt = true;
private LazyHistogramReader(Scanner scanner)
{
this.scanner = scanner;
}
private void allowGet()
{
gotIt = false;
}
@Override
public EncodableHistogram read() throws DataFormatException
{
// prevent double calls to this method
if (gotIt)
throw new IllegalStateException();
gotIt = true;
final String compressedPayloadString = scanner.next();
final ByteBuffer buffer = ByteBuffer.wrap(Base64Helper.parseBase64Binary(compressedPayloadString));
EncodableHistogram histogram = EncodableHistogram.decodeFromCompressedByteBuffer(buffer, 0);
return histogram;
}
}
private final LazyHistogramReader lazyReader;
protected final Scanner scanner;
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified file name.
* @param inputFileName The name of the file to read from
* @throws java.io.FileNotFoundException when unable to find inputFileName
*/
public HistogramLogScanner(final String inputFileName) throws FileNotFoundException {
this(new Scanner(new File(inputFileName)));
}
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified InputStream. Note that
* log readers constructed through this constructor do not assume ownership of stream and will not close it on
* {@link #close()}.
*
* @param inputStream The InputStream to read from
*/
public HistogramLogScanner(final InputStream inputStream) {
this(new Scanner(inputStream));
}
/**
* Constructs a new HistogramLogReader that produces intervals read from the specified file.
* @param inputFile The File to read from
* @throws java.io.FileNotFoundException when unable to find inputFile
*/
public HistogramLogScanner(final File inputFile) throws FileNotFoundException {
this(new Scanner(inputFile));
}
private HistogramLogScanner(Scanner scanner)
{
this.scanner = scanner;
this.lazyReader = new LazyHistogramReader(scanner);
initScanner();
}
private void initScanner() {
scanner.useLocale(Locale.US);
scanner.useDelimiter("[ ,\\r\\n]");
}
/**
* Close underlying scanner.
*/
@Override
public void close()
{
scanner.close();
}
public void process(EventHandler handler) {
while (scanner.hasNextLine()) {
try {
if (scanner.hasNext("\\#.*")) {
// comment line.
// Look for explicit start time or base time notes in comments:
if (scanner.hasNext("#\\[StartTime:")) {
scanner.next("#\\[StartTime:");
if (scanner.hasNextDouble()) {
double startTimeSec = scanner.nextDouble(); // start time represented as seconds since epoch
if (handler.onStartTime(startTimeSec)) {
return;
}
}
} else if (scanner.hasNext("#\\[BaseTime:")) {
scanner.next("#\\[BaseTime:");
if (scanner.hasNextDouble()) {
double baseTimeSec = scanner.nextDouble(); // base time represented as seconds since epoch
if (handler.onBaseTime(baseTimeSec))
{
return;
}
}
} else if (handler.onComment(scanner.next("\\#.*"))) {
return;
}
continue;
}
if (scanner.hasNext("\"StartTimestamp\".*")) {
// Legend line
continue;
}
String tagString = null;
if (scanner.hasNext("Tag\\=.*")) {
tagString = scanner.next("Tag\\=.*").substring(4);
}
// Decode: startTimestamp, intervalLength, maxTime, histogramPayload
final double logTimeStampInSec = scanner.nextDouble(); // Timestamp is expected to be in seconds
final double intervalLengthSec = scanner.nextDouble(); // Timestamp length is expect to be in seconds
scanner.nextDouble(); // Skip maxTime field, as max time can be deduced from the histogram.
lazyReader.allowGet();
if (handler.onHistogram(tagString, logTimeStampInSec, intervalLengthSec, lazyReader))
return;
} catch (Throwable ex) {
if (handler.onException(ex))
return;
} finally {
scanner.nextLine(); // Move to next line.
}
}
return;
}
/**
* Indicates whether or not additional intervals may exist in the log
*
* @return true if additional intervals may exist in the log
*/
public boolean hasNextLine() {
return scanner.hasNextLine();
}
}