Skip to content
Commits on Source (4)
......@@ -4,13 +4,13 @@
<groupId>com.fasterxml.jackson</groupId>
<!-- For 2.9.2 and beyond, new parent pom; extends jackson-bom -->
<artifactId>jackson-base</artifactId>
<version>2.9.4</version>
<version>2.9.8</version>
</parent>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<name>Jackson-core</name>
<version>2.9.4</version>
<version>2.9.8</version>
<packaging>bundle</packaging>
<description>Core Jackson processing abstractions (aka Streaming API), implementation for JSON</description>
<inceptionYear>2008</inceptionYear>
......@@ -20,7 +20,7 @@
<connection>scm:git:git@github.com:FasterXML/jackson-core.git</connection>
<developerConnection>scm:git:git@github.com:FasterXML/jackson-core.git</developerConnection>
<url>http://github.com/FasterXML/jackson-core</url>
<tag>jackson-core-2.9.4</tag>
<tag>jackson-core-2.9.8</tag>
</scm>
<properties>
......@@ -45,7 +45,17 @@ com.fasterxml.jackson.core.*;version=${project.version}
<jdk.module.name>com.fasterxml.jackson.core</jdk.module.name>
</properties>
<!-- parent provides junit dep, not repeated here -->
<!-- Alas, need to include snapshot reference since otherwise can not find
snapshot of parent... -->
<repositories>
<repository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases><enabled>false</enabled></releases>
<snapshots><enabled>true</enabled></snapshots>
</repository>
</repositories>
<build>
<plugins>
......@@ -74,6 +84,11 @@ com.fasterxml.jackson.core.*;version=${project.version}
<excludes>
<exclude>**/failing/**/*.java</exclude>
</excludes>
<!-- 13-Apr-2018, tatu: for debugging [core#400]
<systemPropertyVariables>
<com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers>true</com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers>
</systemPropertyVariables>
-->
</configuration>
</plugin>
<!-- settings are fine, but needed to trigger execution! -->
......
......@@ -145,3 +145,14 @@ Rafal Foltynski (rfoltyns@github)
(2.9.0)
* Contributed#208: Make use of `_matchCount` in `FilteringParserDelegate`
(2.9.0)
Jeroen Borgers (jborgers@github)
* Reported, contributed impl for #400: Add mechanism for forcing `BufferRecycler` released
(to call on shutdown)
(2.9.6)
Doug Roper (htmldoug@github)
* Suggested #463: Ensure that `skipChildren()` of non-blocking `JsonParser` will throw
exception if not enough input
(2.9.6)
......@@ -14,7 +14,34 @@ JSON library.
=== Releases ===
------------------------------------------------------------------------
2.9.4 (24-Jan-2017)
2.9.8 (15-Dec-2018)
#488: Fail earlier on coercions from "too big" `BigInteger` into
fixed-size types (`int`, `long`, `short`)
- Improve exception message for missing Base64 padding (see databind#2183)
2.9.7 (19-Sep-2018)
#476: Problem with `BufferRecycler` via async parser (or when sharing parser
across threads)
#477: Exception while decoding Base64 value with escaped `=` character
#488: Fail earlier on coercions from "too big" `BigInteger` into
fixed-size types (`int`, `long`, `short`)
2.9.6 (12-Jun-2018)
#400: Add mechanism for forcing `BufferRecycler` released (to call on shutdown)
(contributed by Jeroen B)
#460: Failing to link `ObjectCodec` with `JsonFactory` copy constructor
#463: Ensure that `skipChildren()` of non-blocking `JsonParser` will throw
exception if not enough input
(requested by Doug R)
2.9.5 (26-Mar-2018)
No changes since 2.9.4
2.9.4 (24-Jan-2018)
#414: Base64 MIME variant does not ignore white space chars as per RFC2045
(reported by tmoschou@github)
......@@ -62,7 +89,7 @@ JSON library.
#374: Minimal and DefaultPrettyPrinter with configurable separators
(contributed by Rafal F)
2.8.11 (not yet released)
2.8.11 (23-Dec-2017)
#418: ArrayIndexOutOfBoundsException from UTF32Reader.read on invalid input
(reported, contributed fix for by pfitzsimons-r7@github)
......@@ -142,7 +169,23 @@ No changes since 2.8.0
for `getCurrentToken()` and `getCurrentTokenId()`, respectively. Existing methods
will likely be deprecated in 2.9.
2.7.10 (not yet released)
2.7.9.3:
#1872: NullPointerException in SubTypeValidator.validateSubType when
validating Spring interface
#1931: Two more c3p0 gadgets to exploit default typing issue
2.7.9.2 (20-Dec-2017)
#1607: `@JsonIdentityReference` not used when setup on class only
#1628: Don't print to error stream about failure to load JDK 7 types
#1680: Blacklist couple more types for deserialization
#1737: Block more JDK types from polymorphic deserialization
#1855: Blacklist for more serialization gadgets (dbcp/tomcat, spring)
2.7.9.1 (18-Apr-2017)
#1599: Jackson Deserializer security vulnerability
2.7.9 (04-Feb-2017)
......
......@@ -593,7 +593,19 @@ public final class Base64Variant
}
protected void _reportBase64EOF() throws IllegalArgumentException {
throw new IllegalArgumentException("Unexpected end-of-String in base64 content");
throw new IllegalArgumentException(missingPaddingMessage());
}
/**
* Helper method that will construct a message to use in exceptions for cases where input ends
* prematurely in place where padding would be expected.
*
* @since 2.10
*/
public String missingPaddingMessage() {
return String.format("Unexpected end of base64-encoded String: base64 variant '%s' expects padding (one or more '%c' characters) at the end",
getName(), getPaddingChar());
}
}
......@@ -68,9 +68,7 @@ public final class Base64Variants
// Replace plus with hyphen, slash with underscore (and no padding)
sb.setCharAt(sb.indexOf("+"), '-');
sb.setCharAt(sb.indexOf("/"), '_');
/* And finally, let's not split lines either, wouldn't work too
* well with URLs
*/
// And finally, let's not split lines either, wouldn't work too well with URLs
MODIFIED_FOR_URL = new Base64Variant("MODIFIED-FOR-URL", sb.toString(), false, Base64Variant.PADDING_CHAR_NONE, Integer.MAX_VALUE);
}
......
......@@ -286,7 +286,7 @@ public class JsonFactory
*/
protected JsonFactory(JsonFactory src, ObjectCodec codec)
{
_objectCodec = null;
_objectCodec = codec;
_factoryFeatures = src._factoryFeatures;
_parserFeatures = src._parserFeatures;
_generatorFeatures = src._generatorFeatures;
......@@ -960,7 +960,7 @@ public class JsonFactory
// 17-May-2017, tatu: Need to take care not to accidentally create JSON parser
// for non-JSON input:
_requireJSONFactory("Non-blocking source not (yet?) support for this format (%s)");
IOContext ctxt = _createContext(null, false);
IOContext ctxt = _createNonBlockingContext(null);
ByteQuadsCanonicalizer can = _byteSymbolCanonicalizer.makeChild(_factoryFeatures);
return new NonBlockingJsonParser(ctxt, _parserFeatures, can);
}
......@@ -1548,6 +1548,19 @@ public class JsonFactory
return new IOContext(_getBufferRecycler(), srcRef, resourceManaged);
}
/**
* Overridable factory method that actually instantiates desired
* context object for async (non-blocking) parsing
*
* @since 2.9.7
*/
protected IOContext _createNonBlockingContext(Object srcRef) {
// [jackson-core#476]: disable buffer recycling for 2.9 to avoid concurrency issues;
// easiest done by just constructing private "recycler":
BufferRecycler recycler = new BufferRecycler();
return new IOContext(recycler, srcRef, false);
}
/**
* @since 2.8
*/
......
......@@ -823,7 +823,7 @@ public abstract class ParserBase extends ParserMinimalBase
}
} catch (NumberFormatException nex) {
// Can this ever occur? Due to overflow, maybe?
_wrapError("Malformed numeric value '"+_textBuffer.contentsAsString()+"'", nex);
_wrapError("Malformed numeric value ("+_longNumberDesc(_textBuffer.contentsAsString())+")", nex);
}
}
......@@ -842,15 +842,32 @@ public abstract class ParserBase extends ParserMinimalBase
// Probably faster to construct a String, call parse, than to use BigInteger
_numberLong = Long.parseLong(numStr);
_numTypesValid = NR_LONG;
} else {
// 16-Oct-2018, tatu: Need to catch "too big" early due to [jackson-core#488]
if ((expType == NR_INT) || (expType == NR_LONG)) {
_reportTooLongInt(expType, numStr);
}
if ((expType == NR_DOUBLE) || (expType == NR_FLOAT)) {
_numberDouble = NumberInput.parseDouble(numStr);
_numTypesValid = NR_DOUBLE;
} else {
// nope, need the heavy guns... (rare case)
_numberBigInt = new BigInteger(numStr);
_numTypesValid = NR_BIGINT;
}
}
} catch (NumberFormatException nex) {
// Can this ever occur? Due to overflow, maybe?
_wrapError("Malformed numeric value '"+numStr+"'", nex);
_wrapError("Malformed numeric value ("+_longNumberDesc(numStr)+")", nex);
}
}
// @since 2.9.8
protected void _reportTooLongInt(int expType, String rawNum) throws IOException
{
final String numDesc = _longIntegerDesc(rawNum);
_reportError("Numeric value (%s) out of range of %s", numDesc,
(expType == NR_LONG) ? "long" : "int");
}
/*
......@@ -1029,8 +1046,10 @@ public abstract class ParserBase extends ParserMinimalBase
// otherwise try to find actual triplet value
int bits = b64variant.decodeBase64Char(unescaped);
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, unescaped, index);
}
}
return bits;
}
......@@ -1049,8 +1068,11 @@ public abstract class ParserBase extends ParserMinimalBase
// otherwise try to find actual triplet value
int bits = b64variant.decodeBase64Char(unescaped);
if (bits < 0) {
// second check since padding can only be 3rd or 4th byte (index #2 or #3)
if ((bits != Base64Variant.BASE64_VALUE_PADDING) || (index < 2)) {
throw reportInvalidBase64Char(b64variant, unescaped, index);
}
}
return bits;
}
......@@ -1081,6 +1103,12 @@ public abstract class ParserBase extends ParserMinimalBase
return new IllegalArgumentException(base);
}
// since 2.9.8
protected void _handleBase64MissingPadding(Base64Variant b64variant) throws IOException
{
_reportError(b64variant.missingPaddingMessage());
}
/*
/**********************************************************
/* Internal/package methods: other
......
......@@ -249,6 +249,12 @@ public abstract class ParserMinimalBase extends JsonParser
if (--open == 0) {
return this;
}
// 23-May-2018, tatu: [core#463] Need to consider non-blocking case...
} else if (t == JsonToken.NOT_AVAILABLE) {
// Nothing much we can do except to either return `null` (which seems wrong),
// or, what we actually do, signal error
_reportError("Not enough content available for `skipChildren()`: non-blocking parser? (%s)",
getClass().getName());
}
}
}
......@@ -541,12 +547,36 @@ public abstract class ParserMinimalBase extends JsonParser
protected void reportOverflowInt() throws IOException {
_reportError(String.format("Numeric value (%s) out of range of int (%d - %s)",
getText(), Integer.MIN_VALUE, Integer.MAX_VALUE));
_longIntegerDesc(getText()), Integer.MIN_VALUE, Integer.MAX_VALUE));
}
protected void reportOverflowLong() throws IOException {
_reportError(String.format("Numeric value (%s) out of range of long (%d - %s)",
getText(), Long.MIN_VALUE, Long.MAX_VALUE));
_longIntegerDesc(getText()), Long.MIN_VALUE, Long.MAX_VALUE));
}
// @since 2.9.8
protected String _longIntegerDesc(String rawNum) {
int rawLen = rawNum.length();
if (rawLen < 1000) {
return rawNum;
}
if (rawNum.startsWith("-")) {
rawLen -= 1;
}
return String.format("[Integer with %d digits]", rawLen);
}
// @since 2.9.8
protected String _longNumberDesc(String rawNum) {
int rawLen = rawNum.length();
if (rawLen < 1000) {
return rawNum;
}
if (rawNum.startsWith("-")) {
rawLen -= 1;
}
return String.format("[number with %d characters]", rawLen);
}
protected void _reportUnexpectedChar(int ch, String comment) throws JsonParseException
......
......@@ -549,9 +549,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == '"') {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -563,8 +567,10 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
}
ch = _inputBuffer[_inputPtr++];
if (!b64variant.usesPaddingChar(ch)) {
if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
......@@ -582,10 +588,14 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == '"') {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
......@@ -2008,9 +2018,7 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
} while (ptr < inputLen);
}
/* Either ran out of input, or bumped into an escape
* sequence...
*/
// Either ran out of input, or bumped into an escape sequence...
_textBuffer.resetWithCopy(_inputBuffer, _inputPtr, (ptr-_inputPtr));
_inputPtr = ptr;
_finishString2();
......@@ -2700,9 +2708,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == '"') {
decodedData >>= 4;
builder.append(decodedData);
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -2714,8 +2726,10 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
}
ch = _inputBuffer[_inputPtr++];
if (!b64variant.usesPaddingChar(ch)) {
if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
builder.append(decodedData);
......@@ -2734,9 +2748,13 @@ public class ReaderBasedJsonParser // final in 2.3, earlier
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == '"') {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
......
......@@ -481,9 +481,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -492,8 +495,11 @@ public class UTF8DataInputJsonParser
// Ok, must get padding
ch = _inputData.readUnsignedByte();
if (!b64variant.usesPaddingChar(ch)) {
if ((ch != INT_BACKSLASH)
|| _decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
......@@ -508,10 +514,13 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
......@@ -2753,9 +2762,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 4;
builder.append(decodedData);
if (b64variant.usesPadding()) {
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -2763,8 +2775,11 @@ public class UTF8DataInputJsonParser
if (bits == Base64Variant.BASE64_VALUE_PADDING) {
ch = _inputData.readUnsignedByte();
if (!b64variant.usesPaddingChar(ch)) {
if ((ch != INT_BACKSLASH)
|| _decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
builder.append(decodedData);
......@@ -2779,9 +2794,12 @@ public class UTF8DataInputJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
if (b64variant.usesPadding()) {
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
......
......@@ -487,18 +487,14 @@ public class UTF8StreamJsonParser
(_currToken != JsonToken.VALUE_EMBEDDED_OBJECT || _binaryValue == null)) {
_reportError("Current token ("+_currToken+") not VALUE_STRING or VALUE_EMBEDDED_OBJECT, can not access as binary");
}
/* To ensure that we won't see inconsistent data, better clear up
* state...
*/
// To ensure that we won't see inconsistent data, better clear up state...
if (_tokenIncomplete) {
try {
_binaryValue = _decodeBase64(b64variant);
} catch (IllegalArgumentException iae) {
throw _constructError("Failed to decode VALUE_STRING as base64 ("+b64variant+"): "+iae.getMessage());
}
/* let's clear incomplete only now; allows for accessing other
* textual content in error cases
*/
// let's clear incomplete only now; allows for accessing other textual content in error cases
_tokenIncomplete = false;
} else { // may actually require conversion...
if (_binaryValue == null) {
......@@ -588,9 +584,13 @@ public class UTF8StreamJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -602,8 +602,10 @@ public class UTF8StreamJsonParser
}
ch = _inputBuffer[_inputPtr++] & 0xFF;
if (!b64variant.usesPaddingChar(ch)) {
if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
buffer[outputPtr++] = (byte) decodedData;
......@@ -621,10 +623,14 @@ public class UTF8StreamJsonParser
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
if (ch == INT_QUOTE) {
decodedData >>= 2;
buffer[outputPtr++] = (byte) (decodedData >> 8);
buffer[outputPtr++] = (byte) decodedData;
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
break;
}
bits = _decodeBase64Escape(b64variant, ch, 3);
......@@ -3608,10 +3614,14 @@ public class UTF8StreamJsonParser
// First branch: can get padding (-> 1 byte)
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
// could also just be 'missing' padding
if (ch == INT_QUOTE) {
decodedData >>= 4;
builder.append(decodedData);
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 2);
......@@ -3623,8 +3633,10 @@ public class UTF8StreamJsonParser
}
ch = _inputBuffer[_inputPtr++] & 0xFF;
if (!b64variant.usesPaddingChar(ch)) {
if (_decodeBase64Escape(b64variant, ch, 3) != Base64Variant.BASE64_VALUE_PADDING) {
throw reportInvalidBase64Char(b64variant, ch, 3, "expected padding character '"+b64variant.getPaddingChar()+"'");
}
}
// Got 12 bits, only need 8, need to shift
decodedData >>= 4;
builder.append(decodedData);
......@@ -3641,21 +3653,22 @@ public class UTF8StreamJsonParser
bits = b64variant.decodeBase64Char(ch);
if (bits < 0) {
if (bits != Base64Variant.BASE64_VALUE_PADDING) {
// as per [JACKSON-631], could also just be 'missing' padding
if (ch == '"' && !b64variant.usesPadding()) {
// could also just be 'missing' padding
if (ch == INT_QUOTE) {
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
if (b64variant.usesPadding()) {
--_inputPtr; // to keep parser state bit more consistent
_handleBase64MissingPadding(b64variant);
}
return builder.toByteArray();
}
bits = _decodeBase64Escape(b64variant, ch, 3);
}
if (bits == Base64Variant.BASE64_VALUE_PADDING) {
/* With padding we only get 2 bytes; but we have
* to shift it a bit so it is identical to triplet
* case with partial output.
* 3 chars gives 3x6 == 18 bits, of which 2 are
* dummies, need to discard:
*/
// With padding we only get 2 bytes; but we have to shift it
// a bit so it is identical to triplet case with partial output.
// 3 chars gives 3x6 == 18 bits, of which 2 are dummies, need to discard:
decodedData >>= 2;
builder.appendTwoBytes(decodedData);
continue;
......
......@@ -15,6 +15,32 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder;
*/
public class BufferRecyclers
{
/**
* System property that is checked to see if recycled buffers (see {@link BufferRecycler})
* should be tracked, for purpose of forcing release of all such buffers, typically
* during major classloading.
*
* @since 2.9.6
*/
public final static String SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS
= "com.fasterxml.jackson.core.util.BufferRecyclers.trackReusableBuffers";
/*
/**********************************************************
/* Life-cycle
/**********************************************************
*/
/**
* Flag that indicates whether {@link BufferRecycler} instances should be tracked.
*/
private final static ThreadLocalBufferManager _bufferRecyclerTracker;
static {
_bufferRecyclerTracker = "true".equals(System.getProperty(SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS))
? ThreadLocalBufferManager.instance()
: null;
}
/*
/**********************************************************
/* BufferRecyclers for parsers, generators
......@@ -29,6 +55,9 @@ public class BufferRecyclers
final protected static ThreadLocal<SoftReference<BufferRecycler>> _recyclerRef
= new ThreadLocal<SoftReference<BufferRecycler>>();
/**
* Main accessor to call for accessing possibly recycled {@link BufferRecycler} instance.
*/
public static BufferRecycler getBufferRecycler()
{
SoftReference<BufferRecycler> ref = _recyclerRef.get();
......@@ -36,11 +65,36 @@ public class BufferRecyclers
if (br == null) {
br = new BufferRecycler();
_recyclerRef.set(new SoftReference<BufferRecycler>(br));
if (_bufferRecyclerTracker != null) {
ref = _bufferRecyclerTracker.wrapAndTrack(br);
} else {
ref = new SoftReference<BufferRecycler>(br);
}
_recyclerRef.set(ref);
}
return br;
}
/**
* Specialized method that will release all recycled {@link BufferRecycler} if
* (and only if) recycler tracking has been enabled
* (see {@link #SYSTEM_PROPERTY_TRACK_REUSABLE_BUFFERS}).
* This method is usually called on shutdown of the container like Application Server
* to ensure that no references are reachable via {@link ThreadLocal}s as this may cause
* unintentional retention of sizable amounts of memory. It may also be called regularly
* if GC for some reason does not clear up {@link SoftReference}s aggressively enough.
*
* @return Number of buffers released, if tracking enabled (zero or more); -1 if tracking not enabled.
*
* @since 2.9.6
*/
public static int releaseBuffers() {
if (_bufferRecyclerTracker != null) {
return _bufferRecyclerTracker.releaseBuffers();
}
return -1;
}
/*
/**********************************************************
/* JsonStringEncoder
......
package com.fasterxml.jackson.core.util;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* For issue [jackson-core#400] We keep a separate Set of all SoftReferences to BufferRecyclers
* which are (also) referenced using `ThreadLocals`.
* We do this to be able to release them (dereference) in `releaseBuffers()` and `shutdown()`
* method to reduce heap consumption during hot reloading of services where otherwise
* {@link ClassLoader} would have dangling reference via {@link ThreadLocal}s.
* When gc clears a SoftReference, it puts it on a newly introduced referenceQueue.
* We use this queue to release the inactive SoftReferences from the Set.
*
* @since 2.9.6
*/
class ThreadLocalBufferManager
{
/**
* A lock to make sure releaseBuffers is only executed by one thread at a time
* since it iterates over and modifies the allSoftBufRecyclers.
*/
private final Object RELEASE_LOCK = new Object();
/**
* A set of all SoftReferences to all BufferRecyclers to be able to release them on shutdown.
* 'All' means the ones created by this class, in this classloader.
* There may be more from other classloaders.
* We use a HashSet to have quick O(1) add and remove operations.
*<p>
* NOTE: assumption is that {@link SoftReference} has its {@code equals()} and
* {@code hashCode()} implementations defined so that they use object identity, so
* we do not need to use something like {@link IdentityHashMap}
*/
private final Map<SoftReference<BufferRecycler>,Boolean> _trackedRecyclers
= new ConcurrentHashMap<SoftReference<BufferRecycler>, Boolean>();
/**
* Queue where gc will put just-cleared SoftReferences, previously referencing BufferRecyclers.
* We use it to remove the cleared softRefs from the above set.
*/
private final ReferenceQueue<BufferRecycler> _refQueue = new ReferenceQueue<BufferRecycler>();
/*
/**********************************************************
/* Public API
/**********************************************************
*/
/**
* Returns the lazily initialized singleton instance
*/
public static ThreadLocalBufferManager instance() {
return ThreadLocalBufferManagerHolder.manager;
}
/**
* Releases the buffers retained in ThreadLocals. To be called for instance on shutdown event of applications which make use of
* an environment like an appserver which stays alive and uses a thread pool that causes ThreadLocals created by the
* application to survive much longer than the application itself.
* It will clear all bufRecyclers from the SoftRefs and release all SoftRefs itself from our set.
*/
public int releaseBuffers() {
synchronized (RELEASE_LOCK) {
int count = 0;
// does this need to be in sync block too? Looping over Map definitely has to but...
removeSoftRefsClearedByGc(); // make sure the refQueue is empty
for (SoftReference<BufferRecycler> ref : _trackedRecyclers.keySet()) {
ref.clear(); // possibly already cleared by gc, nothing happens in that case
++count;
}
_trackedRecyclers.clear(); //release cleared SoftRefs
return count;
}
}
public SoftReference<BufferRecycler> wrapAndTrack(BufferRecycler br) {
SoftReference<BufferRecycler> newRef;
newRef = new SoftReference<BufferRecycler>(br, _refQueue);
// also retain softRef to br in a set to be able to release it on shutdown
_trackedRecyclers.put(newRef, true);
// gc may have cleared one or more SoftRefs, clean them up to avoid a memleak
removeSoftRefsClearedByGc();
return newRef;
}
/*
/**********************************************************
/* Internal methods
/**********************************************************
*/
/**
* Remove cleared (inactive) SoftRefs from our set. Gc may have cleared one or more,
* and made them inactive. We minimize contention by keeping synchronized sections short:
* the poll/remove methods
*/
private void removeSoftRefsClearedByGc() {
SoftReference<?> clearedSoftRef;
while ((clearedSoftRef = (SoftReference<?>) _refQueue.poll()) != null) {
// uses reference-equality, quick, and O(1) removal by HashSet
_trackedRecyclers.remove(clearedSoftRef);
}
}
/**
* ThreadLocalBufferManagerHolder uses the thread-safe initialize-on-demand, holder class idiom that implicitly
* incorporates lazy initialization by declaring a static variable within a static Holder inner class
*/
private static final class ThreadLocalBufferManagerHolder {
static final ThreadLocalBufferManager manager = new ThreadLocalBufferManager();
}
}
......@@ -505,6 +505,14 @@ public abstract class BaseTest
/**********************************************************
*/
protected static String quote(String str) {
return '"'+str+'"';
}
protected static String aposToQuotes(String json) {
return json.replace("'", "\"");
}
protected byte[] encodeInUTF32BE(String input)
{
int len = input.length();
......@@ -519,12 +527,13 @@ public abstract class BaseTest
return result;
}
protected String quote(String str) {
return '"'+str+'"';
// @since 2.9.7
protected static byte[] utf8Bytes(String str) {
try {
return str.getBytes("UTF-8");
} catch (IOException e) {
throw new RuntimeException(e);
}
protected String aposToQuotes(String json) {
return json.replace("'", "\"");
}
protected void fieldNameFor(StringBuilder sb, int index)
......@@ -548,6 +557,16 @@ public abstract class BaseTest
}
}
// @since 2.9.7
protected JsonFactory sharedStreamFactory() {
return JSON_FACTORY;
}
// @since 2.9.7
protected JsonFactory newStreamFactory() {
return new JsonFactory();
}
protected String fieldNameFor(int index)
{
StringBuilder sb = new StringBuilder(16);
......
......@@ -6,16 +6,14 @@ import static org.junit.Assert.*;
/**
* Unit tests for class {@link Version}.
*
* @date 2017-08-01
* @see Version
*
**/
public class VersionTest{
@Test
public void testCompareToOne() {
Version version = Version.unknownVersion();
Version versionTwo = new Version(0, (-263), (-1820), "");
Version versionTwo = new Version(0, (-263), (-1820), "",
"", "");
assertEquals(263, version.compareTo(versionTwo));
}
......@@ -23,7 +21,8 @@ public class VersionTest{
@Test
public void testCompareToReturningZero() {
Version version = Version.unknownVersion();
Version versionTwo = new Version(0, 0, 0, "");
Version versionTwo = new Version(0, 0, 0, "",
"", "");
assertEquals(0, version.compareTo(versionTwo));
}
......@@ -39,7 +38,8 @@ public class VersionTest{
@Test
public void testCompareToTwo() {
Version version = Version.unknownVersion();
Version versionTwo = new Version((-1), 0, 0, "0.0.0");
Version versionTwo = new Version((-1), 0, 0, "0.0.0",
"", "");
assertTrue(version.compareTo(versionTwo) > 0);
}
......@@ -51,5 +51,4 @@ public class VersionTest{
assertTrue(version.compareTo(versionTwo) < 0);
}
}
......@@ -4,6 +4,8 @@ import static org.junit.Assert.assertArrayEquals;
import java.io.*;
import org.junit.Assert;
import com.fasterxml.jackson.core.*;
public class Base64BinaryParsingTest
......@@ -53,6 +55,12 @@ public class Base64BinaryParsingTest
}
}
public void testWithEscapedPadding() throws IOException {
for (int mode : ALL_MODES) {
_testEscapedPadding(mode);
}
}
public void testInvalidTokenForBase64() throws IOException
{
for (int mode : ALL_MODES) {
......@@ -110,6 +118,74 @@ public class Base64BinaryParsingTest
}
}
public void testOkMissingPadding() throws IOException {
final byte[] DOC1 = new byte[] { (byte) 0xAD };
_testOkMissingPadding(DOC1, MODE_INPUT_STREAM);
_testOkMissingPadding(DOC1, MODE_INPUT_STREAM_THROTTLED);
_testOkMissingPadding(DOC1, MODE_READER);
_testOkMissingPadding(DOC1, MODE_DATA_INPUT);
final byte[] DOC2 = new byte[] { (byte) 0xAC, (byte) 0xDC };
_testOkMissingPadding(DOC2, MODE_INPUT_STREAM);
_testOkMissingPadding(DOC2, MODE_INPUT_STREAM_THROTTLED);
_testOkMissingPadding(DOC2, MODE_READER);
_testOkMissingPadding(DOC2, MODE_DATA_INPUT);
}
private void _testOkMissingPadding(byte[] input, int mode) throws IOException
{
final Base64Variant b64 = Base64Variants.MODIFIED_FOR_URL;
final String encoded = b64.encode(input, false);
JsonParser p = createParser(mode, quote(encoded));
// 1 byte -> 2 encoded chars; 2 bytes -> 3 encoded chars
assertEquals(input.length+1, encoded.length());
assertToken(JsonToken.VALUE_STRING, p.nextToken());
byte[] actual = p.getBinaryValue(b64);
Assert.assertArrayEquals(input, actual);
p.close();
}
public void testFailDueToMissingPadding() throws IOException {
final String DOC1 = quote("fQ"); // 1 bytes, no padding
_testFailDueToMissingPadding(DOC1, MODE_INPUT_STREAM);
_testFailDueToMissingPadding(DOC1, MODE_INPUT_STREAM_THROTTLED);
_testFailDueToMissingPadding(DOC1, MODE_READER);
_testFailDueToMissingPadding(DOC1, MODE_DATA_INPUT);
final String DOC2 = quote("A/A"); // 2 bytes, no padding
_testFailDueToMissingPadding(DOC2, MODE_INPUT_STREAM);
_testFailDueToMissingPadding(DOC2, MODE_INPUT_STREAM_THROTTLED);
_testFailDueToMissingPadding(DOC2, MODE_READER);
_testFailDueToMissingPadding(DOC2, MODE_DATA_INPUT);
}
private void _testFailDueToMissingPadding(String doc, int mode) throws IOException {
final String EXP_EXCEPTION_MATCH = "Unexpected end of base64-encoded String: base64 variant 'MIME' expects padding";
// First, without getting text value first:
JsonParser p = createParser(mode, doc);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
try {
/*byte[] b =*/ p.getBinaryValue(Base64Variants.MIME);
fail("Should not pass");
} catch (JsonParseException e) {
verifyException(e, EXP_EXCEPTION_MATCH);
}
p.close();
// second, access String first
p = createParser(mode, doc);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
/*String str =*/ p.getText();
try {
/*byte[] b =*/ p.getBinaryValue(Base64Variants.MIME);
fail("Should not pass");
} catch (JsonParseException e) {
verifyException(e, EXP_EXCEPTION_MATCH);
}
p.close();
}
/*
/**********************************************************
/* Test helper methods
......@@ -129,7 +205,7 @@ public class Base64BinaryParsingTest
Base64Variants.PEM
};
JsonFactory jsonFactory = new JsonFactory();
JsonFactory jsonFactory = sharedStreamFactory();
final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
StringWriter chars = null;
for (int len : LENS) {
......@@ -192,7 +268,7 @@ public class Base64BinaryParsingTest
139000
};
JsonFactory jsonFactory = new JsonFactory();
JsonFactory jsonFactory = sharedStreamFactory();
final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
StringWriter chars = null;
......@@ -271,7 +347,7 @@ public class Base64BinaryParsingTest
private void _testInArray(int mode) throws IOException
{
JsonFactory f = new JsonFactory();
JsonFactory f = sharedStreamFactory();
final int entryCount = 7;
......@@ -333,4 +409,53 @@ public class Base64BinaryParsingTest
}
p.close();
}
private void _testEscapedPadding(int mode) throws IOException
{
// Input: "Test!" -> "VGVzdCE="
final String DOC = quote("VGVzdCE\\u003d");
// 06-Sep-2018, tatu: actually one more, test escaping of padding
JsonParser p = createParser(mode, DOC);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("Test!", new String(p.getBinaryValue(), "US-ASCII"));
if (mode != MODE_DATA_INPUT) {
assertNull(p.nextToken());
}
p.close();
// also, try out alternate access method
p = createParser(mode, DOC);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("Test!", new String(_readBinary(p), "US-ASCII"));
if (mode != MODE_DATA_INPUT) {
assertNull(p.nextToken());
}
p.close();
// and then different padding; "X" -> "WA=="
final String DOC2 = quote("WA\\u003D\\u003D");
p = createParser(mode, DOC2);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("X", new String(p.getBinaryValue(), "US-ASCII"));
if (mode != MODE_DATA_INPUT) {
assertNull(p.nextToken());
}
p.close();
p = createParser(mode, DOC2);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("X", new String(_readBinary(p), "US-ASCII"));
if (mode != MODE_DATA_INPUT) {
assertNull(p.nextToken());
}
p.close();
}
private byte[] _readBinary(JsonParser p) throws IOException
{
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
p.readBinaryValue(bytes);
return bytes.toByteArray();
}
}
package com.fasterxml.jackson.core.json;
import java.io.*;
import java.util.Iterator;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.type.ResolvedType;
import com.fasterxml.jackson.core.type.TypeReference;
public class JsonFactoryTest
extends com.fasterxml.jackson.core.BaseTest
{
private static class BogusCodec extends ObjectCodec {
@Override
public Version version() { return null; }
@Override
public <T> T readValue(JsonParser p, Class<T> valueType) throws IOException {
return null;
}
@Override
public <T> T readValue(JsonParser p, TypeReference<?> valueTypeRef) throws IOException {
return null;
}
@Override
public <T> T readValue(JsonParser p, ResolvedType valueType) throws IOException {
return null;
}
@Override
public <T> Iterator<T> readValues(JsonParser p, Class<T> valueType) throws IOException {
return null;
}
@Override
public <T> Iterator<T> readValues(JsonParser p, TypeReference<?> valueTypeRef) throws IOException {
return null;
}
@Override
public <T> Iterator<T> readValues(JsonParser p, ResolvedType valueType) throws IOException {
return null;
}
@Override
public void writeValue(JsonGenerator gen, Object value) throws IOException {
}
@Override
public <T extends TreeNode> T readTree(JsonParser p) throws IOException {
return null;
}
@Override
public void writeTree(JsonGenerator gen, TreeNode tree) throws IOException {
}
@Override
public TreeNode createObjectNode() {
return null;
}
@Override
public TreeNode createArrayNode() {
return null;
}
@Override
public JsonParser treeAsTokens(TreeNode n) {
return null;
}
@Override
public <T> T treeToValue(TreeNode n, Class<T> valueType) throws JsonProcessingException {
return null;
}
}
// for testing [core#460]
@SuppressWarnings("serial")
static class CustomFactory extends JsonFactory {
public CustomFactory(JsonFactory f, ObjectCodec codec) {
super(f, codec);
}
}
/*
/**********************************************************************
/* Test methods
/**********************************************************************
*/
public void testGeneratorFeatures() throws Exception
{
JsonFactory f = new JsonFactory();
......@@ -115,21 +200,32 @@ public class JsonFactoryTest
{
JsonFactory jf = new JsonFactory();
// first, verify defaults
assertNull(jf.getCodec());
assertTrue(jf.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
assertFalse(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertFalse(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
// then change, verify that changes "stick"
jf.disable(JsonFactory.Feature.INTERN_FIELD_NAMES);
jf.enable(JsonParser.Feature.ALLOW_COMMENTS);
jf.enable(JsonGenerator.Feature.ESCAPE_NON_ASCII);
// then change, verify that changes "stick"
ObjectCodec codec = new BogusCodec();
jf.setCodec(codec);
assertFalse(jf.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
assertTrue(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertTrue(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
assertSame(codec, jf.getCodec());
JsonFactory jf2 = jf.copy();
assertFalse(jf2.isEnabled(JsonFactory.Feature.INTERN_FIELD_NAMES));
assertTrue(jf.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertTrue(jf.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
assertTrue(jf2.isEnabled(JsonParser.Feature.ALLOW_COMMENTS));
assertTrue(jf2.isEnabled(JsonGenerator.Feature.ESCAPE_NON_ASCII));
// 16-May-2018, tatu: But! Note that despited [core#460], this should NOT copy it back
assertNull(jf2.getCodec());
// However: real copy constructor SHOULD copy it
JsonFactory jf3 = new CustomFactory(jf, codec);
assertSame(codec, jf3.getCodec());
}
}
package com.fasterxml.jackson.core.json.async;
import java.io.IOException;
import java.util.ArrayList;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.async.AsyncTestBase;
import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
public class AsyncConcurrencyTest extends AsyncTestBase
{
private final static JsonFactory JSON_F = new JsonFactory();
static {
// To make it pass, try:
// JSON_F.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING);
}
private final static String TEXT1 = "Short";
private final static String TEXT2 = "Some longer text";
private final static String TEXT3 = "and yet more";
private final static String TEXT4 = "... Longest yet although not superbly long still (see 'apos'?)";
private final static byte[] JSON_DOC = utf8Bytes(String.format(
"[\"%s\", \"%s\",\n\"%s\",\"%s\" ]", TEXT1, TEXT2, TEXT3, TEXT4));
private class WorkUnit {
private int stage = 0;
private AsyncReaderWrapper parser;
private boolean errored = false;
public boolean process() throws Exception {
// short-cut through if this instance has already failed
if (errored) {
return false;
}
try {
switch (stage++) {
case 0:
parser = asyncForBytes(JSON_F, 100, JSON_DOC, 0);
break;
case 1:
_assert(JsonToken.START_ARRAY);
break;
case 2:
_assert(TEXT1);
break;
case 3:
_assert(TEXT2);
break;
case 4:
_assert(TEXT3);
break;
case 5:
_assert(TEXT4);
break;
case 6:
_assert(JsonToken.END_ARRAY);
break;
default:
/*
if (parser.nextToken() != null) {
throw new IOException("Unexpected token at "+stage+"; expected `null`, got "+parser.currentToken());
}
*/
parser.close();
parser = null;
stage = 0;
return true;
}
} catch (Exception e) {
errored = true;
throw e;
}
return false;
}
private void _assert(String exp) throws IOException {
_assert(JsonToken.VALUE_STRING);
String str = parser.currentText();
if (!exp.equals(str)) {
throw new IOException("Unexpected VALUE_STRING: expected '"+exp+"', got '"+str+"'");
}
}
private void _assert(JsonToken exp) throws IOException {
JsonToken t = parser.nextToken();
if (t != exp) {
throw new IOException("Unexpected token at "+stage+"; expected "+exp+", got "+t);
}
}
}
// [jackson-core#476]
public void testConcurrentAsync() throws Exception
{
final int MAX_ROUNDS = 30;
for (int i = 0; i < MAX_ROUNDS; ++i) {
_testConcurrentAsyncOnce(i, MAX_ROUNDS);
}
}
private void _testConcurrentAsyncOnce(final int round, final int maxRounds) throws Exception
{
final int numThreads = 3;
final ExecutorService executor = Executors.newFixedThreadPool(numThreads);
final AtomicInteger errorCount = new AtomicInteger(0);
final AtomicInteger completedCount = new AtomicInteger(0);
final AtomicReference<String> errorRef = new AtomicReference<String>();
// First, add a few shared work units
final ArrayBlockingQueue<WorkUnit> q = new ArrayBlockingQueue<WorkUnit>(20);
for (int i = 0; i < 7; ++i) {
q.add(new WorkUnit());
}
// then invoke swarm of workers on it...
final int REP_COUNT = 99000;
ArrayList<Future<?>> futures = new ArrayList<Future<?>>();
for (int i = 0; i < REP_COUNT; i++) {
Callable<Void> c = new Callable<Void>() {
@Override
public Void call() throws Exception {
WorkUnit w = q.take();
try {
if (w.process()) {
completedCount.incrementAndGet();
}
} catch (Throwable t) {
if (errorCount.getAndIncrement() == 0) {
errorRef.set(t.toString());
}
} finally {
q.add(w);
}
return null;
}
};
futures.add(executor.submit(c));
}
executor.shutdown();
executor.awaitTermination(5, TimeUnit.SECONDS);
int count = errorCount.get();
if (count > 0) {
fail("Expected no problems (round "+round+"/"+maxRounds
+"); got "+count+", first with: "+errorRef.get());
}
final int EXP_COMPL = ((REP_COUNT + 7) / 8);
int compl = completedCount.get();
if (compl < (EXP_COMPL-10) || compl > EXP_COMPL) {
fail("Expected about "+EXP_COMPL+" completed rounds, got: "+compl);
}
}
}
package com.fasterxml.jackson.core.json.async;
import java.io.IOException;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.async.AsyncTestBase;
import com.fasterxml.jackson.core.filter.FilteringParserDelegate;
import com.fasterxml.jackson.core.filter.TokenFilter;
// [core#462], [core#463]
public class AsyncTokenFilterTest extends AsyncTestBase
{
private final JsonFactory JSON_F = new JsonFactory();
private final static String INPUT_STRING = aposToQuotes("{'a': 1, 'b': [2, {'c': 3}]}");
private final static byte[] INPUT_BYTES = utf8Bytes(INPUT_STRING);
private final static TokenFilter TOKEN_FILTER = new TokenFilter() {
@Override
public TokenFilter includeProperty(String name) {
return name == "a" ? TokenFilter.INCLUDE_ALL : null;
}
};
private final static JsonToken[] EXPECTED_TOKENS = new JsonToken[]{
JsonToken.START_OBJECT,
JsonToken.FIELD_NAME,
JsonToken.VALUE_NUMBER_INT,
JsonToken.END_OBJECT
};
// Passes if (but only if) all content is actually available
public void testFilteredNonBlockingParserAllContent() throws IOException
{
NonBlockingJsonParser nonBlockingParser = (NonBlockingJsonParser) JSON_F.createNonBlockingByteArrayParser();
FilteringParserDelegate filteredParser = new FilteringParserDelegate(nonBlockingParser,
TOKEN_FILTER, true, true);
nonBlockingParser.feedInput(INPUT_BYTES, 0, INPUT_BYTES.length);
int expectedIdx = 0;
while (expectedIdx < EXPECTED_TOKENS.length) {
// grab next token
JsonToken actual = filteredParser.nextToken();
// make sure it's the right one and mark it as seen.
assertToken(EXPECTED_TOKENS[expectedIdx], actual);
expectedIdx++;
}
filteredParser.close();
nonBlockingParser.close();
}
public void testSkipChildrenFailOnSplit() throws IOException
{
NonBlockingJsonParser nbParser = (NonBlockingJsonParser) JSON_F.createNonBlockingByteArrayParser();
FilteringParserDelegate filteredParser = new FilteringParserDelegate(nbParser,
TOKEN_FILTER, true, true);
nbParser.feedInput(INPUT_BYTES, 0, 5);
assertToken(JsonToken.START_OBJECT, nbParser.nextToken());
try {
nbParser.skipChildren();
fail("Should not pass!");
} catch (JsonParseException e) {
verifyException(e, "not enough content available");
verifyException(e, "skipChildren()");
}
nbParser.close();
filteredParser.close();
}
}
package com.fasterxml.jackson.core.read;
import java.math.BigInteger;
import com.fasterxml.jackson.core.*;
public class NumberOverflowTest
extends com.fasterxml.jackson.core.BaseTest
{
private final JsonFactory FACTORY = new JsonFactory();
// NOTE: this should be long enough to trigger perf problems
private final static int BIG_NUM_LEN = 199999;
private final static String BIG_POS_INTEGER;
static {
StringBuilder sb = new StringBuilder(BIG_NUM_LEN);
for (int i = 0; i < BIG_NUM_LEN; ++i) {
sb.append('9');
}
BIG_POS_INTEGER = sb.toString();
}
private final static String BIG_POS_DOC = "["+BIG_POS_INTEGER+"]";
private final static String BIG_NEG_DOC = "[ -"+BIG_POS_INTEGER+"]";
public void testSimpleLongOverflow() throws Exception
{
BigInteger below = BigInteger.valueOf(Long.MIN_VALUE);
below = below.subtract(BigInteger.ONE);
BigInteger above = BigInteger.valueOf(Long.MAX_VALUE);
above = above.add(BigInteger.ONE);
String DOC_BELOW = below.toString() + " ";
String DOC_ABOVE = below.toString() + " ";
for (int mode : ALL_MODES) {
JsonParser p = createParser(FACTORY, mode, DOC_BELOW);
p.nextToken();
try {
long x = p.getLongValue();
fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
} catch (JsonParseException e) {
verifyException(e, "out of range of long");
}
p.close();
p = createParser(mode, DOC_ABOVE);
p.nextToken();
try {
long x = p.getLongValue();
fail("Expected an exception for underflow (input "+p.getText()+"): instead, got long value: "+x);
} catch (JsonParseException e) {
verifyException(e, "out of range of long");
}
p.close();
}
}
// Note: only 4 cardinal types; `short`, `byte` and `char` use same code paths
// Note: due to [jackson-core#493], we'll skip DataInput-backed parser
// [jackson-core#488]
public void testMaliciousLongOverflow() throws Exception
{
for (int mode : ALL_STREAMING_MODES) {
for (String doc : new String[] { BIG_POS_DOC, BIG_NEG_DOC }) {
JsonParser p = createParser(mode, doc);
assertToken(JsonToken.START_ARRAY, p.nextToken());
assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
try {
p.getLongValue();
fail("Should not pass");
} catch (JsonParseException e) {
verifyException(e, "out of range of long");
verifyException(e, "Integer with "+BIG_NUM_LEN+" digits");
}
p.close();
}
}
}
// [jackson-core#488]
public void testMaliciousIntOverflow() throws Exception
{
for (int mode : ALL_STREAMING_MODES) {
for (String doc : new String[] { BIG_POS_DOC, BIG_NEG_DOC }) {
JsonParser p = createParser(mode, doc);
assertToken(JsonToken.START_ARRAY, p.nextToken());
assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
try {
p.getIntValue();
fail("Should not pass");
} catch (JsonParseException e) {
verifyException(e, "out of range of int");
verifyException(e, "Integer with "+BIG_NUM_LEN+" digits");
}
p.close();
}
}
}
// [jackson-core#488]
public void testMaliciousBigIntToDouble() throws Exception
{
for (int mode : ALL_STREAMING_MODES) {
final String doc = BIG_POS_DOC;
JsonParser p = createParser(mode, doc);
assertToken(JsonToken.START_ARRAY, p.nextToken());
assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
double d = p.getDoubleValue();
assertEquals(Double.valueOf(BIG_POS_INTEGER), d);
assertToken(JsonToken.END_ARRAY, p.nextToken());
p.close();
}
}
// [jackson-core#488]
public void testMaliciousBigIntToFloat() throws Exception
{
for (int mode : ALL_STREAMING_MODES) {
final String doc = BIG_POS_DOC;
JsonParser p = createParser(mode, doc);
assertToken(JsonToken.START_ARRAY, p.nextToken());
assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
float f = p.getFloatValue();
assertEquals(Float.valueOf(BIG_POS_INTEGER), f);
assertToken(JsonToken.END_ARRAY, p.nextToken());
p.close();
}
}
}