Skip to content
Commits on Source (2)
Release 4.5.9
-------------------
This is a maintenance release that fixes a number defects discovered since 4.5.8.
Changelog:
-------------------
* HTTPCLIENT-1991: incorrect handling of non-standard DNS entries by PublicSuffixMatcher
Contributed by Oleg Kalnichevski <olegk at apache.org>
* Fix bug in URIBuilder#isPathEmpty method to verify if encodedPath is an empty string
Contributed by Varun Nandi <varunkn at amazon.com>
* HTTPCLIENT-1984: Add normalize URI to RequestConfig copy constructor
Contributed by Matt Nelson <matt.nelson at cerner.com>
* HTTPCLIENT-1976: Unsafe deserialization in DefaultHttpCacheEntrySerializer
Contributed by Artem Smotrakov <artem.smotrakov ar gmail.com>
Release 4.5.8
-------------------
This is a maintenance release that makes request URI normalization configurable on per request basis
and also ports several improvements in URI handling from HttpCore master.
Changelog:
-------------------
* HTTPCLIENT-1969: Filter out weak cipher suites.
Contributed by Artem Smotrakov <artem.smotrakov at sap.com>
* HTTPCLIENT-1968: Preserve escaped PATHSAFE characters when normalizing URI path segments.
Contributed by Oleg Kalnichevski <olegk at apache.org>
* HTTPCLIENT-1968: URIBuilder to split path component into path segments when digesting a URI
(ported from HttpCore master).
Contributed by Oleg Kalnichevski <olegk at apache.org>
* Improved cache key generation (ported from HttpCore master).
Contributed by Oleg Kalnichevski <olegk at apache.org>
* HTTPCLIENT-1968: added utility methods to parse and format URI path segments (ported
from HttpCore master).
Contributed by Oleg Kalnichevski <olegk at apache.org>
* HTTPCLIENT-1968: Make normalization of URI paths optional.
Contributed by Tamas Cservenak <tamas at cservenak.net>
* Some well known proxies respond with Content-Length=0, when returning 304. For robustness, always use the
cached entity's content length, as modern browsers do.
Contributed by Author: Jayson Raymond <jraymond at accelerantmobile.com>
Release 4.5.7
-------------------
......
......@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-client</artifactId>
<version>4.5.7</version>
<version>4.5.9</version>
</parent>
<artifactId>fluent-hc</artifactId>
<name>Apache HttpClient Fluent API</name>
......
......@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-client</artifactId>
<version>4.5.7</version>
<version>4.5.9</version>
</parent>
<artifactId>httpclient-cache</artifactId>
<name>Apache HttpClient Cache</name>
......
......@@ -27,9 +27,8 @@
package org.apache.http.impl.client.cache;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
......@@ -44,7 +43,11 @@ import org.apache.http.annotation.Contract;
import org.apache.http.annotation.ThreadingBehavior;
import org.apache.http.client.cache.HeaderConstants;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URIUtils;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.util.Args;
/**
* @since 4.1
......@@ -54,6 +57,54 @@ class CacheKeyGenerator {
private static final URI BASE_URI = URI.create("http://example.com/");
static URIBuilder getRequestUriBuilder(final HttpRequest request) throws URISyntaxException {
if (request instanceof HttpUriRequest) {
final URI uri = ((HttpUriRequest) request).getURI();
if (uri != null) {
return new URIBuilder(uri);
}
}
return new URIBuilder(request.getRequestLine().getUri());
}
static URI getRequestUri(final HttpRequest request, final HttpHost target) throws URISyntaxException {
Args.notNull(request, "HTTP request");
Args.notNull(target, "Target");
final URIBuilder uriBuilder = getRequestUriBuilder(request);
// Decode path segments to preserve original behavior for backward compatibility
final String path = uriBuilder.getPath();
if (path != null) {
uriBuilder.setPathSegments(URLEncodedUtils.parsePathSegments(path));
}
if (!uriBuilder.isAbsolute()) {
uriBuilder.setScheme(target.getSchemeName());
uriBuilder.setHost(target.getHostName());
uriBuilder.setPort(target.getPort());
}
return uriBuilder.build();
}
static URI normalize(final URI requestUri) throws URISyntaxException {
Args.notNull(requestUri, "URI");
final URIBuilder builder = new URIBuilder(requestUri.isAbsolute() ? URIUtils.resolve(BASE_URI, requestUri) : requestUri) ;
if (builder.getHost() != null) {
if (builder.getScheme() == null) {
builder.setScheme("http");
}
if (builder.getPort() <= -1) {
if ("http".equalsIgnoreCase(builder.getScheme())) {
builder.setPort(80);
} else if ("https".equalsIgnoreCase(builder.getScheme())) {
builder.setPort(443);
}
}
}
builder.setFragment(null);
return builder.build();
}
/**
* For a given {@link HttpHost} and {@link HttpRequest} get a URI from the
* pair that I can use as an identifier KEY into my HttpCache
......@@ -63,45 +114,23 @@ class CacheKeyGenerator {
* @return String the extracted URI
*/
public String getURI(final HttpHost host, final HttpRequest req) {
if (isRelativeRequest(req)) {
return canonicalizeUri(String.format("%s%s", host.toString(), req.getRequestLine().getUri()));
try {
final URI uri = normalize(getRequestUri(req, host));
return uri.toASCIIString();
} catch (final URISyntaxException ex) {
return req.getRequestLine().getUri();
}
return canonicalizeUri(req.getRequestLine().getUri());
}
public String canonicalizeUri(final String uri) {
try {
final URI normalized = URIUtils.resolve(BASE_URI, uri);
final URL u = new URL(normalized.toASCIIString());
final String protocol = u.getProtocol();
final String hostname = u.getHost();
final int port = canonicalizePort(u.getPort(), protocol);
final String path = u.getPath();
final String query = u.getQuery();
final String file = (query != null) ? (path + "?" + query) : path;
final URL out = new URL(protocol, hostname, port, file);
return out.toString();
} catch (final IllegalArgumentException e) {
return uri;
} catch (final MalformedURLException e) {
final URI normalized = normalize(URIUtils.resolve(BASE_URI, uri));
return normalized.toASCIIString();
} catch (final URISyntaxException ex) {
return uri;
}
}
private int canonicalizePort(final int port, final String protocol) {
if (port == -1 && "http".equalsIgnoreCase(protocol)) {
return 80;
} else if (port == -1 && "https".equalsIgnoreCase(protocol)) {
return 443;
}
return port;
}
private boolean isRelativeRequest(final HttpRequest req) {
final String requestUri = req.getRequestLine().getUri();
return ("*".equals(requestUri) || requestUri.startsWith("/"));
}
protected String getFullHeaderValue(final Header[] headers) {
if (headers == null) {
return "";
......
......@@ -151,13 +151,10 @@ class CachedHttpResponseGenerator {
if (transferEncodingIsPresent(response)) {
return;
}
Header contentLength = response.getFirstHeader(HTTP.CONTENT_LEN);
if (contentLength == null) {
contentLength = new BasicHeader(HTTP.CONTENT_LEN, Long.toString(entity
.getContentLength()));
response.setHeader(contentLength);
}
// Some well known proxies respond with Content-Length=0, when returning 304. For robustness, always
// use the cached entity's content length, as modern browsers do.
final Header contentLength = new BasicHeader(HTTP.CONTENT_LEN, Long.toString(entity.getContentLength()));
response.setHeader(contentLength);
}
private boolean transferEncodingIsPresent(final HttpResponse response) {
......
......@@ -754,7 +754,7 @@ public class CachingExec implements ClientExecChain {
final URI uri = conditionalRequest.getURI();
if (uri != null) {
try {
conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route));
conditionalRequest.setURI(URIUtils.rewriteURIForRoute(uri, route, context.getRequestConfig().isNormalizeUri()));
} catch (final URISyntaxException ex) {
throw new ProtocolException("Invalid URI: " + uri, ex);
}
......
......@@ -30,7 +30,12 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.ObjectStreamClass;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.http.annotation.Contract;
import org.apache.http.annotation.ThreadingBehavior;
......@@ -49,6 +54,22 @@ import org.apache.http.client.cache.HttpCacheEntrySerializer;
@Contract(threading = ThreadingBehavior.IMMUTABLE)
public class DefaultHttpCacheEntrySerializer implements HttpCacheEntrySerializer {
private static final List<Pattern> ALLOWED_CLASS_PATTERNS = Collections.unmodifiableList(Arrays.asList(
Pattern.compile("^(\\[L)?org\\.apache\\.http\\.(.*)"),
Pattern.compile("^(\\[L)?java\\.util\\.(.*)"),
Pattern.compile("^(\\[L)?java\\.lang\\.(.*)$"),
Pattern.compile("^\\[B$")));
private final List<Pattern> allowedClassPatterns;
DefaultHttpCacheEntrySerializer(final Pattern... allowedClassPatterns) {
this.allowedClassPatterns = Collections.unmodifiableList(Arrays.asList(allowedClassPatterns));
}
public DefaultHttpCacheEntrySerializer() {
this.allowedClassPatterns = ALLOWED_CLASS_PATTERNS;
}
@Override
public void writeTo(final HttpCacheEntry cacheEntry, final OutputStream os) throws IOException {
final ObjectOutputStream oos = new ObjectOutputStream(os);
......@@ -61,7 +82,7 @@ public class DefaultHttpCacheEntrySerializer implements HttpCacheEntrySerializer
@Override
public HttpCacheEntry readFrom(final InputStream is) throws IOException {
final ObjectInputStream ois = new ObjectInputStream(is);
final ObjectInputStream ois = new RestrictedObjectInputStream(is, allowedClassPatterns);
try {
return (HttpCacheEntry) ois.readObject();
} catch (final ClassNotFoundException ex) {
......@@ -71,4 +92,32 @@ public class DefaultHttpCacheEntrySerializer implements HttpCacheEntrySerializer
}
}
private static class RestrictedObjectInputStream extends ObjectInputStream {
private final List<Pattern> allowedClassPatterns;
private RestrictedObjectInputStream(final InputStream in, final List<Pattern> patterns) throws IOException {
super(in);
this.allowedClassPatterns = patterns;
}
@Override
protected Class<?> resolveClass(final ObjectStreamClass desc) throws IOException, ClassNotFoundException {
if (isProhibited(desc)) {
throw new HttpCacheEntrySerializationException(String.format(
"Class %s is not allowed for deserialization", desc.getName()));
}
return super.resolveClass(desc);
}
private boolean isProhibited(final ObjectStreamClass desc) {
for (final Pattern pattern : allowedClassPatterns) {
if (pattern.matcher(desc.getName()).matches()) {
return false;
}
}
return true;
}
}
}
......@@ -32,11 +32,13 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectOutputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.Header;
......@@ -44,6 +46,7 @@ import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.cache.HeaderConstants;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializationException;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.Resource;
import org.apache.http.message.BasicHeader;
......@@ -51,6 +54,8 @@ import org.apache.http.message.BasicStatusLine;
import org.junit.Before;
import org.junit.Test;
import com.sun.rowset.JdbcRowSetImpl;
public class TestHttpCacheEntrySerializers {
private static final Charset UTF8 = Charset.forName("UTF-8");
......@@ -67,6 +72,43 @@ public class TestHttpCacheEntrySerializers {
readWriteVerify(makeCacheEntryWithVariantMap());
}
@Test(expected = HttpCacheEntrySerializationException.class)
public void throwExceptionIfUnsafeDeserialization() throws IOException {
impl.readFrom(new ByteArrayInputStream(serializeProhibitedObject()));
}
@Test(expected = HttpCacheEntrySerializationException.class)
public void allowClassesToBeDeserialized() throws IOException {
impl = new DefaultHttpCacheEntrySerializer(
Pattern.compile("javax.sql.rowset.BaseRowSet"),
Pattern.compile("com.sun.rowset.JdbcRowSetImpl"));
readVerify(serializeProhibitedObject());
}
@Test(expected = HttpCacheEntrySerializationException.class)
public void allowClassesToBeDeserializedByRegex() throws IOException {
impl = new DefaultHttpCacheEntrySerializer(
Pattern.compile(("^com\\.sun\\.rowset\\.(.*)")),
Pattern.compile("^javax\\.sql\\.rowset\\.BaseRowSet$"));
readVerify(serializeProhibitedObject());
}
private byte[] serializeProhibitedObject() throws IOException {
final JdbcRowSetImpl jdbcRowSet = new JdbcRowSetImpl();
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
final ObjectOutputStream oos = new ObjectOutputStream(baos);
try {
oos.writeObject(jdbcRowSet);
} finally {
oos.close();
}
return baos.toByteArray();
}
private void readVerify(final byte[] data) throws IOException {
impl.readFrom(new ByteArrayInputStream(data));
}
public void readWriteVerify(final HttpCacheEntry writeEntry) throws IOException {
// write the entry
final ByteArrayOutputStream out = new ByteArrayOutputStream();
......
......@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-client</artifactId>
<version>4.5.7</version>
<version>4.5.9</version>
</parent>
<artifactId>httpclient-osgi</artifactId>
<name>Apache HttpClient OSGi bundle</name>
......
......@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-client</artifactId>
<version>4.5.7</version>
<version>4.5.9</version>
</parent>
<artifactId>httpclient-win</artifactId>
<name>Apache HttpClient Windows features</name>
......
......@@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-client</artifactId>
<version>4.5.7</version>
<version>4.5.9</version>
</parent>
<artifactId>httpclient</artifactId>
<name>Apache HttpClient</name>
......
......@@ -137,7 +137,7 @@ public class DefaultRedirectHandler implements RedirectHandler {
try {
final URI requestURI = new URI(request.getRequestLine().getUri());
final URI absoluteRequestURI = URIUtils.rewriteURI(requestURI, target, true);
final URI absoluteRequestURI = URIUtils.rewriteURI(requestURI, target, URIUtils.DROP_FRAGMENT_AND_NORMALIZE);
uri = URIUtils.resolve(absoluteRequestURI, uri);
} catch (final URISyntaxException ex) {
throw new ProtocolException(ex.getMessage(), ex);
......@@ -161,7 +161,7 @@ public class DefaultRedirectHandler implements RedirectHandler {
uri.getHost(),
uri.getPort(),
uri.getScheme());
redirectURI = URIUtils.rewriteURI(uri, target, true);
redirectURI = URIUtils.rewriteURI(uri, target, URIUtils.DROP_FRAGMENT_AND_NORMALIZE);
} catch (final URISyntaxException ex) {
throw new ProtocolException(ex.getMessage(), ex);
}
......
......@@ -337,14 +337,14 @@ public class DefaultRequestDirector implements RequestDirector {
// Make sure the request URI is absolute
if (!uri.isAbsolute()) {
final HttpHost target = route.getTargetHost();
uri = URIUtils.rewriteURI(uri, target, true);
uri = URIUtils.rewriteURI(uri, target, URIUtils.DROP_FRAGMENT_AND_NORMALIZE);
} else {
uri = URIUtils.rewriteURI(uri);
}
} else {
// Make sure the request URI is relative
if (uri.isAbsolute()) {
uri = URIUtils.rewriteURI(uri, null, true);
uri = URIUtils.rewriteURI(uri, null, URIUtils.DROP_FRAGMENT_AND_NORMALIZE);
} else {
uri = URIUtils.rewriteURI(uri);
}
......
......@@ -60,12 +60,13 @@ public class RequestConfig implements Cloneable {
private final int connectTimeout;
private final int socketTimeout;
private final boolean contentCompressionEnabled;
private final boolean normalizeUri;
/**
* Intended for CDI compatibility
*/
protected RequestConfig() {
this(false, null, null, false, null, false, false, false, 0, false, null, null, 0, 0, 0, true);
this(false, null, null, false, null, false, false, false, 0, false, null, null, 0, 0, 0, true, true);
}
RequestConfig(
......@@ -84,7 +85,8 @@ public class RequestConfig implements Cloneable {
final int connectionRequestTimeout,
final int connectTimeout,
final int socketTimeout,
final boolean contentCompressionEnabled) {
final boolean contentCompressionEnabled,
final boolean normalizeUri) {
super();
this.expectContinueEnabled = expectContinueEnabled;
this.proxy = proxy;
......@@ -102,6 +104,7 @@ public class RequestConfig implements Cloneable {
this.connectTimeout = connectTimeout;
this.socketTimeout = socketTimeout;
this.contentCompressionEnabled = contentCompressionEnabled;
this.normalizeUri = normalizeUri;
}
/**
......@@ -332,6 +335,18 @@ public class RequestConfig implements Cloneable {
return contentCompressionEnabled;
}
/**
* Determines whether client should normalize URIs in requests or not.
* <p>
* Default: {@code true}
* </p>
*
* @since 4.5.8
*/
public boolean isNormalizeUri() {
return normalizeUri;
}
@Override
protected RequestConfig clone() throws CloneNotSupportedException {
return (RequestConfig) super.clone();
......@@ -356,6 +371,7 @@ public class RequestConfig implements Cloneable {
builder.append(", connectTimeout=").append(connectTimeout);
builder.append(", socketTimeout=").append(socketTimeout);
builder.append(", contentCompressionEnabled=").append(contentCompressionEnabled);
builder.append(", normalizeUri=").append(normalizeUri);
builder.append("]");
return builder.toString();
}
......@@ -383,7 +399,8 @@ public class RequestConfig implements Cloneable {
.setConnectTimeout(config.getConnectTimeout())
.setSocketTimeout(config.getSocketTimeout())
.setDecompressionEnabled(config.isDecompressionEnabled())
.setContentCompressionEnabled(config.isContentCompressionEnabled());
.setContentCompressionEnabled(config.isContentCompressionEnabled())
.setNormalizeUri(config.isNormalizeUri());
}
public static class Builder {
......@@ -404,6 +421,7 @@ public class RequestConfig implements Cloneable {
private int connectTimeout;
private int socketTimeout;
private boolean contentCompressionEnabled;
private boolean normalizeUri;
Builder() {
super();
......@@ -416,6 +434,7 @@ public class RequestConfig implements Cloneable {
this.connectTimeout = -1;
this.socketTimeout = -1;
this.contentCompressionEnabled = true;
this.normalizeUri = true;
}
public Builder setExpectContinueEnabled(final boolean expectContinueEnabled) {
......@@ -513,6 +532,11 @@ public class RequestConfig implements Cloneable {
return this;
}
public Builder setNormalizeUri(final boolean normalizeUri) {
this.normalizeUri = normalizeUri;
return this;
}
public RequestConfig build() {
return new RequestConfig(
expectContinueEnabled,
......@@ -530,7 +554,8 @@ public class RequestConfig implements Cloneable {
connectionRequestTimeout,
connectTimeout,
socketTimeout,
contentCompressionEnabled);
contentCompressionEnabled,
normalizeUri);
}
}
......
......@@ -108,7 +108,7 @@ public class EntityBuilder {
* <li>{@link #setParameters(java.util.List)}</li>
* <li>{@link #setParameters(org.apache.http.NameValuePair...)}</li>
* <li>{@link #setFile(java.io.File)}</li>
* <ul>
* </ul>
*/
public EntityBuilder setText(final String text) {
clearContent();
......@@ -250,7 +250,7 @@ public class EntityBuilder {
* <li>{@link #setParameters(java.util.List)}</li>
* <li>{@link #setParameters(org.apache.http.NameValuePair...)}</li>
* <li>{@link #setSerializable(java.io.Serializable)}</li>
* <ul>
* </ul>
*/
public EntityBuilder setFile(final File file) {
clearContent();
......
......@@ -30,9 +30,10 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import org.apache.http.Consts;
import org.apache.http.NameValuePair;
import org.apache.http.conn.util.InetAddressUtils;
......@@ -53,8 +54,8 @@ public class URIBuilder {
private String encodedUserInfo;
private String host;
private int port;
private String path;
private String encodedPath;
private List<String> pathSegments;
private String encodedQuery;
private List<NameValuePair> queryParams;
private String query;
......@@ -112,6 +113,13 @@ public class URIBuilder {
return null;
}
private List <String> parsePath(final String path, final Charset charset) {
if (path != null && !path.isEmpty()) {
return URLEncodedUtils.parsePathSegments(path, charset);
}
return null;
}
/**
* Builds a {@link URI} instance.
*/
......@@ -147,8 +155,8 @@ public class URIBuilder {
}
if (this.encodedPath != null) {
sb.append(normalizePath(this.encodedPath, sb.length() == 0));
} else if (this.path != null) {
sb.append(encodePath(normalizePath(this.path, sb.length() == 0)));
} else if (this.pathSegments != null) {
sb.append(encodePath(this.pathSegments));
}
if (this.encodedQuery != null) {
sb.append("?").append(this.encodedQuery);
......@@ -186,7 +194,7 @@ public class URIBuilder {
this.encodedUserInfo = uri.getRawUserInfo();
this.userInfo = uri.getUserInfo();
this.encodedPath = uri.getRawPath();
this.path = uri.getPath();
this.pathSegments = parsePath(uri.getRawPath(), this.charset != null ? this.charset : Consts.UTF_8);
this.encodedQuery = uri.getRawQuery();
this.queryParams = parseQuery(uri.getRawQuery(), this.charset != null ? this.charset : Consts.UTF_8);
this.encodedFragment = uri.getRawFragment();
......@@ -197,8 +205,8 @@ public class URIBuilder {
return URLEncodedUtils.encUserInfo(userInfo, this.charset != null ? this.charset : Consts.UTF_8);
}
private String encodePath(final String path) {
return URLEncodedUtils.encPath(path, this.charset != null ? this.charset : Consts.UTF_8);
private String encodePath(final List<String> pathSegments) {
return URLEncodedUtils.formatSegments(pathSegments, this.charset != null ? this.charset : Consts.UTF_8);
}
private String encodeUrlForm(final List<NameValuePair> params) {
......@@ -259,9 +267,36 @@ public class URIBuilder {
/**
* Sets URI path. The value is expected to be unescaped and may contain non ASCII characters.
*
* @return this.
*/
public URIBuilder setPath(final String path) {
this.path = path;
return setPathSegments(path != null ? URLEncodedUtils.splitPathSegments(path) : null);
}
/**
* Sets URI path. The value is expected to be unescaped and may contain non ASCII characters.
*
* @return this.
*
* @since 4.5.8
*/
public URIBuilder setPathSegments(final String... pathSegments) {
this.pathSegments = pathSegments.length > 0 ? Arrays.asList(pathSegments) : null;
this.encodedSchemeSpecificPart = null;
this.encodedPath = null;
return this;
}
/**
* Sets URI path. The value is expected to be unescaped and may contain non ASCII characters.
*
* @return this.
*
* @since 4.5.8
*/
public URIBuilder setPathSegments(final List<String> pathSegments) {
this.pathSegments = pathSegments != null && pathSegments.size() > 0 ? new ArrayList<String>(pathSegments) : null;
this.encodedSchemeSpecificPart = null;
this.encodedPath = null;
return this;
......@@ -462,7 +497,7 @@ public class URIBuilder {
* @since 4.3
*/
public boolean isOpaque() {
return this.path == null;
return isPathEmpty();
}
public String getScheme() {
......@@ -481,14 +516,41 @@ public class URIBuilder {
return this.port;
}
/**
* @since 4.5.8
*/
public boolean isPathEmpty() {
return (this.pathSegments == null || this.pathSegments.isEmpty()) &&
(this.encodedPath == null || this.encodedPath.isEmpty());
}
/**
* @since 4.5.8
*/
public List<String> getPathSegments() {
return this.pathSegments != null ? new ArrayList<String>(this.pathSegments) : Collections.<String>emptyList();
}
public String getPath() {
return this.path;
if (this.pathSegments == null) {
return null;
}
final StringBuilder result = new StringBuilder();
for (final String segment : this.pathSegments) {
result.append('/').append(segment);
}
return result.toString();
}
/**
* @since 4.5.8
*/
public boolean isQueryEmpty() {
return (this.queryParams == null || this.queryParams.isEmpty()) && this.encodedQuery == null;
}
public List<NameValuePair> getQueryParams() {
return this.queryParams != null
? new ArrayList<NameValuePair>(this.queryParams)
: new ArrayList<NameValuePair>();
return this.queryParams != null ? new ArrayList<NameValuePair>(this.queryParams) : Collections.<NameValuePair>emptyList();
}
public String getFragment() {
......
......@@ -28,6 +28,9 @@ package org.apache.http.client.utils;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Stack;
......@@ -45,6 +48,44 @@ import org.apache.http.util.TextUtils;
*/
public class URIUtils {
/**
* Flags that control how URI is being rewritten.
*
* @since 4.5.8
*/
public enum UriFlag {
DROP_FRAGMENT,
NORMALIZE
}
/**
* Empty set of uri flags.
*
* @since 4.5.8
*/
public static final EnumSet<UriFlag> NO_FLAGS = EnumSet.noneOf(UriFlag.class);
/**
* Set of uri flags containing {@link UriFlag#DROP_FRAGMENT}.
*
* @since 4.5.8
*/
public static final EnumSet<UriFlag> DROP_FRAGMENT = EnumSet.of(UriFlag.DROP_FRAGMENT);
/**
* Set of uri flags containing {@link UriFlag#NORMALIZE}.
*
* @since 4.5.8
*/
public static final EnumSet<UriFlag> NORMALIZE = EnumSet.of(UriFlag.NORMALIZE);
/**
* Set of uri flags containing {@link UriFlag#DROP_FRAGMENT} and {@link UriFlag#NORMALIZE}.
*
* @since 4.5.8
*/
public static final EnumSet<UriFlag> DROP_FRAGMENT_AND_NORMALIZE = EnumSet.of(UriFlag.DROP_FRAGMENT, UriFlag.NORMALIZE);
/**
* Constructs a {@link URI} using all the parameters. This should be
* used instead of
......@@ -125,12 +166,40 @@ public class URIUtils {
*
* @throws URISyntaxException
* If the resulting URI is invalid.
* @deprecated (4.5.8) Use {@link #rewriteURI(URI, HttpHost, EnumSet)}
*/
@Deprecated
public static URI rewriteURI(
final URI uri,
final HttpHost target,
final boolean dropFragment) throws URISyntaxException {
final boolean dropFragment) throws URISyntaxException
{
return rewriteURI(uri, target, dropFragment ? DROP_FRAGMENT : NO_FLAGS);
}
/**
* A convenience method for creating a new {@link URI} whose scheme, host
* and port are taken from the target host, but whose path, query and
* fragment are taken from the existing URI. What exactly is used and how
* is driven by the passed in flags. The path is set to "/" if not explicitly specified.
*
* @param uri
* Contains the path, query and fragment to use.
* @param target
* Contains the scheme, host and port to use.
* @param flags
* True if the fragment should not be copied.
*
* @throws URISyntaxException
* If the resulting URI is invalid.
* @since 4.5.8
*/
public static URI rewriteURI(
final URI uri,
final HttpHost target,
final EnumSet<UriFlag> flags) throws URISyntaxException {
Args.notNull(uri, "URI");
Args.notNull(flags, "URI flags");
if (uri.isOpaque()) {
return uri;
}
......@@ -144,36 +213,34 @@ public class URIUtils {
uribuilder.setHost(null);
uribuilder.setPort(-1);
}
if (dropFragment) {
if (flags.contains(UriFlag.DROP_FRAGMENT)) {
uribuilder.setFragment(null);
}
final String path = uribuilder.getPath();
if (TextUtils.isEmpty(path)) {
uribuilder.setPath("/");
} else {
final StringBuilder buf = new StringBuilder(path.length());
boolean foundSlash = false;
for (int i = 0; i < path.length(); i++) {
final char ch = path.charAt(i);
if (ch != '/' || !foundSlash) {
buf.append(ch);
if (flags.contains(UriFlag.NORMALIZE)) {
final List<String> pathSegments = new ArrayList<String>(uribuilder.getPathSegments());
for (final Iterator<String> it = pathSegments.iterator(); it.hasNext(); ) {
final String pathSegment = it.next();
if (pathSegment.isEmpty() && it.hasNext()) {
it.remove();
}
foundSlash = ch == '/';
}
uribuilder.setPath(buf.toString());
uribuilder.setPathSegments(pathSegments);
}
if (uribuilder.isPathEmpty()) {
uribuilder.setPathSegments("");
}
return uribuilder.build();
}
/**
* A convenience method for
* {@link URIUtils#rewriteURI(URI, HttpHost, boolean)} that always keeps the
* {@link URIUtils#rewriteURI(URI, HttpHost, EnumSet)} that always keeps the
* fragment.
*/
public static URI rewriteURI(
final URI uri,
final HttpHost target) throws URISyntaxException {
return rewriteURI(uri, target, false);
return rewriteURI(uri, target, NORMALIZE);
}
/**
......@@ -196,6 +263,9 @@ public class URIUtils {
if (uribuilder.getUserInfo() != null) {
uribuilder.setUserInfo(null);
}
if (uribuilder.getPathSegments().isEmpty()) {
uribuilder.setPathSegments("");
}
if (TextUtils.isEmpty(uribuilder.getPath())) {
uribuilder.setPath("/");
}
......@@ -218,17 +288,32 @@ public class URIUtils {
* @since 4.4
*/
public static URI rewriteURIForRoute(final URI uri, final RouteInfo route) throws URISyntaxException {
return rewriteURIForRoute(uri, route, true);
}
/**
* A convenience method that optionally converts the original {@link java.net.URI} either
* to a relative or an absolute form as required by the specified route.
*
* @param uri
* original URI.
* @throws URISyntaxException
* If the resulting URI is invalid.
*
* @since 4.5.8
*/
public static URI rewriteURIForRoute(final URI uri, final RouteInfo route, final boolean normalizeUri) throws URISyntaxException {
if (uri == null) {
return null;
}
if (route.getProxyHost() != null && !route.isTunnelled()) {
// Make sure the request URI is absolute
return uri.isAbsolute()
? rewriteURI(uri)
: rewriteURI(uri, route.getTargetHost(), true);
? rewriteURI(uri)
: rewriteURI(uri, route.getTargetHost(), normalizeUri ? DROP_FRAGMENT_AND_NORMALIZE : DROP_FRAGMENT);
}
// Make sure the request URI is relative
return uri.isAbsolute() ? rewriteURI(uri, null, true) : rewriteURI(uri);
return uri.isAbsolute() ? rewriteURI(uri, null, normalizeUri ? DROP_FRAGMENT_AND_NORMALIZE : DROP_FRAGMENT) : rewriteURI(uri);
}
/**
......@@ -283,39 +368,32 @@ public class URIUtils {
*
* @param uri the original URI
* @return the URI without dot segments
*
* @since 4.5
*/
static URI normalizeSyntax(final URI uri) throws URISyntaxException {
public static URI normalizeSyntax(final URI uri) throws URISyntaxException {
if (uri.isOpaque() || uri.getAuthority() == null) {
// opaque and file: URIs
return uri;
}
Args.check(uri.isAbsolute(), "Base URI must be absolute");
final URIBuilder builder = new URIBuilder(uri);
final String path = builder.getPath();
if (path != null && !path.equals("/")) {
final String[] inputSegments = path.split("/");
final Stack<String> outputSegments = new Stack<String>();
for (final String inputSegment : inputSegments) {
if ((inputSegment.isEmpty()) || (".".equals(inputSegment))) {
// Do nothing
} else if ("..".equals(inputSegment)) {
if (!outputSegments.isEmpty()) {
outputSegments.pop();
}
} else {
outputSegments.push(inputSegment);
final List<String> inputSegments = builder.getPathSegments();
final Stack<String> outputSegments = new Stack<String>();
for (final String inputSegment : inputSegments) {
if (".".equals(inputSegment)) {
// Do nothing
} else if ("..".equals(inputSegment)) {
if (!outputSegments.isEmpty()) {
outputSegments.pop();
}
} else {
outputSegments.push(inputSegment);
}
final StringBuilder outputBuffer = new StringBuilder();
for (final String outputSegment : outputSegments) {
outputBuffer.append('/').append(outputSegment);
}
if (path.lastIndexOf('/') == path.length() - 1) {
// path.endsWith("/") || path.equals("")
outputBuffer.append('/');
}
builder.setPath(outputBuffer.toString());
}
if (outputSegments.size() == 0) {
outputSegments.add("");
}
builder.setPathSegments(outputSegments);
if (builder.getScheme() != null) {
builder.setScheme(builder.getScheme().toLowerCase(Locale.ROOT));
}
......
......@@ -36,7 +36,9 @@ import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Scanner;
......@@ -68,6 +70,12 @@ public class URLEncodedUtils {
private static final char QP_SEP_A = '&';
private static final char QP_SEP_S = ';';
private static final String NAME_VALUE_SEPARATOR = "=";
private static final char PATH_SEPARATOR = '/';
private static final BitSet PATH_SEPARATORS = new BitSet(256);
static {
PATH_SEPARATORS.set(PATH_SEPARATOR);
}
/**
* @deprecated 4.5 Use {@link #parse(URI, Charset)}
......@@ -78,19 +86,12 @@ public class URLEncodedUtils {
}
/**
* Returns a list of {@link NameValuePair NameValuePairs} as built from the URI's query portion. For example, a URI
* of {@code http://example.org/path/to/file?a=1&b=2&c=3} would return a list of three NameValuePairs, one for a=1,
* one for b=2, and one for c=3. By convention, {@code '&'} and {@code ';'} are accepted as parameter separators.
* <p>
* This is typically useful while parsing an HTTP PUT.
* Returns a list of {@link NameValuePair}s URI query parameters.
* By convention, {@code '&'} and {@code ';'} are accepted as parameter separators.
*
* This API is currently only used for testing.
*
* @param uri
* URI to parse
* @param charset
* Charset to use while parsing the query
* @return a list of {@link NameValuePair} as built from the URI's query portion.
* @param uri input URI.
* @param charset parameter charset.
* @return list of query parameters.
*
* @since 4.5
*/
......@@ -230,14 +231,12 @@ public class URLEncodedUtils {
}
/**
* Returns a list of {@link NameValuePair NameValuePairs} as parsed from the given string using the given character
* encoding. By convention, {@code '&'} and {@code ';'} are accepted as parameter separators.
* Returns a list of {@link NameValuePair}s URI query parameters.
* By convention, {@code '&'} and {@code ';'} are accepted as parameter separators.
*
* @param s
* text to parse.
* @param charset
* Encoding to use when decoding the parameters.
* @return a list of {@link NameValuePair} as built from the URI's query portion.
* @param s URI query component.
* @param charset charset to use when decoding the parameters.
* @return list of query parameters.
*
* @since 4.2
*/
......@@ -254,13 +253,10 @@ public class URLEncodedUtils {
* Returns a list of {@link NameValuePair NameValuePairs} as parsed from the given string using the given character
* encoding.
*
* @param s
* text to parse.
* @param charset
* Encoding to use when decoding the parameters.
* @param separators
* element separators.
* @return a list of {@link NameValuePair} as built from the URI's query portion.
* @param s input text.
* @param charset parameter charset.
* @param separators parameter separators.
* @return list of query parameters.
*
* @since 4.3
*/
......@@ -274,8 +270,7 @@ public class URLEncodedUtils {
}
/**
* Returns a list of {@link NameValuePair NameValuePairs} as parsed from the given string using
* the given character encoding.
* Returns a list of {@link NameValuePair}s parameters.
*
* @param buf
* text to parse.
......@@ -321,6 +316,98 @@ public class URLEncodedUtils {
return list;
}
static List<String> splitSegments(final CharSequence s, final BitSet separators) {
final ParserCursor cursor = new ParserCursor(0, s.length());
// Skip leading separator
if (cursor.atEnd()) {
return Collections.emptyList();
}
if (separators.get(s.charAt(cursor.getPos()))) {
cursor.updatePos(cursor.getPos() + 1);
}
final List<String> list = new ArrayList<String>();
final StringBuilder buf = new StringBuilder();
for (;;) {
if (cursor.atEnd()) {
list.add(buf.toString());
break;
}
final char current = s.charAt(cursor.getPos());
if (separators.get(current)) {
list.add(buf.toString());
buf.setLength(0);
} else {
buf.append(current);
}
cursor.updatePos(cursor.getPos() + 1);
}
return list;
}
static List<String> splitPathSegments(final CharSequence s) {
return splitSegments(s, PATH_SEPARATORS);
}
/**
* Returns a list of URI path segments.
*
* @param s URI path component.
* @param charset parameter charset.
* @return list of segments.
*
* @since 4.5
*/
public static List<String> parsePathSegments(final CharSequence s, final Charset charset) {
Args.notNull(s, "Char sequence");
final List<String> list = splitPathSegments(s);
for (int i = 0; i < list.size(); i++) {
list.set(i, urlDecode(list.get(i), charset != null ? charset : Consts.UTF_8, false));
}
return list;
}
/**
* Returns a list of URI path segments.
*
* @param s URI path component.
* @return list of segments.
*
* @since 4.5
*/
public static List<String> parsePathSegments(final CharSequence s) {
return parsePathSegments(s, Consts.UTF_8);
}
/**
* Returns a string consisting of joint encoded path segments.
*
* @param segments the segments.
* @param charset parameter charset.
* @return URI path component
*
* @since 4.5
*/
public static String formatSegments(final Iterable<String> segments, final Charset charset) {
Args.notNull(segments, "Segments");
final StringBuilder result = new StringBuilder();
for (final String segment : segments) {
result.append(PATH_SEPARATOR).append(urlEncode(segment, charset, PATHSAFE, false));
}
return result.toString();
}
/**
* Returns a string consisting of joint encoded path segments.
*
* @param segments the segments.
* @return URI path component
*
* @since 4.5
*/
public static String formatSegments(final String... segments) {
return formatSegments(Arrays.asList(segments), Consts.UTF_8);
}
/**
* Returns a String that is suitable for use as an {@code application/x-www-form-urlencoded}
* list of parameters in an HTTP PUT or HTTP POST.
......@@ -454,6 +541,8 @@ public class URLEncodedUtils {
*/
private static final BitSet URLENCODER = new BitSet(256);
private static final BitSet PATH_SPECIAL = new BitSet(256);
static {
// unreserved chars
// alpha characters
......@@ -491,9 +580,8 @@ public class URLEncodedUtils {
// URL path safe
PATHSAFE.or(UNRESERVED);
PATHSAFE.set('/'); // segment separator
PATHSAFE.set(';'); // param separator
PATHSAFE.set(':'); // rest as per list in 2396, i.e. : @ & = + $ ,
PATHSAFE.set(':'); // RFC 2396
PATHSAFE.set('@');
PATHSAFE.set('&');
PATHSAFE.set('=');
......@@ -501,6 +589,9 @@ public class URLEncodedUtils {
PATHSAFE.set('$');
PATHSAFE.set(',');
PATH_SPECIAL.or(PATHSAFE);
PATH_SPECIAL.set('/');
RESERVED.set(';');
RESERVED.set('/');
RESERVED.set('?');
......@@ -683,7 +774,7 @@ public class URLEncodedUtils {
}
/**
* Encode a String using the {@link #PATHSAFE} set of characters.
* Encode a String using the {@link #PATH_SPECIAL} set of characters.
* <p>
* Used by URIBuilder to encode path segments.
*
......@@ -692,7 +783,7 @@ public class URLEncodedUtils {
* @return the encoded string
*/
static String encPath(final String content, final Charset charset) {
return urlEncode(content, charset, PATHSAFE, false);
return urlEncode(content, charset, PATH_SPECIAL, false);
}
}
......@@ -36,7 +36,9 @@ import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import javax.net.SocketFactory;
import javax.net.ssl.HostnameVerifier;
......@@ -163,6 +165,15 @@ public class SSLConnectionSocketFactory implements LayeredConnectionSocketFactor
public static final X509HostnameVerifier STRICT_HOSTNAME_VERIFIER
= StrictHostnameVerifier.INSTANCE;
private static final String WEAK_KEY_EXCHANGES
= "^(TLS|SSL)_(NULL|ECDH_anon|DH_anon|DH_anon_EXPORT|DHE_RSA_EXPORT|DHE_DSS_EXPORT|"
+ "DSS_EXPORT|DH_DSS_EXPORT|DH_RSA_EXPORT|RSA_EXPORT|KRB5_EXPORT)_(.*)";
private static final String WEAK_CIPHERS
= "^(TLS|SSL)_(.*)_WITH_(NULL|DES_CBC|DES40_CBC|DES_CBC_40|3DES_EDE_CBC|RC4_128|RC4_40|RC2_CBC_40)_(.*)";
private static final List<Pattern> WEAK_CIPHER_SUITE_PATTERNS = Collections.unmodifiableList(Arrays.asList(
Pattern.compile(WEAK_KEY_EXCHANGES, Pattern.CASE_INSENSITIVE),
Pattern.compile(WEAK_CIPHERS, Pattern.CASE_INSENSITIVE)));
private final Log log = LogFactory.getLog(getClass());
/**
......@@ -183,6 +194,15 @@ public class SSLConnectionSocketFactory implements LayeredConnectionSocketFactor
return new SSLConnectionSocketFactory(SSLContexts.createDefault(), getDefaultHostnameVerifier());
}
static boolean isWeakCipherSuite(final String cipherSuite) {
for (final Pattern pattern : WEAK_CIPHER_SUITE_PATTERNS) {
if (pattern.matcher(cipherSuite).matches()) {
return true;
}
}
return false;
}
private static String[] split(final String s) {
if (TextUtils.isBlank(s)) {
return null;
......@@ -392,6 +412,18 @@ public class SSLConnectionSocketFactory implements LayeredConnectionSocketFactor
}
if (supportedCipherSuites != null) {
sslsock.setEnabledCipherSuites(supportedCipherSuites);
} else {
// If cipher suites are not explicitly set, remove all insecure ones
final String[] allCipherSuites = sslsock.getEnabledCipherSuites();
final List<String> enabledCipherSuites = new ArrayList<String>(allCipherSuites.length);
for (final String cipherSuite : allCipherSuites) {
if (!isWeakCipherSuite(cipherSuite)) {
enabledCipherSuites.add(cipherSuite);
}
}
if (!enabledCipherSuites.isEmpty()) {
sslsock.setEnabledCipherSuites(enabledCipherSuites.toArray(new String[enabledCipherSuites.size()]));
}
}
if (this.log.isDebugEnabled()) {
......
......@@ -142,17 +142,17 @@ public final class PublicSuffixMatcher {
if (domain.startsWith(".")) {
return null;
}
String domainName = null;
String segment = domain.toLowerCase(Locale.ROOT);
final String normalized = domain.toLowerCase(Locale.ROOT);
String segment = normalized;
String result = null;
while (segment != null) {
// An exception rule takes priority over any other matching rule.
if (hasException(IDN.toUnicode(segment), expectedType)) {
final String key = IDN.toUnicode(segment);
if (hasException(key, expectedType)) {
return segment;
}
if (hasRule(IDN.toUnicode(segment), expectedType)) {
break;
if (hasRule(key, expectedType)) {
return result;
}
final int nextdot = segment.indexOf('.');
......@@ -160,15 +160,13 @@ public final class PublicSuffixMatcher {
if (nextSegment != null) {
if (hasRule("*." + IDN.toUnicode(nextSegment), expectedType)) {
break;
return result;
}
}
if (nextdot != -1) {
domainName = segment;
}
result = segment;
segment = nextSegment;
}
return domainName;
return normalized;
}
/**
......