diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a41fa94..212ebd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * Added "disableSdkBehaviorSettingsUpdates()" init config method for disabling SDK behavior settings requests. * Improved request queue handling with a built-in backoff mechanism which is enabled by default. When the server takes 10 seconds or longer to answer while few requests are queued, the SDK pauses sending for 60 seconds. * Added "disableBackoffMechanism()" init config method to disable the backoff behavior. +* Added "enableRequestCompression()" init config method. Request bodies between 1 KB and 1 MB are then sent gzip-compressed, once the server's SDK behavior settings answer says it reads them. It does not apply in backend mode, which does not use the SDK behavior settings. * Added support for server requested SDK log gathering. When the server asks for it, the SDK uploads its own internal log lines, even if logging is disabled. While consent is required, a line is uploaded only while the consent of the feature it belongs to is given. * Added support for the SDK connection test. When the server asks for it, the SDK checks that it can reach every endpoint it depends on and reports the results. * Added "setMaxKeyLength(int)" init config method to limit the length of event keys, view names, segmentation keys and custom user property keys. Longer keys are truncated. There is no limit by default, and the key length limit of the SDK behavior settings overrides it. diff --git a/sdk-java/api/sdk-java.txt b/sdk-java/api/sdk-java.txt index 3a6c1a0e..0aec5f02 100644 --- a/sdk-java/api/sdk-java.txt +++ b/sdk-java/api/sdk-java.txt @@ -23,6 +23,7 @@ package ly.count.sdk.java { method public ly.count.sdk.java.Config! enableParameterTamperingProtection(String!); method public ly.count.sdk.java.Config! enableRemoteConfigAutomaticTriggers(); method public ly.count.sdk.java.Config! enableRemoteConfigValueCaching(); + method public ly.count.sdk.java.Config! enableRequestCompression(); method @Deprecated public ly.count.sdk.java.Config! enableTestMode(); method @Deprecated public ly.count.sdk.java.Config! enableUsePOST(); method public ly.count.sdk.java.Config! enrollABOnRCDownload(); @@ -131,6 +132,7 @@ package ly.count.sdk.java { field protected int networkRequestCooldown; field protected Set! publicKeyPins; field protected List! remoteConfigGlobalCallbacks; + field protected boolean requestCompressionEnabled; field protected int requestQueueMaxSize; field protected boolean requiresConsent; field protected String! salt; @@ -407,6 +409,7 @@ package ly.count.sdk.java.internal { } public interface ConfigurationProvider { + method public default void blockRequestCompression(); method public boolean getAutomaticCrashReportingEnabled(); method public boolean getAutomaticSessionTrackingEnabled(); method public boolean getAutomaticViewTrackingEnabled(); @@ -449,6 +452,7 @@ package ly.count.sdk.java.internal { method public int getUserPropertyCacheLimit(); method public ly.count.sdk.java.internal.ConfigurationProvider.FilterList!> getUserPropertyFilterList(); method public boolean getViewTrackingEnabled(); + method public default boolean isRequestCompressionAllowed(); method public boolean isRequestQueueMaxSizeFromBehaviorSettings(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/Config.java b/sdk-java/src/main/java/ly/count/sdk/java/Config.java index f49dac67..da5a8242 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/Config.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/Config.java @@ -272,6 +272,12 @@ public class Config { */ protected boolean sdkBehaviorSettingsRequestsDisabled = false; + /** + * Whether request bodies are sent gzip-compressed, once the server's SDK behavior settings + * answer says it reads them. + */ + protected boolean requestCompressionEnabled = false; + /** * Whether the backoff mechanism may hold the request queue back after the server answered * slowly. The SDK behavior settings can override it. @@ -1588,6 +1594,22 @@ public Config disableSdkBehaviorSettingsUpdates() { return this; } + /** + * Compress request bodies between 1 KB and 1 MB with gzip before sending them. The SDK + * compresses only after the server's SDK behavior settings answer said it reads compressed + * bodies, so older servers keep getting plain requests. It needs the SDK behavior settings + * updates, which backend mode and {@link #disableSdkBehaviorSettingsUpdates()} turn off. A + * compressed request that the server, or something in front of it, answers without accepting + * it stops compression until the next SDK behavior settings answer says again that the server + * reads compressed bodies. + * + * @return {@code this} instance for method chaining + */ + public Config enableRequestCompression() { + this.requestCompressionEnabled = true; + return this; + } + /** * Disable the backoff mechanism. While it is on, a request the server takes long to answer * makes the SDK wait before it sends the next queued request, which gives a busy server room diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java index 43356e0d..9845a302 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java @@ -297,6 +297,25 @@ public interface ConfigurationProvider { */ @Nonnull Set getJourneyTriggerViews(); + /** + * Whether a request body may be sent gzip-compressed now: the developer turned request + * compression on, the current server said in its SDK behavior settings answer that it reads + * compressed bodies, and no compressed request was refused since that answer. + * + * @return {@code true} when a request body may be compressed + */ + default boolean isRequestCompressionAllowed() { + return false; + } + + /** + * Stops request compression until the next SDK behavior settings answer says that the server + * reads compressed bodies, or the SDK starts again; called after the server, or something in + * front of it, answered a compressed request without accepting it. + */ + default void blockRequestCompression() { + } + /** * The decision of the {@code lg} directive, which sits next to {@code c} in a live response. * diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java index 66a8326a..e9f065f2 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.util.concurrent.CompletableFuture; +import org.json.JSONException; import org.json.JSONObject; /** @@ -93,8 +94,12 @@ private JSONObject doInBackground(String requestData, String customEndpoint, Tra return new JSONObject(receivedBuffer); } else { L.d("[ImmediateRequestMaker] Encountered problem while making a immediate server request, :[" + receivedBuffer + "]"); + cp.blockCompressionIfCompressed(connection); return null; } + } catch (JSONException | StringIndexOutOfBoundsException e) { + L.d("[ImmediateRequestMaker] Received an answer that is empty or not JSON while making a immediate server request, [" + e.getMessage() + "]"); + cp.blockCompressionIfCompressed(connection); } catch (Exception e) { L.d("[ImmediateRequestMaker] Received exception while making a immediate server request " + e.getMessage()); } finally { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java index 004afb9a..721ccf64 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java @@ -184,6 +184,15 @@ public boolean isSdkBehaviorSettingsRequestsDisabled() { return sdkBehaviorSettingsRequestsDisabled; } + /** + * Whether the developer turned request compression on, see {@link Config#enableRequestCompression()}. + * + * @return {@code true} after {@link Config#enableRequestCompression()} + */ + boolean isRequestCompressionEnabled() { + return requestCompressionEnabled; + } + /** * Whether the SDK sends its health check, see {@link Config#disableHealthCheck()}. * diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java index a5a212fc..0bfd5520 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Config; @@ -38,6 +40,8 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv static final String keyRConfig = "c"; static final String keyRLogGathering = "lg"; static final String keyRConnectionTest = "ct"; + //top level key, a sibling of 'c', read from a live response and never stored + static final String keyRGzip = "gzip"; static final String keyRTracking = "tracking"; static final String keyRNetworking = "networking"; @@ -112,6 +116,10 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv private boolean serverConfigRequestsDisabled = false; private volatile boolean running = false; + + private final AtomicBoolean requestCompressionEnabled = new AtomicBoolean(false); + final AtomicReference gzipAcceptingServerUrl = new AtomicReference<>(null); + final AtomicBoolean requestCompressionBlocked = new AtomicBoolean(false); /** * How many times the device ID changed without merge, written under this module's lock, so a * response requested before such a change can tell its gather was armed for the previous user. @@ -230,6 +238,7 @@ public void init(InternalConfig config) { super.init(config); boolean backendMode = config.isBackendModeEnabled(); serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled(); + requestCompressionEnabled.set(config.isRequestCompressionEnabled()); L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]"); seedFromDeveloperConfig(config); @@ -899,7 +908,11 @@ void fetchConfigFromServer(@Nonnull InternalConfig config) { } try { - String requestData = ModuleRequests.prepareRequiredParams(config).add("method", "sc").toString(); + Params params = ModuleRequests.prepareRequiredParams(config).add("method", "sc"); + if (requestCompressionEnabled.get()) { + params.add("gzip", "1"); + } + String requestData = params.toString(); Transport transport = config.sdk.networking.getTransport(); final long fetchStartNs = System.nanoTime(); final int requestedAfterDeviceIdChanges = deviceIdChangesWithoutMerge; @@ -946,6 +959,14 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS L.d("[ModuleConfiguration] onServerConfigResponse, retrieved configuration response: [" + response + "]"); long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L; boolean connectionTestRequested = extractConnectionTestFlag(response); + boolean gzipAccepted = extractGzipFlag(response); + if (requestCompressionEnabled.get()) { + gzipAcceptingServerUrl.set(gzipAccepted ? requestedServerUrl : null); + if (gzipAccepted && requestCompressionBlocked.compareAndSet(true, false)) { + L.i("[ModuleConfiguration] onServerConfigResponse, a behavior settings answer says that the server reads compressed bodies, requests are compressed again"); + } + L.d("[ModuleConfiguration] onServerConfigResponse, the server " + (gzipAccepted ? "reads" : "does not say it reads") + " compressed request bodies"); + } boolean changed; synchronized (this) { @@ -967,6 +988,21 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS } } + /** + * Reads and removes the {@code gzip} flag of a live response, so it is never stored: only an + * answer that carries {@code true} says that the server reads compressed request bodies. + * + * @param serverConfigResponse the live response, changed in place + * @return {@code true} only when the response had {@code "gzip": true} + */ + static boolean extractGzipFlag(@Nullable JSONObject serverConfigResponse) { + if (serverConfigResponse == null || !serverConfigResponse.has(keyRGzip)) { + return false; + } + + return Boolean.TRUE.equals(serverConfigResponse.remove(keyRGzip)); + } + /** * Reads and removes the {@code ct} flag of a live response, so it is never stored and can never * come back from storage. Booleans and numbers count by value; a string counts unless it is @@ -1060,6 +1096,21 @@ private synchronized void stopServerConfigUpdateTimer() { } /** {@inheritDoc} */ + @Override + public boolean isRequestCompressionAllowed() { + String acceptingUrl = gzipAcceptingServerUrl.get(); + InternalConfig config = internalConfig; + return requestCompressionEnabled.get() && !requestCompressionBlocked.get() && acceptingUrl != null && config != null + && acceptingUrl.equals(config.getServerURL().toString()); + } + + @Override + public void blockRequestCompression() { + if (requestCompressionBlocked.compareAndSet(false, true)) { + L.i("[ModuleConfiguration] blockRequestCompression, a compressed request was refused, requests are sent uncompressed until a behavior settings answer says that the server reads compressed bodies, or the SDK starts again"); + } + } + @Override public boolean getNetworkingEnabled() { return currentVNetworking; diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java index 4bcfd1e4..3bd05c6e 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java @@ -64,6 +64,7 @@ public class Transport implements X509TrustManager { private Log L = null; private static final String PARAMETER_TAMPERING_DIGEST = "SHA-256"; private static final String CHECKSUM = "checksum256"; + static final String REQUEST_COMPRESSION_MARKER = "content_encoding=gzip"; private InternalConfig config; private SSLContext sslContext; // ssl context to use if pinning is enabled @@ -150,6 +151,22 @@ HttpURLConnection connection(final Request request) throws IOException { request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); } + byte[] compressedBody = new byte[0]; + if (!usingGET && maybePictureData == null) { + if (config.getParameterTamperingProtectionSalt() != null) { + request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); + } + String markedPath = compressedRequestPath(path); + if (markedPath != null && config.getConfigurationProvider().isRequestCompressionAllowed()) { + String body = request.params.toString(); + compressedBody = Utils.gzipRequestBody(body, L); + if (compressedBody.length > 0) { + path = markedPath; + L.d("[Transport] connection, the request body is sent gzip-compressed, [" + body.length() + "] -> [" + compressedBody.length + "] bytes"); + } + } + } + HttpURLConnection connection = openConnection(path, request.params.toString(), usingGET); connection.setConnectTimeout(1000 * config.getNetworkConnectionTimeout()); connection.setReadTimeout(1000 * config.getNetworkReadTimeout()); @@ -191,17 +208,19 @@ HttpURLConnection connection(final Request request) throws IOException { writer.append("--").append(boundary).append("--").append(Utils.CRLF).flush(); } else { //picture data is "null". If it was sent, we send "null" to server to clear the image there - //we send a normal request in HTTP POST - if (config.getParameterTamperingProtectionSalt() != null) { - request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); - } + //we send a normal request in HTTP POST, its checksum was added above connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); output = connection.getOutputStream(); - writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true); + if (compressedBody.length > 0) { + output.write(compressedBody); + output.flush(); + } else { + writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true); - writer.write(request.params.toString()); - writer.flush(); + writer.write(request.params.toString()); + writer.flush(); + } } } finally { if (writer != null) { @@ -222,6 +241,44 @@ HttpURLConnection connection(final Request request) throws IOException { return connection; } + /** + * The URL a compressed request goes to: the request URL with the compression marker as its only + * query. Endpoints end with an empty query, as in {@code /i?}, which the marker fills. + * + * @param url the URL the request goes to + * @return the URL with the marker, or {@code null} when the URL already has a query, where the + * marker could not be the only one + */ + static String compressedRequestPath(@Nonnull String url) { + int query = url.indexOf('?'); + if (query == -1) { + return url + "?" + REQUEST_COMPRESSION_MARKER; + } + return query == url.length() - 1 ? url + REQUEST_COMPRESSION_MARKER : null; + } + + /** + * Tells whether a connection made by {@link #connection(Request)} sends its body gzip-compressed. + * + * @param connection the connection + * @return {@code true} when its URL carries the compression marker as its query + */ + static boolean isCompressedRequest(@Nullable HttpURLConnection connection) { + return connection != null && REQUEST_COMPRESSION_MARKER.equals(connection.getURL().getQuery()); + } + + /** + * Stops request compression when the connection sent a compressed body; called after its answer + * was not accepted. + * + * @param connection the connection whose answer was not accepted + */ + void blockCompressionIfCompressed(@Nullable HttpURLConnection connection) { + if (isCompressedRequest(connection)) { + config.getConfigurationProvider().blockRequestCompression(); + } + } + /** * Opens the GET of one connection test probe: a bare request, built as * {@link #openBareGetConnection(String, int)} builds it, that verifies the server against the @@ -444,7 +501,11 @@ public Boolean send() { SDKCore.instance.onRequestCompleted(request, response, code, requestOwner); - return processResponse(code, response, request.storageId()); + Boolean accepted = processResponse(code, response, request.storageId()); + if (!Boolean.TRUE.equals(accepted)) { + blockCompressionIfCompressed(connection); + } + return accepted; } catch (IOException e) { L.d("[network] Error while sending request " + request + " " + e); notifyOwnerOfAttemptWithoutResponse(request, requestOwner); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java index b7ac34c8..51b4785b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; /** * Utility class @@ -31,6 +32,9 @@ public class Utils { public static final String UTF8 = "UTF-8"; public static final String CRLF = "\r\n"; public static final char[] BASE_16 = "0123456789ABCDEF".toCharArray(); + static final int REQUEST_COMPRESSION_MIN_BYTES = 1024; + // the server refuses a compressed body that is larger than this once decoded + static final int REQUEST_COMPRESSION_MAX_BYTES = 1024 * 1024; /** * Joins objects with a separator @@ -183,6 +187,33 @@ public static String digestHex(final String digestName, final String string, Log } } + /** + * Compresses a request body with gzip, for sending it with the compression marker as the URL query. + * + * @param body the url encoded request body, checksum included + * @param L logger + * @return the gzip bytes, or an empty array when the body is smaller than + * {@link #REQUEST_COMPRESSION_MIN_BYTES} or larger than {@link #REQUEST_COMPRESSION_MAX_BYTES}, + * compressing fails, or the result is not smaller than the body + */ + static byte[] gzipRequestBody(String body, Log L) { + byte[] plain = body.getBytes(StandardCharsets.UTF_8); + if (plain.length < REQUEST_COMPRESSION_MIN_BYTES || plain.length > REQUEST_COMPRESSION_MAX_BYTES) { + return new byte[0]; + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(plain.length / 2); + try (GZIPOutputStream gzip = new GZIPOutputStream(out)) { + gzip.write(plain); + } catch (IOException e) { + L.d("[Utils] gzipRequestBody, compressing failed, the body is sent uncompressed: [" + e + "]"); + return new byte[0]; + } + + byte[] compressed = out.toByteArray(); + return compressed.length < plain.length ? compressed : new byte[0]; + } + /** * Get hexadecimal string representation of a byte array * diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java new file mode 100644 index 00000000..2e5595c5 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java @@ -0,0 +1,82 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPInputStream; + +/** + * One request as {@link GzipStandInServer} received it: the body decoded and its parameters read + * the way the Countly server reads them, which takes a body as gzip only when the compression + * marker is the only query of the URL. + */ +final class GzipStandInRequest { + + static final String MARKER_QUERY = "content_encoding=gzip"; + + final String method; + final String path; + final boolean compressed; + final String body; + final int bodyBytes; + final String contentType; + final String contentEncoding; + final Map params; + final AtomicBoolean answered = new AtomicBoolean(false); + + /** + * Reads a request from its exchange. + * + * @param exchange the exchange the request arrived in + * @throws IOException when the body cannot be read or decoded + */ + GzipStandInRequest(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getPath(); + String query = exchange.getRequestURI().getRawQuery(); + compressed = MARKER_QUERY.equals(query); + byte[] raw = readAll(exchange.getRequestBody()); + bodyBytes = raw.length; + body = new String(compressed ? gunzip(raw) : raw, StandardCharsets.UTF_8); + contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + contentEncoding = exchange.getRequestHeaders().getFirst("Content-Encoding"); + params = TestUtils.parseQueryParams(compressed || !Utils.isNotEmpty(query) ? body : query); + params.replaceAll((key, value) -> Utils.urldecode(value)); + } + + /** + * Decodes gzip bytes. + * + * @param compressed gzip bytes + * @return the decoded bytes + * @throws IOException when the bytes are not gzip + */ + static byte[] gunzip(byte[] compressed) throws IOException { + try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + return readAll(in); + } + } + + /** + * Reads a stream to its end. + * + * @param stream the stream to read + * @return every byte of the stream + * @throws IOException when the stream cannot be read + */ + static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read = stream.read(buffer); + while (read != -1) { + out.write(buffer, 0, read); + read = stream.read(buffer); + } + return out.toByteArray(); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java new file mode 100644 index 00000000..558ec569 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java @@ -0,0 +1,161 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.json.JSONObject; +import org.junit.Assert; + +/** + * A Countly server stand-in on loopback for the request compression tests, which records every + * request. It answers the SDK behavior settings with the answer set by {@link #answer(int, boolean)}, + * remote config with {@link #remoteConfigCode} and {@link #remoteConfigAnswer}, and anything else + * with success; it can refuse or drop compressed requests and hold its settings answer until a + * latch opens. + */ +final class GzipStandInServer { + + final HttpServer http; + final List received = new CopyOnWriteArrayList<>(); + final AtomicReference settingsAnswer = new AtomicReference<>("{}"); + final AtomicReference remoteConfigAnswer = new AtomicReference<>("{}"); + final AtomicInteger remoteConfigCode = new AtomicInteger(200); + final AtomicInteger refuseCompressed = new AtomicInteger(0); + final AtomicBoolean dropCompressed = new AtomicBoolean(false); + final AtomicReference holdSettings = new AtomicReference<>(null); + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + GzipStandInServer() throws IOException { + http = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + http.createContext("/", this::handle); + http.setExecutor(Executors.newCachedThreadPool()); + http.start(); + } + + /** + * Sets the settings answer: an {@code lkl} value that tells it apart, with "gzip": true when + * the server reads compressed bodies. + * + * @param lkl the key length limit of the answer + * @param gzip whether the answer says that the server reads compressed bodies + */ + void answer(int lkl, boolean gzip) { + JSONObject answer = new JSONObject().put("v", 1).put("t", 1_700_000_000_000L).put("c", new JSONObject().put("lkl", lkl)); + if (gzip) { + answer.put("gzip", true); + } + settingsAnswer.set(answer.toString()); + } + + /** + * Records a request and answers it. + * + * @param exchange the exchange the request arrived in + * @throws IOException when the request cannot be read or answered + */ + private void handle(HttpExchange exchange) throws IOException { + GzipStandInRequest request = new GzipStandInRequest(exchange); + received.add(request); + String method = request.params.get("method"); + int code = 200; + String response = "{\"result\":\"Success\"}"; + if ("sc".equals(method)) { + CountDownLatch hold = holdSettings.get(); + if (hold != null) { + try { + hold.await(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + response = settingsAnswer.get(); + } else if ("rc".equals(method)) { + code = remoteConfigCode.get(); + response = remoteConfigAnswer.get(); + } + if (request.compressed && dropCompressed.get()) { + exchange.close(); + return; + } + if (request.compressed && refuseCompressed.getAndUpdate(left -> left > 0 ? left - 1 : 0) > 0) { + code = 400; + response = "{\"result\":\"Invalid request body\"}"; + } + byte[] out = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(code, out.length == 0 ? -1 : out.length); + try (OutputStream stream = exchange.getResponseBody()) { + stream.write(out); + } + request.answered.set(true); + } + + /** + * The URL the SDK reaches this server at. + * + * @return the server URL + */ + String url() { + return "http://127.0.0.1:" + http.getAddress().getPort(); + } + + /** + * How many received requests match. + * + * @param filter the requests to count + * @return the number of matching requests + */ + int count(Predicate filter) { + return (int) received.stream().filter(filter).count(); + } + + /** + * Every received request that matches, in arrival order. + * + * @param filter the requests to return + * @return the matching requests + */ + List all(Predicate filter) { + return received.stream().filter(filter).collect(Collectors.toList()); + } + + /** + * Waits for the first matching request and returns it. + * + * @param filter the request to wait for + * @return the first matching request + * @throws InterruptedException when the wait is interrupted + */ + GzipStandInRequest await(Predicate filter) throws InterruptedException { + Assert.assertTrue("no matching request arrived at " + url() + ", it received " + received.size(), TestUtils.waitFor(15_000, () -> count(filter) > 0)); + return received.stream().filter(filter).findFirst().orElse(null); + } + + /** + * Opens a held settings answer and stops the server. + */ + void stop() { + CountDownLatch hold = holdSettings.get(); + if (hold != null) { + hold.countDown(); + } + http.stop(0); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java index 2e69a4ef..318abc3c 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java @@ -44,7 +44,7 @@ @RunWith(JUnit4.class) public class ModuleConfigurationTests { - private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct"); + private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct", "gzip"); private ServerConfigResponder server; diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java new file mode 100644 index 00000000..d72dada1 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java @@ -0,0 +1,498 @@ +package ly.count.sdk.java.internal; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Request compression, against HTTP servers on loopback that read a body as gzip the way the + * Countly server does: only when the compression marker is the only query of the URL. Each + * settings answer carries its own {@code lkl} value, so a test can wait until the SDK has read it. + * The assertions are on what the servers received. + */ +@RunWith(JUnit4.class) +public class RequestCompressionTests { + + private static final String SALT = "gzip-salt"; + private static final int BIG = 5000; + + private GzipStandInServer server; + private GzipStandInServer otherServer; + + /** + * Starts every test without SDK files and with a new server. + */ + @Before + public void beforeTest() throws IOException { + TestUtils.createCleanTestState(); + server = new GzipStandInServer(); + } + + /** + * Stops the SDK, deletes its data and stops the servers. + */ + @After + public void afterTest() { + Countly.instance().halt(); + server.stop(); + if (otherServer != null) { + otherServer.stop(); + } + } + + /** + * Without {@link Config#enableRequestCompression()} the settings request does not ask for + * gzip, and a big request goes plain although the server says it reads gzip. + */ + @Test + public void compressionOff_settingsRequestDoesNotAskAndRequestsArePlain() throws InterruptedException { + server.answer(101, true); + Countly.instance().init(config(server)); + awaitSettings(101); + + queueRequest("big", BIG); + GzipStandInRequest big = server.await(withMarker("big")); + + Assert.assertNull(server.await(withMethod("sc")).params.get("gzip")); + Assert.assertFalse(big.compressed); + Assert.assertEquals(BIG, big.params.get("payload").length()); + Assert.assertFalse(provider().isRequestCompressionAllowed()); + } + + /** + * Once the server says it reads gzip, a body of at least 1 KB goes compressed to the endpoint + * with the marker as the only query, as a form body without a Content-Encoding header, and its + * checksum is the one of the plain body. + */ + @Test + public void compressionOn_serverReadsGzip_bigRequestIsCompressed() throws InterruptedException { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression().enableParameterTamperingProtection(SALT)); + awaitSettings(101); + Assert.assertTrue(provider().isRequestCompressionAllowed()); + + queueRequest("big", BIG); + GzipStandInRequest big = server.await(withMarker("big")); + + Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); + Assert.assertTrue(big.compressed); + Assert.assertEquals("/i", big.path); + Assert.assertTrue("the compressed body has " + big.bodyBytes + " bytes", big.bodyBytes < 1000); + Assert.assertEquals("application/x-www-form-urlencoded", big.contentType); + Assert.assertNull(big.contentEncoding); + Assert.assertEquals(repeat('a', BIG), big.params.get("payload")); + String[] signed = big.body.split("&checksum256="); + Assert.assertEquals(2, signed.length); + Assert.assertEquals(Utils.digestHex("SHA-256", signed[0] + SALT, null), signed[1]); + } + + /** + * A body under 1 KB goes plain, also as a POST, and a bigger one after it is compressed. + */ + @Test + public void compressionOn_smallRequestStaysPlain() throws InterruptedException { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression().enableForcedHTTPPost()); + awaitSettings(101); + + queueRequest("small", 300); + GzipStandInRequest small = server.await(withMarker("small")); + queueRequest("big", BIG); + GzipStandInRequest big = server.await(withMarker("big")); + + Assert.assertEquals("POST", small.method); + Assert.assertFalse(small.compressed); + Assert.assertTrue(small.bodyBytes < 1024); + Assert.assertTrue(big.compressed); + } + + /** + * A settings answer without the gzip key keeps every body plain. + */ + @Test + public void compressionOn_serverDoesNotSayGzip_requestsArePlain() throws InterruptedException { + server.answer(101, false); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("big", BIG); + GzipStandInRequest big = server.await(withMarker("big")); + + Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); + Assert.assertFalse(big.compressed); + Assert.assertFalse(provider().isRequestCompressionAllowed()); + } + + /** + * A body over 1 MB goes plain, as the server refuses a bigger compressed body, and compression + * stays on for the requests after it. + */ + @Test + public void compressionOn_bodyOverOneMegabyteStaysPlain_compressionStaysOn() throws InterruptedException { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("huge", Utils.REQUEST_COMPRESSION_MAX_BYTES); + GzipStandInRequest huge = server.await(withMarker("huge")); + queueRequest("big", BIG); + GzipStandInRequest big = server.await(withMarker("big")); + + Assert.assertFalse(huge.compressed); + Assert.assertTrue(huge.bodyBytes > Utils.REQUEST_COMPRESSION_MAX_BYTES); + Assert.assertTrue(big.compressed); + } + + /** + * A refused compressed request stays queued and goes plain on its next try, and so do the + * requests after it. + */ + @Test + public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws InterruptedException { + server.answer(101, true); + server.refuseCompressed.set(1); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("refused", BIG); + server.await(withMarker("refused")); + awaitSendHandled(); + Countly.instance().attemptToSendStoredRequests(); + Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("refused")) == 2)); + queueRequest("after", BIG); + GzipStandInRequest after = server.await(withMarker("after")); + + List refused = server.all(withMarker("refused")); + Assert.assertEquals(Arrays.asList(true, false), compressedFlags(refused)); + Assert.assertEquals(refused.get(0).params, refused.get(1).params); + Assert.assertFalse(after.compressed); + } + + /** + * After a refusal stopped compression, a settings answer without gzip leaves it stopped, and the + * next one with "gzip": true turns it on again. + */ + @Test + public void compressedRequestRefused_nextGzipAnswerTurnsCompressionOnAgain() throws InterruptedException { + server.answer(101, true); + server.refuseCompressed.set(1); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + queueRequest("refused", BIG); + server.await(withMarker("refused")); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> !provider().isRequestCompressionAllowed())); + + server.answer(102, false); + fetchSettings(); + awaitSettings(102); + Assert.assertFalse(provider().isRequestCompressionAllowed()); + + server.answer(103, true); + fetchSettings(); + awaitSettings(103); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> provider().isRequestCompressionAllowed())); + awaitSendHandled(); + Countly.instance().attemptToSendStoredRequests(); + queueRequest("after_gzip_answer", BIG); + GzipStandInRequest after = server.await(withMarker("after_gzip_answer")); + + Assert.assertTrue(after.compressed); + Assert.assertEquals(Arrays.asList(true, false), compressedFlags(server.all(withMarker("refused")))); + } + + /** + * A compressed remote config download answered with JSON leaves compression on. + */ + @Test + public void compressedImmediateRequestAnsweredWithJson_compressionStaysOn() throws InterruptedException { + Assert.assertTrue(immediateRequestKeepsCompression(200, "{\"color\":\"red\"}")); + } + + /** + * A compressed remote config download answered with a page, as a proxy in front of the server + * may send, was not accepted: compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithPage_stopsCompression() throws InterruptedException { + Assert.assertFalse(immediateRequestKeepsCompression(200, "Bad gateway")); + } + + /** + * A compressed remote config download answered with an error status was not accepted: + * compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithError_stopsCompression() throws InterruptedException { + Assert.assertFalse(immediateRequestKeepsCompression(400, "{\"result\":\"Invalid request body\"}")); + } + + /** + * A compressed remote config download answered with nothing was not accepted: compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithNothing_stopsCompression() throws InterruptedException { + Assert.assertFalse(immediateRequestKeepsCompression(200, "")); + } + + /** + * A compressed request that failed without an answer does not stop compression: the next try + * goes compressed again. + */ + @Test + public void compressedRequestWithoutAnswer_compressionStaysOn() throws InterruptedException { + server.answer(101, true); + server.dropCompressed.set(true); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("dropped", BIG); + server.await(withMarker("dropped")); + awaitSendHandled(); + server.dropCompressed.set(false); + Countly.instance().attemptToSendStoredRequests(); + Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("dropped").and(r -> r.answered.get())) == 1)); + + List dropped = server.all(withMarker("dropped")); + Assert.assertTrue(dropped.size() >= 2); + for (GzipStandInRequest received : dropped) { + Assert.assertTrue(received.compressed); + } + Assert.assertTrue(provider().isRequestCompressionAllowed()); + } + + /** + * The gzip key is read from the live answer only, never stored with the settings. + */ + @Test + public void gzipAnswer_isNotStored() throws InterruptedException { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + ModuleConfiguration configuration = SDKCore.instance.module(ModuleConfiguration.class); + synchronized (configuration) { + Assert.assertFalse(configuration.latestRetrievedConfiguration.has("gzip")); + } + String stored = SDKCore.instance.sdkStorage.getServerConfig(); + Assert.assertNotNull(stored); + Assert.assertFalse(stored.contains("gzip")); + } + + /** + * After a server URL change the requests go plain, until the new server says it reads gzip. + */ + @Test + public void serverUrlChange_newServerGetsPlainRequestsUntilItSaysGzip() throws IOException, InterruptedException { + server.answer(101, true); + otherServer = new GzipStandInServer(); + otherServer.answer(102, true); + CountDownLatch release = new CountDownLatch(1); + otherServer.holdSettings.set(release); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + Countly.instance().changeServerUrl(otherServer.url()); + otherServer.await(withMethod("sc")); + queueRequest("before_answer", BIG); + GzipStandInRequest before = otherServer.await(withMarker("before_answer")); + release.countDown(); + awaitSettings(102); + queueRequest("after_answer", BIG); + GzipStandInRequest after = otherServer.await(withMarker("after_answer")); + + Assert.assertFalse(before.compressed); + Assert.assertTrue(after.compressed); + Assert.assertEquals(0, server.count(withMarker("before_answer"))); + } + + /** + * Only an answer with "gzip": true counts, and the key is removed from it. + */ + @Test + public void extractGzipFlag_onlyTrueCounts() { + JSONObject accepted = new JSONObject("{\"v\":1,\"t\":1,\"c\":{},\"gzip\":true}"); + Assert.assertTrue(ModuleConfiguration.extractGzipFlag(accepted)); + Assert.assertFalse(accepted.has("gzip")); + + for (String answer : new String[] { "{\"gzip\":false}", "{\"gzip\":\"true\"}", "{\"gzip\":1}", "{\"c\":{\"gzip\":true}}", "{}" }) { + Assert.assertFalse(answer, gzipFlagOf(answer)); + } + Assert.assertFalse(ModuleConfiguration.extractGzipFlag(null)); + } + + /** + * Bodies under 1 KB and over 1 MB are left alone, the ones between are compressed into gzip + * that decodes to the body. + */ + @Test + public void gzipRequestBody_bounds() throws IOException { + Log log = new Log(Config.LoggingLevel.OFF, null); + Assert.assertEquals(0, Utils.gzipRequestBody(repeat('a', Utils.REQUEST_COMPRESSION_MIN_BYTES - 1), log).length); + Assert.assertEquals(0, Utils.gzipRequestBody(repeat('a', Utils.REQUEST_COMPRESSION_MAX_BYTES + 1), log).length); + for (int size : new int[] { Utils.REQUEST_COMPRESSION_MIN_BYTES, Utils.REQUEST_COMPRESSION_MAX_BYTES }) { + String body = "app_key=key&payload=" + repeat('a', size - 20); + byte[] compressed = Utils.gzipRequestBody(body, log); + Assert.assertTrue(compressed.length > 0 && compressed.length < body.length()); + Assert.assertEquals(body, gunzipText(compressed)); + } + } + + /** + * The marker fills the empty query of an endpoint, or becomes the query of a URL without one; a + * URL that already has a query cannot carry it. + */ + @Test + public void compressedRequestPath_markerIsTheOnlyQuery() { + Assert.assertEquals("http://host/i?content_encoding=gzip", Transport.compressedRequestPath("http://host/i?")); + Assert.assertEquals("http://host/o/sdk?content_encoding=gzip", Transport.compressedRequestPath("http://host/o/sdk")); + Assert.assertNull(Transport.compressedRequestPath("http://host/o/sdk?method=rc")); + } + + // region helpers + + /** + * Downloads remote config with 120 keys, a compressed POST, answered with the given status and + * text, then tells whether a big request sent after it was compressed. + */ + private boolean immediateRequestKeepsCompression(int remoteConfigCode, String remoteConfigAnswer) throws InterruptedException { + server.answer(101, true); + server.remoteConfigCode.set(remoteConfigCode); + server.remoteConfigAnswer.set(remoteConfigAnswer); + Countly.instance().init(config(server, Config.Feature.RemoteConfig).enableRequestCompression()); + awaitSettings(101); + + String[] keys = new String[120]; + for (int i = 0; i < keys.length; i++) { + keys[i] = "remote_config_key_" + i; + } + CountDownLatch downloaded = new CountDownLatch(1); + Countly.instance().remoteConfig().downloadSpecificKeys(keys, (result, error, fullValueUpdate, values) -> downloaded.countDown()); + Assert.assertTrue(downloaded.await(15, TimeUnit.SECONDS)); + queueRequest("after_download", BIG); + GzipStandInRequest after = server.await(withMarker("after_download")); + + GzipStandInRequest download = server.await(withMethod("rc")); + Assert.assertTrue(download.compressed); + Assert.assertEquals(120, new org.json.JSONArray(download.params.get("keys")).length()); + return after.compressed; + } + + /** + * A configuration that sends to the given server without request cooldowns or health check. + */ + private static Config config(GzipStandInServer server, Config.Feature... features) { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + + Config config = new Config(server.url(), TestUtils.SERVER_APP_KEY, directory); + config.setApplicationVersion(TestUtils.APPLICATION_VERSION); + config.setCustomDeviceId(TestUtils.DEVICE_ID); + config.disableHealthCheck(); + config.enableFeatures(features); + config.setNetworkRequestCooldown(0); + config.setNetworkImportantRequestCooldown(0); + return config; + } + + /** + * The SDK behavior settings in effect. + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * Waits until the SDK has read the settings answer that carries the given {@code lkl} value. + */ + private static void awaitSettings(int lkl) throws InterruptedException { + Assert.assertTrue("the settings answer with lkl " + lkl + " was not read", TestUtils.waitFor(15_000, () -> provider().getMaxKeyLength() == lkl)); + } + + /** + * Fetches the settings again, as the refresh timer does. + */ + private static void fetchSettings() { + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * Queues a request with a payload of the given length, told apart by its marker parameter. + */ + private static void queueRequest(String marker, int payloadLength) { + InternalConfig config = SDKCore.instance.config; + Request request = new Request(); + request.params.add(ModuleRequests.prepareRequiredParams(config)); + request.params.add("marker", marker); + request.params.add("payload", repeat('a', payloadLength)); + ModuleRequests.pushAsync(config, request); + } + + /** + * Waits until the SDK finished handling the request it is sending, so a send asked for next is + * not folded into it. + */ + private static void awaitSendHandled() throws InterruptedException { + Assert.assertTrue("the send in progress must end", TestUtils.waitFor(10_000, () -> !SDKCore.instance.networking.isSending())); + } + + /** + * Matches the request that carries the given marker parameter. + */ + private static Predicate withMarker(String marker) { + return r -> marker.equals(r.params.get("marker")); + } + + /** + * Matches an immediate request of the given method. + */ + private static Predicate withMethod(String method) { + return r -> method.equals(r.params.get("method")); + } + + /** + * Whether each request came compressed. + */ + private static List compressedFlags(List requests) { + return requests.stream().map(r -> r.compressed).collect(Collectors.toList()); + } + + /** + * A text of the given length made of one character. + */ + private static String repeat(char c, int length) { + char[] chars = new char[length]; + Arrays.fill(chars, c); + return new String(chars); + } + + /** + * Reads the gzip flag of a settings answer given as text. + */ + private static boolean gzipFlagOf(String answer) { + return ModuleConfiguration.extractGzipFlag(new JSONObject(answer)); + } + + /** + * Decodes gzip bytes into text. + */ + private static String gunzipText(byte[] compressed) throws IOException { + return new String(GzipStandInRequest.gunzip(compressed), StandardCharsets.UTF_8); + } + + // endregion +}