DEV Community

Yogesh Manware
Yogesh Manware

Posted on • Updated on

Compressing and Decompressing Messages (Java/NodeJs) while working with AWS Kinesis

We use AWS kinesis stream for event sourcing. AWS Kinesis has a limit of 1MB. With this restriction and combination of Java and NodeJS based microservices, we needed a way to compress and decompress the message across platforms. NodeJs and Java has inbuilt support for Compression and Decompression. Both platform use Gzip format.

Alt Observation for JSON messages

Following are the examples in Java and NodeJs. It is possible to compress in Java, decompress in NodeJs and vice versa.

Compress and decompress in Java

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStreamReader;
import java.util.Base64;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

public class GzipUtil {

    private static final String payload = "Compressing messages and decompressing messages";

    public static void main(String[] args) throws Exception {
        String encodedStr = compress(payload);
        System.out.println("Compressed String: " + encodedStr);
        String decodedStr = decompress(encodedStr);
        System.out.println("Decompressed String: " + decodedStr);
    }

    public static String decompress(String str) throws Exception {
        GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(Base64.getDecoder().decode(str)));
        BufferedReader bf = new BufferedReader(new InputStreamReader(gis, "UTF-8"));
        String outStr = "";
        String line;
        while ((line=bf.readLine())!=null) {
            outStr += line;
        }
        System.out.println("Decompressed String length : " + outStr.length());
        return outStr;
    }

    public static String compress(String str) throws Exception {
        System.out.println("Original String Length : " + str.length());
        ByteArrayOutputStream obj=new ByteArrayOutputStream();
        GZIPOutputStream gzip = new GZIPOutputStream(obj);
        gzip.write(str.getBytes("UTF-8"));
        gzip.close();
        String base64Encoded = Base64.getEncoder().encodeToString(obj.toByteArray());
        System.out.println("Compressed String length : " + base64Encoded.length());
        return base64Encoded;
    }
}
Enter fullscreen mode Exit fullscreen mode

Compress and Decompress in NodeJS

const zlib = require('zlib'); //inbuilt in NodeJs
const sizeof = require('object-sizeof'); //npm

const input = "Compressing messages and decompressing messages";

(async function () {
    // compress
    console.info(`String size: ${sizeof(input)}`);
    let buffer = await zlib.deflateSync(input);
    const compressedString = buffer.toString('base64');
    console.info(`compressed String size: ${sizeof(compressedString)}`);
    // decompress
    buffer = await zlib.unzipSync(Buffer.from(compressedString, 'base64'));
    console.info(`decoded string : ${buffer.toString()}`);
})()
Enter fullscreen mode Exit fullscreen mode

Top comments (0)