Merge branch 'release-0.6.0'

README.md

@@ -19,6 +19,61 @@ wants to share into users' inbox space using the recipients' public key so that
 - For storage systems that do not support file versioning natively (i.e. minio) this library provides versioning
 capability too.
 
+## Features
+
+-  Proprietary software **friendly license**
+-  **Flexibility** - you can easily change encryption and configure or customize other aspects of library
+-  AES encryption using **CMS-envelopes** for increased security and interoperability with other languages
+-  **Extra protection layer** - encryption using securely generated keys that are completely unrelated to your password
+-  **Client side encryption** - you own your data
+-  Works with filesystem and Amazon S3 compatible storage - S3, minio, CEPH, etc.
+-  File names are encrypted
+-  Thorough testing
+
+## Building project
+Without tests:
+```bash
+mvn clean install -DskipTests=true
+```
+Full build:
+```bash
+mvn clean install
+```
+
+## Quick demo
+
+[Here](datasafe-rest-impl/DEMO.md) you can find quick demo of project capabilities with instructions how to use it.
+
+## Adding to your project
+
+Datasafe is available from maven-central repository, you can add it to your project using:
+```xml
+<dependency>
+    <groupId>de.adorsys</groupId>
+    <artifactId>datasafe-business</artifactId>
+    <version>0.5.0</version>
+</dependency>
+``` 
+
+To add filesystem storage provider:
+```xml
+<dependency>
+    <groupId>de.adorsys</groupId>
+    <artifactId>datasafe-storage-impl-fs</artifactId>
+    <version>0.5.0</version>
+</dependency>
+``` 
+
+To add S3 storage provider:
+```xml
+<dependency>
+    <groupId>de.adorsys</groupId>
+    <artifactId>datasafe-storage-impl-s3</artifactId>
+    <version>0.5.0</version>
+</dependency>
+``` 
+
+
 # Project overview
 In short, Datasafe [core logic](datasafe-business/src/main/java/de/adorsys/datasafe/business/impl/service/DefaultDatasafeServices.java)
 provides these key services:

datasafe-business/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datasafe</artifactId>
         <groupId>de.adorsys</groupId>
-        <version>0.5</version>
+        <version>0.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/BasicFunctionalityTest.java

@@ -17,8 +17,10 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import org.testcontainers.shaded.com.google.common.collect.ImmutableSet;
 
+import java.io.ByteArrayInputStream;
 import java.io.OutputStream;
 import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -31,6 +33,8 @@
 @Slf4j
 class BasicFunctionalityTest extends BaseE2ETest {
 
+    private static final int LARGE_SIZE = 10 * 1024 * 1024 + 100;
+
     private StorageService storage;
     private Uri location;
 
@@ -87,6 +91,34 @@ void testMultipleRecipientsSharing(WithStorageProvider.StorageDescriptor descrip
         assertThat(readFromInbox.read(ReadRequest.forDefaultPrivate(jamie, multiShareFile))).hasContent(MESSAGE_ONE);
     }
 
+    @SneakyThrows
+    @ParameterizedTest
+    @MethodSource("allStorages")
+    void testMultipleRecipientsSharingLargeChunk(WithStorageProvider.StorageDescriptor descriptor) {
+        init(descriptor);
+
+        UserIDAuth john = registerUser("john");
+        UserIDAuth jane = registerUser("jane");
+        UserIDAuth jamie = registerUser("jamie");
+
+        String multiShareFile = "multishare.txt";
+        byte[] bytes = new byte[LARGE_SIZE];
+        ThreadLocalRandom.current().nextBytes(bytes);
+        try (OutputStream os = writeToInbox.write(WriteRequest.forDefaultPublic(
+                ImmutableSet.of(john.getUserID(), jane.getUserID(), jamie.getUserID()),
+                multiShareFile))
+        ) {
+            os.write(bytes);
+        }
+
+        assertThat(readFromInbox.read(ReadRequest.forDefaultPrivate(john, multiShareFile)))
+                .hasSameContentAs(new ByteArrayInputStream(bytes));
+        assertThat(readFromInbox.read(ReadRequest.forDefaultPrivate(jane, multiShareFile)))
+                .hasSameContentAs(new ByteArrayInputStream(bytes));
+        assertThat(readFromInbox.read(ReadRequest.forDefaultPrivate(jamie, multiShareFile)))
+                .hasSameContentAs(new ByteArrayInputStream(bytes));
+    }
+
     @ParameterizedTest
     @MethodSource("allStorages")
     void testWriteToPrivateListPrivateReadPrivateAndSendToAndReadFromInbox(

datasafe-business/src/test/java/de/adorsys/datasafe/business/impl/e2e/MultiDFSFunctionalityTest.java

@@ -19,18 +19,15 @@
 import de.adorsys.datasafe.storage.api.RegexDelegatingStorage;
 import de.adorsys.datasafe.storage.api.StorageService;
 import de.adorsys.datasafe.storage.api.UriBasedAuthStorageService;
+import de.adorsys.datasafe.types.api.utils.ExecutorServiceUtil;
 import de.adorsys.datasafe.storage.impl.s3.S3ClientFactory;
 import de.adorsys.datasafe.storage.impl.s3.S3StorageService;
 import de.adorsys.datasafe.types.api.actions.ListRequest;
 import de.adorsys.datasafe.types.api.actions.ReadRequest;
 import de.adorsys.datasafe.types.api.actions.WriteRequest;
 import de.adorsys.datasafe.types.api.context.BaseOverridesRegistry;
 import de.adorsys.datasafe.types.api.context.overrides.OverridesRegistry;
-import de.adorsys.datasafe.types.api.resource.AbsoluteLocation;
-import de.adorsys.datasafe.types.api.resource.BasePrivateResource;
-import de.adorsys.datasafe.types.api.resource.BasePublicResource;
-import de.adorsys.datasafe.types.api.resource.ResolvedResource;
-import de.adorsys.datasafe.types.api.resource.StorageIdentifier;
+import de.adorsys.datasafe.types.api.resource.*;
 import de.adorsys.datasafe.types.api.shared.BaseMockitoTest;
 import lombok.SneakyThrows;
 import lombok.experimental.Delegate;
@@ -52,7 +49,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -76,7 +72,8 @@ class MultiDFSFunctionalityTest extends BaseMockitoTest {
     private static final String FILES_TWO = "filestwobucket";
     private static final String INBOX = "inboxbucket";
 
-    private static final ExecutorService EXECUTOR = Executors.newFixedThreadPool(5);
+    private static final ExecutorService EXECUTOR = ExecutorServiceUtil
+            .submitterExecutesOnStarvationExecutingService(5, 5);
 
     private static Map<String, GenericContainer> minios = new HashMap<>();
     private static Map<String, String> endpointsByHost = new HashMap<>();

datasafe-directory/datasafe-directory-api/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>de.adorsys</groupId>
     <artifactId>datasafe-directory</artifactId>
-    <version>0.5</version>
+    <version>0.6.0</version>
   </parent>
   <artifactId>datasafe-directory-api</artifactId>
 	<dependencies>

datasafe-directory/datasafe-directory-impl/pom.xml

@@ -3,7 +3,7 @@
   <parent>
     <groupId>de.adorsys</groupId>
     <artifactId>datasafe-directory</artifactId>
-    <version>0.5</version>
+    <version>0.6.0</version>
   </parent>
   <artifactId>datasafe-directory-impl</artifactId>
 	<dependencies>

datasafe-directory/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datasafe</artifactId>
         <groupId>de.adorsys</groupId>
-        <version>0.5</version>
+        <version>0.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

datasafe-encryption/datasafe-encryption-api/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>de.adorsys</groupId>
 		<artifactId>datasafe-encryption</artifactId>
-		<version>0.5</version>
+		<version>0.6.0</version>
 	</parent>
 	<artifactId>datasafe-encryption-api</artifactId>
 	<dependencies>

datasafe-encryption/datasafe-encryption-impl/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>de.adorsys</groupId>
 		<artifactId>datasafe-encryption</artifactId>
-		<version>0.5</version>
+		<version>0.6.0</version>
 	</parent>
 	<artifactId>datasafe-encryption-impl</artifactId>
 	<dependencies>

datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/CMSEncryptionServiceImpl.java

@@ -23,7 +23,6 @@
 import java.io.OutputStream;
 import java.security.Key;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
@@ -39,9 +38,6 @@
 @RuntimeDelegate
 public class CMSEncryptionServiceImpl implements CMSEncryptionService {
 
-    private final Map<Integer, Decryptor> decryptors = new HashMap<>();
-
-
     private CMSEncryptionConfig encryptionConfig;
 
     @Inject

datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/cmsencryption/DefaultCMSEncryptionConfig.java

@@ -15,6 +15,6 @@ public class DefaultCMSEncryptionConfig implements CMSEncryptionConfig {
 
     @Inject
     public DefaultCMSEncryptionConfig() {
-        algorithm = DatasafeCryptoAlgorithm.AES256_CBC;
+        algorithm = DatasafeCryptoAlgorithm.AES256_GCM;
     }
 }

datasafe-encryption/datasafe-encryption-impl/src/main/java/de/adorsys/datasafe/encrypiton/impl/document/CMSDocumentWriteService.java

@@ -11,14 +11,17 @@
 import de.adorsys.datasafe.types.api.resource.AbsoluteLocation;
 import de.adorsys.datasafe.types.api.resource.PrivateResource;
 import de.adorsys.datasafe.types.api.resource.WithCallback;
+import de.adorsys.datasafe.types.api.utils.CustomizableByteArrayOutputStream;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 
 import javax.inject.Inject;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
@@ -40,11 +43,19 @@ public CMSDocumentWriteService(StorageWriteService writeService,
     @Override
     public OutputStream write(Map<PublicKeyIDWithPublicKey, AbsoluteLocation> recipientsWithInbox) {
 
-        FanOutStream dfsSink = new FanOutStream(
-                recipientsWithInbox.values().stream()
-                        .map(it -> writeService.write(WithCallback.noCallback(it)))
-                        .collect(Collectors.toList())
-        );
+        int maxChunkSize = recipientsWithInbox.values().stream()
+                .map(writeService::flushChunkSize)
+                .filter(Optional::isPresent)
+                .mapToInt(Optional::get)
+                .max()
+                .orElse(-1);
+
+        List<OutputStream> recipients = recipientsWithInbox.values().stream()
+                .map(it -> writeService.write(WithCallback.noCallback(it)))
+                .collect(Collectors.toList());
+
+        FanOutStream dfsSink = maxChunkSize > 0 ?
+                new ChunkableFanOutStream(recipients, maxChunkSize) : new FanOutStream(recipients);
 
         OutputStream encryptionSink = cms.buildEncryptionOutputStream(
                 dfsSink,
@@ -115,9 +126,9 @@ private static void doClose(OutputStream stream) {
      * byte to multiple recipients.
      */
     @RequiredArgsConstructor
-    private static final class FanOutStream extends OutputStream {
+    private static class FanOutStream extends OutputStream {
 
-        private final List<OutputStream> destinations;
+        protected final List<OutputStream> destinations;
 
         @Override
         public void write(int b) throws IOException {
@@ -137,9 +148,96 @@ public void write(byte[] bytes, int off, int len) throws IOException {
         @SneakyThrows
         public void close() {
             super.close();
-            for (OutputStream destination : destinations) {
+            Iterator<OutputStream> dest = destinations.iterator();
+            while (dest.hasNext()) {
+                dest.next().close();
+                dest.remove();
+            }
+        }
+    }
+
+    /**
+     * Buffered fan-out stream, so that same data won't get replicated multiple times for chunked consumers.
+     * Such consumers retain buffer that is equal to chunk size, in order to eliminate this extra buffer
+     * this class can be used (assuming all-equal chunk size).
+     */
+    private static class ChunkableFanOutStream extends FanOutStream {
+
+        private final int chunkSize;
+        private final CustomizableByteArrayOutputStream os;
+
+        private ChunkableFanOutStream(List<OutputStream> destinations, int chunkSize) {
+            super(destinations);
+
+            this.chunkSize = chunkSize;
+            this.os = new CustomizableByteArrayOutputStream(32, Integer.MAX_VALUE - 1, 0.5);
+        }
+
+        @Override
+        public void write(int b) throws IOException {
+            if (!needsFlush(1)) {
+                os.write(b);
+                return;
+            }
+
+            os.write(b);
+            doFlush();
+        }
+
+        @Override
+        public void write(byte[] bytes, int off, int len) throws IOException {
+            if (!needsFlush(len)) {
+                os.write(bytes, off, len);
+                return;
+            }
+
+            os.write(bytes, off, len);
+            doFlush();
+        }
+
+        @Override
+        @SneakyThrows
+        public void close() {
+            if (os.size() == 0) {
+                super.close();
+                return;
+            }
+
+            // when closing stream immediately it is ok not to write in chunks - memory will
+            // be retained only for 1 destination
+            byte[] tailChunk = os.getBufferOrCopy();
+            int size = os.size();
+
+            Iterator<OutputStream> dest = destinations.iterator();
+            while (dest.hasNext()) {
+                OutputStream destination = dest.next();
+                destination.write(tailChunk, 0, size);
                 destination.close();
+                dest.remove();
+            }
+        }
+
+        private void doFlush() throws IOException {
+            byte[] bytes = os.getBufferOrCopy();
+            int size = os.size();
+
+            // write only in chunks of declared size
+            int chunksToWrite = size / chunkSize;
+            int written = 0;
+            for (int chunkNum = 0; chunkNum < chunksToWrite; chunkNum++) {
+                super.write(bytes, written, chunkSize);
+                written += chunkSize;
+            }
+
+            // retain tail bytes, non proportional to `chunkSize`:
+            os.reset();
+            if (written < size) {
+                os.write(bytes, written, size - written);
             }
         }
+
+        private boolean needsFlush(int addedBytes) {
+            return os.size() + addedBytes > chunkSize;
+        }
     }
 }

datasafe-encryption/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datasafe</artifactId>
         <groupId>de.adorsys</groupId>
-        <version>0.5</version>
+        <version>0.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>