Merge branch 'Release_2.1'

DEPENDENCIES

@@ -2,4 +2,4 @@
 ## Prefer git tags instead of SHA hashes when possible.
 
 libpEpAdapter=Release_2.1.21
-pEpEngine=Release_2.1.41
+pEpEngine=Release_2.1.42

src/codegen/gen_java_Message.ysl2

@@ -33,16 +33,19 @@ tstylesheet {
             import java.util.Date;
             import java.util.HashMap;
             import java.io.Serializable;
+            import java.util.concurrent.atomic.AtomicLong;
 
             public class «$cname» implements MessageInterface, AutoCloseable, Serializable {
                 private static final long serialVersionUID = 2119420428331150924L;
                 private long handle;
+                private static AtomicLong instanceCount = new AtomicLong(0);
 
                 native long init();
                 native void release(long handle);
 
                 public «$cname»() {
                     handle = init();
+                    instanceCount.getAndIncrement();
                 }
 
                 private native long _«$cname»(
@@ -52,6 +55,7 @@ tstylesheet {
                 public «$cname»(String mime_text) {
                     byte[] _mime_text = Utils.toUTF8(mime_text);
                     handle = _«$cname»(_mime_text);
+                    instanceCount.getAndIncrement();
                 }
 
                 private native byte[] _encodeMIME() throws pEpException;
@@ -62,11 +66,19 @@ tstylesheet {
 
                 private «$cname»(long h) {
                     handle = h;
+                    instanceCount.getAndIncrement();
                 }
 
-                public final void close() {
+                public synchronized final void close() {
+                    if(handle != 0) {
                         release(handle);
                         handle = 0;
+                        instanceCount.getAndDecrement();
+                    }
+                }
+
+                public static synchronized AtomicLong getInstanceCount() {
+                    return instanceCount;
                 }
 
                 final protected long getHandle() {

test/java/foundation/pEp/jniadapter/test/Makefile.conf

@@ -21,7 +21,7 @@ LD_LIB_PATH=.:$(DIST_DIR)
 # Java cmd-line options
 JAVA_OPT_ASSERTIONS=-enableassertions
 JAVA_OPT_XCHECK=-Xcheck:jni
-; JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
+# JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
 JAVA_OPT_CLASSPATH=-cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH)
 JAVA_OPT_LIBARY_PATH=-Djava.library.path=$(LD_LIB_PATH)
 

test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java

@@ -1,78 +1,156 @@
 package foundation.pEp.jniadapter.test.jni135;
 
-import foundation.pEp.jniadapter.Blob;
+import foundation.pEp.jniadapter.*;
-import foundation.pEp.jniadapter.Engine;
-import foundation.pEp.jniadapter.Message;
-import foundation.pEp.jniadapter.decrypt_message_Return;
-import foundation.pEp.jniadapter.test.utils.CTXBase;
 import foundation.pEp.jniadapter.test.utils.AdapterTestUtils;
+import foundation.pEp.jniadapter.test.utils.model.Role;
+import foundation.pEp.jniadapter.test.utils.model.TestModel;
+import foundation.pEp.jniadapter.test.utils.model.TestNode;
+import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity;
+import foundation.pEp.pitytest.AbstractTestContext;
 import foundation.pEp.pitytest.TestSuite;
 import foundation.pEp.pitytest.TestUnit;
 import foundation.pEp.pitytest.utils.TestUtils;
 
-import java.lang.ref.WeakReference;
 import java.util.Vector;
 
 import static foundation.pEp.pitytest.TestLogger.log;
 
+/*
+Test for JNI-135 - Heavy memory consumption - memory leaks
+relates to:
+* JNI-148 - Mem-mgmt: Defined behaviour of Message.close()
+* JNI-160 - Mem-mgmt: Resource Instrumentation for class Message
 
-class Jni135TestContext extends CTXBase {
+We are simply encrypting and decrypting in a cycles with 2 own identities alice and bob.
-    @Override
+The idea is that you fire up your mem-monitoring tool of choice and convince yourself about the
-    public CTXBase init() throws Throwable {
+reality of the amount leaked memory in relation to the number of cycles.
-        super.init();
-        return this;
-    }
-}
 
-class TestAlice {
+Since JNI-160, this test also prints the result of Message.getInstanceCount(). Please see ticket for more details.
-    public static void gc() {
-//        log("gc start");
-        Object obj = new Object();
-        WeakReference ref = new WeakReference<Object>(obj);
-        obj = null;
-        while (ref.get() != null) {
-            System.gc();
-        }
-//        log("gc end");
-    }
 
-    public static void main(String[] args) throws Exception {
+This test suite proofs 3 things:
-//        TestUtils.readKey();
+* If you ignore mem-mgmt, you WILL leak memory
-        TestSuite.getDefault().setVerbose(true);
+* If you do mem-mgmt right, you DONT leak memory
-        TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN);
+* If you lost all references to an unreleased message obj, it will be unreleasable forever.
-        CTXBase jni135Ctx = new Jni135TestContext();
+
+To experiment with this, you can run a single test only and make use of the options:
+* repeatCount - how many iterations/cycles
+* msgSizeMB - attachement size of one message
+* EncFormat - the encryption format
+
+ To run a single test only, comment out the last line:
+"TestSuite.getDefault().run();"
+
+and run the TestUnit directly using TestUnit.run():
+
+new TestUnit<ctxtype>("bla", new ctxType(), ctx -> {
+    // code
+}).run();
+
+ */
+
+
+// FOR TEST CONFIG DO NOT TWEAK HERE, THESE ARE DEFAULTS
+class Jni135TestContext extends AbstractTestContext {
+    // Model
+    public TestModel<pEpTestIdentity, TestNode<pEpTestIdentity>> model = new TestModel(pEpTestIdentity::new, TestNode::new);
+
+    // Basic
+    public Engine engine;
+
+    // Identities
+    public Identity alice;
+    public Identity bob;
+
+    // Test config defaults
+    public int repeatCount = 2000;
+    public int msgSizeMB = 1;
+    public Message.EncFormat encFormat = Message.EncFormat.PEPEncInlineEA;
 
-        Engine.setDebugLogEnabled(false);
+    public Jni135TestContext init() throws Throwable {
+        engine = new Engine();
+        // fetch data for the idents
+        alice = model.getIdent(Role.ALICE).pEpIdent;
+        bob = model.getIdent(Role.BOB).pEpIdent;
 
-        new TestUnit<CTXBase>("setDir() == getDir() ", new Jni135TestContext(), ctx -> {
+        // create keys etc..
-            ctx.alice = ctx.engine.myself(ctx.alice);
+        alice = engine.myself(alice);
-            ctx.bob = ctx.engine.myself(ctx.bob);
+        bob = engine.myself(bob);
 
+        return this;
+    }
+}
+
+class TestAlice {
+    public static void proofLeakNonLeak(Jni135TestContext ctx, boolean wannaLeak) {
         int cycles = 0;
-            while (true) {
+        while (cycles < ctx.repeatCount) {
             Message msg1Plain = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
-                Blob bigBlob = AdapterTestUtils.makeNewTestBlob(10000000, "atti1", "text/plain");
+            Blob bigBlob = AdapterTestUtils.makeNewTestBlob(ctx.msgSizeMB * 1024 * 1024, "atti1", "text/plain");
             Vector<Blob> atts = new Vector<Blob>();
             atts.add(bigBlob);
             msg1Plain.setAttachments(atts);
 
-                if (false) {
+            Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, ctx.encFormat);
-                    Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, Message.EncFormat.PEP);
             decrypt_message_Return decRet = ctx.engine.decrypt_message(msg1Enc, null, 0);
-
             assert decRet != null : "could not decrypt message";
-//                    decRet.dst.close();
+
-//                    msg1Enc.close();
+            if (!wannaLeak) {
-//                    gc();
+                decRet.dst.close();
-//                    msg1Plain.close();
+                msg1Enc.close();
+                msg1Plain.close();
+                log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount());
+                assert Message.getInstanceCount().get() == 0 : "Leaking messages";
+            } else {
+                log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount());
+                assert Message.getInstanceCount().get() > 0 : "We should be leaking messages, actually";
+            }
+        }
     }
-                log("cycles: " + cycles++);
+
+    public static void main(String[] args) throws Exception {
+        TestSuite.getDefault().setVerbose(true);
+        TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN);
+
+        {
+            Jni135TestContext ctxDontLeak = new Jni135TestContext();
+            // Config
+            ctxDontLeak.repeatCount = 3;
+            ctxDontLeak.msgSizeMB = 1;
+
+            new TestUnit<Jni135TestContext>("Proof leaking messages ", ctxDontLeak, ctx -> {
+                proofLeakNonLeak(ctx, false);
+            });
         }
 
-        }).run();
+        {
+            Jni135TestContext ctxDoLeak = new Jni135TestContext();
+            // Config
+            ctxDoLeak.repeatCount = 3;
+            ctxDoLeak.msgSizeMB = 1;
 
+            new TestUnit<Jni135TestContext>("Proof NOT leaking messages ", ctxDoLeak, ctx -> {
+                proofLeakNonLeak(ctx, true);
+            });
+        }
 
-//        TestSuite.getDefault().run();
+        {
+            Jni135TestContext ctxLostRefs = new Jni135TestContext();
+            // Config
+            ctxLostRefs.repeatCount = 3;
+            ctxLostRefs.msgSizeMB = 1;
+
+            new TestUnit<Jni135TestContext>("Lost refs cant be recovered", ctxLostRefs, ctx -> {
+                try {
+                    proofLeakNonLeak(ctx, false);
+                    assert false : "Lost references to message objects should never be recoverable";
+                } catch (Throwable t) {
+                }
+// To run one individual test only
+            })//.run(); // comment this in
+            ;
+        }
+// AND
+        TestSuite.getDefault().run(); // comment this out
     }
 }
 

test/java/foundation/pEp/jniadapter/test/jni161/Makefile

@@ -0,0 +1,34 @@
+	include ../../../../../../../Makefile.conf
+include ../Makefile.conf
+
+TEST_UNIT_NAME=jni161
+
+JAVA_CLASSES+= \
+ 	TestAlice.class
+
+.PHONY: pitytest compile alice test clean
+
+all: alice compile
+
+pitytest:
+	$(MAKE) -C $(PITYTEST_DIR)
+
+alice: compile clean-pep-home-alice
+	cd $(JAVA_CWD);pwd;HOME=$(JAVA_PEP_HOME_DIR_ALICE) $(JAVA) $(JAVA_PKG_BASENAME).$(TEST_UNIT_NAME).TestAlice
+
+compile: $(JAVA_CLASSES) pitytest
+
+%.class: %.java
+	cd $(JAVA_CWD);javac -cp $(CLASSPATH) $(JAVA_PKG_BASEPATH)/$(TEST_UNIT_NAME)/$<
+
+clean:
+	rm -f $(JAVA_CLASSES)
+	rm -f *.class
+	rm -f *.log
+	rm -Rf .gnupg
+	rm -Rf .lldb
+
+clean-pep-home: clean-pep-home-alice
+
+clean-pep-home-alice:
+	rm -rf $(PEP_HOME_DIR_ALICE)/.pEp

test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java

@@ -0,0 +1,132 @@
+package foundation.pEp.jniadapter.test.jni161;
+
+import foundation.pEp.jniadapter.Engine;
+import foundation.pEp.jniadapter.Identity;
+import foundation.pEp.jniadapter.Message;
+import foundation.pEp.jniadapter.test.utils.AdapterTestUtils;
+import foundation.pEp.jniadapter.test.utils.model.Role;
+import foundation.pEp.jniadapter.test.utils.model.TestModel;
+import foundation.pEp.jniadapter.test.utils.model.TestNode;
+import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity;
+import foundation.pEp.pitytest.AbstractTestContext;
+import foundation.pEp.pitytest.TestSuite;
+import foundation.pEp.pitytest.TestUnit;
+import foundation.pEp.pitytest.utils.TestUtils;
+
+import java.util.Vector;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import static foundation.pEp.pitytest.TestLogger.log;
+
+
+class Jni161TestContext extends AbstractTestContext {
+    // Model
+    public TestModel<pEpTestIdentity, TestNode<pEpTestIdentity>> model = new TestModel(pEpTestIdentity::new, TestNode::new);
+
+    // Basic
+    public Engine engine;
+
+    // Identities
+    public Identity alice;
+    public Identity bob;
+
+    public int blobSizeMB = 1;
+    public int blobCount = 3;
+
+    public Vector<Message> messages = new Vector<>();
+
+    public Jni161TestContext init() throws Throwable {
+        engine = new Engine();
+        // fetch data for the idents
+        alice = model.getIdent(Role.ALICE).pEpIdent;
+        bob = model.getIdent(Role.BOB).pEpIdent;
+
+        // create keys etc..
+        alice = engine.myself(alice);
+        bob = engine.myself(bob);
+
+        return this;
+    }
+}
+
+class TestAlice {
+    public static void main(String[] args) throws Exception {
+        TestSuite.getDefault().setVerbose(true);
+        TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN);
+
+        Jni161TestContext ctx161 = new Jni161TestContext();
+
+        new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
+            while (true) {
+                log("Thread: " + Thread.currentThread().getId() + " - Alloc");
+                Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
+                msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
+                msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
+//                msg.close();
+                TestUtils.sleep(10);
+            }
+        });
+
+        new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
+            new Thread(() -> {
+                while (true) {
+                    log("Thread: " + Thread.currentThread().getId() + " - Alloc");
+                    Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
+                    msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
+                    msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
+                    ctx.messages.add(msg);
+                }
+            }).start();
+
+            new Thread(() -> {
+                while (true) {
+//                    log("Thread: " + Thread.currentThread().getId() + " - Dealloc");
+                    if(ctx.messages.size() > 0) {
+                        log("Messages: " + ctx.messages.size());
+                        log("InstanceCount: " + Message.getInstanceCount());
+                        ctx.messages.remove(0).close();
+                    }
+                }
+            }).start();
+        }).run();
+
+
+        new TestUnit<Jni161TestContext>("Proof leaking messages ", ctx161, ctx -> {
+            ExecutorService allocPool = Executors.newFixedThreadPool(10);
+            ExecutorService deallocPool = Executors.newFixedThreadPool(10);
+            while (true) {
+                ThreadPoolExecutor allocExec = (ThreadPoolExecutor) allocPool;
+                if (allocExec.getActiveCount() < allocExec.getMaximumPoolSize()) {
+                    allocPool.submit(() -> {
+                        log("Thread: " + Thread.currentThread().getId() + " - Alloc");
+                        Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing);
+                        msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount));
+                        msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024));
+                        ctx.messages.add(msg);
+                        log("MessagesADD: " + ctx.messages.size());
+                    });
+                }
+
+                if (ctx.messages.size() > 1) {
+                    ThreadPoolExecutor deallocExec = (ThreadPoolExecutor) deallocPool;
+                    if (deallocExec.getActiveCount() < deallocExec.getMaximumPoolSize()) {
+                        deallocPool.submit(() -> {
+                            log("Thread: " + Thread.currentThread().getId() + " - Dealloc");
+                            log("Messages: " + ctx.messages.size());
+                            log("InstanceCount: " + Message.getInstanceCount());
+                            for (Message msg : ctx.messages) {
+                                msg.close();
+                            }
+                        });
+                    }
+                }
+                TestUtils.sleep(10);
+            }
+        });
+//        TestSuite.getDefault().run(); // comment this out
+    }
+}
+
+