diff --git a/DEPENDENCIES b/DEPENDENCIES index a585eeb..20e502e 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -2,4 +2,4 @@ ## Prefer git tags instead of SHA hashes when possible. libpEpAdapter=Release_2.1.21 -pEpEngine=Release_2.1.41 \ No newline at end of file +pEpEngine=Release_2.1.42 \ No newline at end of file diff --git a/src/codegen/gen_java_Message.ysl2 b/src/codegen/gen_java_Message.ysl2 index bea2c4a..8b13da8 100644 --- a/src/codegen/gen_java_Message.ysl2 +++ b/src/codegen/gen_java_Message.ysl2 @@ -33,16 +33,19 @@ tstylesheet { import java.util.Date; import java.util.HashMap; import java.io.Serializable; + import java.util.concurrent.atomic.AtomicLong; public class «$cname» implements MessageInterface, AutoCloseable, Serializable { private static final long serialVersionUID = 2119420428331150924L; private long handle; + private static AtomicLong instanceCount = new AtomicLong(0); native long init(); native void release(long handle); public «$cname»() { handle = init(); + instanceCount.getAndIncrement(); } private native long _«$cname»( @@ -52,6 +55,7 @@ tstylesheet { public «$cname»(String mime_text) { byte[] _mime_text = Utils.toUTF8(mime_text); handle = _«$cname»(_mime_text); + instanceCount.getAndIncrement(); } private native byte[] _encodeMIME() throws pEpException; @@ -62,11 +66,19 @@ tstylesheet { private «$cname»(long h) { handle = h; + instanceCount.getAndIncrement(); } - public final void close() { - release(handle); - handle = 0; + public synchronized final void close() { + if(handle != 0) { + release(handle); + handle = 0; + instanceCount.getAndDecrement(); + } + } + + public static synchronized AtomicLong getInstanceCount() { + return instanceCount; } final protected long getHandle() { diff --git a/test/java/foundation/pEp/jniadapter/test/Makefile.conf b/test/java/foundation/pEp/jniadapter/test/Makefile.conf index da68255..4fadee3 100644 --- a/test/java/foundation/pEp/jniadapter/test/Makefile.conf +++ b/test/java/foundation/pEp/jniadapter/test/Makefile.conf @@ -21,7 +21,7 @@ LD_LIB_PATH=.:$(DIST_DIR) # Java cmd-line options JAVA_OPT_ASSERTIONS=-enableassertions JAVA_OPT_XCHECK=-Xcheck:jni -; JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH) +# JAVA_OPT_REMOTE_DEBUGGER=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=localhost:5005 -cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH) JAVA_OPT_CLASSPATH=-cp $(CLASSPATH) -Djava.library.path=$(LD_LIB_PATH) JAVA_OPT_LIBARY_PATH=-Djava.library.path=$(LD_LIB_PATH) diff --git a/test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java b/test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java index 8a367d6..9e1d9a5 100644 --- a/test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java +++ b/test/java/foundation/pEp/jniadapter/test/jni135/TestAlice.java @@ -1,78 +1,156 @@ package foundation.pEp.jniadapter.test.jni135; -import foundation.pEp.jniadapter.Blob; -import foundation.pEp.jniadapter.Engine; -import foundation.pEp.jniadapter.Message; -import foundation.pEp.jniadapter.decrypt_message_Return; -import foundation.pEp.jniadapter.test.utils.CTXBase; +import foundation.pEp.jniadapter.*; import foundation.pEp.jniadapter.test.utils.AdapterTestUtils; +import foundation.pEp.jniadapter.test.utils.model.Role; +import foundation.pEp.jniadapter.test.utils.model.TestModel; +import foundation.pEp.jniadapter.test.utils.model.TestNode; +import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity; +import foundation.pEp.pitytest.AbstractTestContext; import foundation.pEp.pitytest.TestSuite; import foundation.pEp.pitytest.TestUnit; import foundation.pEp.pitytest.utils.TestUtils; -import java.lang.ref.WeakReference; import java.util.Vector; import static foundation.pEp.pitytest.TestLogger.log; +/* +Test for JNI-135 - Heavy memory consumption - memory leaks +relates to: +* JNI-148 - Mem-mgmt: Defined behaviour of Message.close() +* JNI-160 - Mem-mgmt: Resource Instrumentation for class Message + +We are simply encrypting and decrypting in a cycles with 2 own identities alice and bob. +The idea is that you fire up your mem-monitoring tool of choice and convince yourself about the +reality of the amount leaked memory in relation to the number of cycles. + +Since JNI-160, this test also prints the result of Message.getInstanceCount(). Please see ticket for more details. + +This test suite proofs 3 things: +* If you ignore mem-mgmt, you WILL leak memory +* If you do mem-mgmt right, you DONT leak memory +* If you lost all references to an unreleased message obj, it will be unreleasable forever. + +To experiment with this, you can run a single test only and make use of the options: +* repeatCount - how many iterations/cycles +* msgSizeMB - attachement size of one message +* EncFormat - the encryption format + + To run a single test only, comment out the last line: +"TestSuite.getDefault().run();" + +and run the TestUnit directly using TestUnit.run(): + +new TestUnit("bla", new ctxType(), ctx -> { + // code +}).run(); + + */ + + +// FOR TEST CONFIG DO NOT TWEAK HERE, THESE ARE DEFAULTS +class Jni135TestContext extends AbstractTestContext { + // Model + public TestModel> model = new TestModel(pEpTestIdentity::new, TestNode::new); + + // Basic + public Engine engine; + + // Identities + public Identity alice; + public Identity bob; + + // Test config defaults + public int repeatCount = 2000; + public int msgSizeMB = 1; + public Message.EncFormat encFormat = Message.EncFormat.PEPEncInlineEA; + + public Jni135TestContext init() throws Throwable { + engine = new Engine(); + // fetch data for the idents + alice = model.getIdent(Role.ALICE).pEpIdent; + bob = model.getIdent(Role.BOB).pEpIdent; + + // create keys etc.. + alice = engine.myself(alice); + bob = engine.myself(bob); -class Jni135TestContext extends CTXBase { - @Override - public CTXBase init() throws Throwable { - super.init(); return this; } } class TestAlice { - public static void gc() { -// log("gc start"); - Object obj = new Object(); - WeakReference ref = new WeakReference(obj); - obj = null; - while (ref.get() != null) { - System.gc(); + public static void proofLeakNonLeak(Jni135TestContext ctx, boolean wannaLeak) { + int cycles = 0; + while (cycles < ctx.repeatCount) { + Message msg1Plain = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing); + Blob bigBlob = AdapterTestUtils.makeNewTestBlob(ctx.msgSizeMB * 1024 * 1024, "atti1", "text/plain"); + Vector atts = new Vector(); + atts.add(bigBlob); + msg1Plain.setAttachments(atts); + + Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, ctx.encFormat); + decrypt_message_Return decRet = ctx.engine.decrypt_message(msg1Enc, null, 0); + assert decRet != null : "could not decrypt message"; + + if (!wannaLeak) { + decRet.dst.close(); + msg1Enc.close(); + msg1Plain.close(); + log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount()); + assert Message.getInstanceCount().get() == 0 : "Leaking messages"; + } else { + log("cycle nr: " + cycles++ + " / Message.getInstanceCount(): " + Message.getInstanceCount()); + assert Message.getInstanceCount().get() > 0 : "We should be leaking messages, actually"; + } } -// log("gc end"); } public static void main(String[] args) throws Exception { -// TestUtils.readKey(); TestSuite.getDefault().setVerbose(true); TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN); - CTXBase jni135Ctx = new Jni135TestContext(); - - Engine.setDebugLogEnabled(false); - - new TestUnit("setDir() == getDir() ", new Jni135TestContext(), ctx -> { - ctx.alice = ctx.engine.myself(ctx.alice); - ctx.bob = ctx.engine.myself(ctx.bob); - - int cycles = 0; - while (true) { - Message msg1Plain = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing); - Blob bigBlob = AdapterTestUtils.makeNewTestBlob(10000000, "atti1", "text/plain"); - Vector atts = new Vector(); - atts.add(bigBlob); - msg1Plain.setAttachments(atts); - - if (false) { - Message msg1Enc = ctx.engine.encrypt_message(msg1Plain, null, Message.EncFormat.PEP); - decrypt_message_Return decRet = ctx.engine.decrypt_message(msg1Enc, null, 0); - - assert decRet != null : "could not decrypt message"; -// decRet.dst.close(); -// msg1Enc.close(); -// gc(); -// msg1Plain.close(); - } - log("cycles: " + cycles++); - } - }).run(); + { + Jni135TestContext ctxDontLeak = new Jni135TestContext(); + // Config + ctxDontLeak.repeatCount = 3; + ctxDontLeak.msgSizeMB = 1; + new TestUnit("Proof leaking messages ", ctxDontLeak, ctx -> { + proofLeakNonLeak(ctx, false); + }); + } + + { + Jni135TestContext ctxDoLeak = new Jni135TestContext(); + // Config + ctxDoLeak.repeatCount = 3; + ctxDoLeak.msgSizeMB = 1; + + new TestUnit("Proof NOT leaking messages ", ctxDoLeak, ctx -> { + proofLeakNonLeak(ctx, true); + }); + } -// TestSuite.getDefault().run(); + { + Jni135TestContext ctxLostRefs = new Jni135TestContext(); + // Config + ctxLostRefs.repeatCount = 3; + ctxLostRefs.msgSizeMB = 1; + + new TestUnit("Lost refs cant be recovered", ctxLostRefs, ctx -> { + try { + proofLeakNonLeak(ctx, false); + assert false : "Lost references to message objects should never be recoverable"; + } catch (Throwable t) { + } +// To run one individual test only + })//.run(); // comment this in + ; + } +// AND + TestSuite.getDefault().run(); // comment this out } } diff --git a/test/java/foundation/pEp/jniadapter/test/jni161/Makefile b/test/java/foundation/pEp/jniadapter/test/jni161/Makefile new file mode 100644 index 0000000..98958e7 --- /dev/null +++ b/test/java/foundation/pEp/jniadapter/test/jni161/Makefile @@ -0,0 +1,34 @@ + include ../../../../../../../Makefile.conf +include ../Makefile.conf + +TEST_UNIT_NAME=jni161 + +JAVA_CLASSES+= \ + TestAlice.class + +.PHONY: pitytest compile alice test clean + +all: alice compile + +pitytest: + $(MAKE) -C $(PITYTEST_DIR) + +alice: compile clean-pep-home-alice + cd $(JAVA_CWD);pwd;HOME=$(JAVA_PEP_HOME_DIR_ALICE) $(JAVA) $(JAVA_PKG_BASENAME).$(TEST_UNIT_NAME).TestAlice + +compile: $(JAVA_CLASSES) pitytest + +%.class: %.java + cd $(JAVA_CWD);javac -cp $(CLASSPATH) $(JAVA_PKG_BASEPATH)/$(TEST_UNIT_NAME)/$< + +clean: + rm -f $(JAVA_CLASSES) + rm -f *.class + rm -f *.log + rm -Rf .gnupg + rm -Rf .lldb + +clean-pep-home: clean-pep-home-alice + +clean-pep-home-alice: + rm -rf $(PEP_HOME_DIR_ALICE)/.pEp diff --git a/test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java b/test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java new file mode 100644 index 0000000..bca15c0 --- /dev/null +++ b/test/java/foundation/pEp/jniadapter/test/jni161/TestAlice.java @@ -0,0 +1,132 @@ +package foundation.pEp.jniadapter.test.jni161; + +import foundation.pEp.jniadapter.Engine; +import foundation.pEp.jniadapter.Identity; +import foundation.pEp.jniadapter.Message; +import foundation.pEp.jniadapter.test.utils.AdapterTestUtils; +import foundation.pEp.jniadapter.test.utils.model.Role; +import foundation.pEp.jniadapter.test.utils.model.TestModel; +import foundation.pEp.jniadapter.test.utils.model.TestNode; +import foundation.pEp.jniadapter.test.utils.model.pEpTestIdentity; +import foundation.pEp.pitytest.AbstractTestContext; +import foundation.pEp.pitytest.TestSuite; +import foundation.pEp.pitytest.TestUnit; +import foundation.pEp.pitytest.utils.TestUtils; + +import java.util.Vector; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +import static foundation.pEp.pitytest.TestLogger.log; + + +class Jni161TestContext extends AbstractTestContext { + // Model + public TestModel> model = new TestModel(pEpTestIdentity::new, TestNode::new); + + // Basic + public Engine engine; + + // Identities + public Identity alice; + public Identity bob; + + public int blobSizeMB = 1; + public int blobCount = 3; + + public Vector messages = new Vector<>(); + + public Jni161TestContext init() throws Throwable { + engine = new Engine(); + // fetch data for the idents + alice = model.getIdent(Role.ALICE).pEpIdent; + bob = model.getIdent(Role.BOB).pEpIdent; + + // create keys etc.. + alice = engine.myself(alice); + bob = engine.myself(bob); + + return this; + } +} + +class TestAlice { + public static void main(String[] args) throws Exception { + TestSuite.getDefault().setVerbose(true); + TestSuite.getDefault().setTestColor(TestUtils.TermColor.GREEN); + + Jni161TestContext ctx161 = new Jni161TestContext(); + + new TestUnit("Proof leaking messages ", ctx161, ctx -> { + while (true) { + log("Thread: " + Thread.currentThread().getId() + " - Alloc"); + Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing); + msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount)); + msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024)); +// msg.close(); + TestUtils.sleep(10); + } + }); + + new TestUnit("Proof leaking messages ", ctx161, ctx -> { + new Thread(() -> { + while (true) { + log("Thread: " + Thread.currentThread().getId() + " - Alloc"); + Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing); + msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount)); + msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024)); + ctx.messages.add(msg); + } + }).start(); + + new Thread(() -> { + while (true) { +// log("Thread: " + Thread.currentThread().getId() + " - Dealloc"); + if(ctx.messages.size() > 0) { + log("Messages: " + ctx.messages.size()); + log("InstanceCount: " + Message.getInstanceCount()); + ctx.messages.remove(0).close(); + } + } + }).start(); + }).run(); + + + new TestUnit("Proof leaking messages ", ctx161, ctx -> { + ExecutorService allocPool = Executors.newFixedThreadPool(10); + ExecutorService deallocPool = Executors.newFixedThreadPool(10); + while (true) { + ThreadPoolExecutor allocExec = (ThreadPoolExecutor) allocPool; + if (allocExec.getActiveCount() < allocExec.getMaximumPoolSize()) { + allocPool.submit(() -> { + log("Thread: " + Thread.currentThread().getId() + " - Alloc"); + Message msg = AdapterTestUtils.makeNewTestMessage(ctx.alice, ctx.bob, Message.Direction.Outgoing); + msg.setAttachments(AdapterTestUtils.makeNewTestBlobList(ctx.blobSizeMB * 1024 * 1024, "dummyblob", "text/plain", ctx.blobCount)); + msg.setLongmsg(TestUtils.randomASCIIString(TestUtils.EASCIICharClassName.Alpha, ctx.blobSizeMB * 1024 * 1024)); + ctx.messages.add(msg); + log("MessagesADD: " + ctx.messages.size()); + }); + } + + if (ctx.messages.size() > 1) { + ThreadPoolExecutor deallocExec = (ThreadPoolExecutor) deallocPool; + if (deallocExec.getActiveCount() < deallocExec.getMaximumPoolSize()) { + deallocPool.submit(() -> { + log("Thread: " + Thread.currentThread().getId() + " - Dealloc"); + log("Messages: " + ctx.messages.size()); + log("InstanceCount: " + Message.getInstanceCount()); + for (Message msg : ctx.messages) { + msg.close(); + } + }); + } + } + TestUtils.sleep(10); + } + }); +// TestSuite.getDefault().run(); // comment this out + } +} + +