pEpEngine=Release_2.1.62

libpEpAdapter=Release_2.1.22

.gitignore

@@ -24,6 +24,7 @@ build-windows/Release/
 build-windows/packages/
 build-windows/.vs/
 build-windows/pEpPythonAdapter.vcxproj.*
+.vscode/
 launch.json
 settings.json
 # Default ignored files

DEPENDENCIES

@@ -1,6 +1,5 @@
 # 1st Party Dependencies
 ## Prefer git tags instead of SHA hashes when possible.
 
-libpEpAdapter=Release_2.1.17
+libpEpAdapter=Release_2.1.22
-pEpEngine=Release_2.1.23
+pEpEngine=Release_2.1.62
-sequoia=365d00a08bec6a5a48d48a7c7893d78c27092b59

Makefile.conf

@@ -8,7 +8,7 @@ BUILD_INPLACE = ./src/pEp/_pEp.cpython-38-darwin.so
 PYTHON_ARTIFACTS += ./.eggs
 PYTHON_ARTIFACTS += ./src/pEp.egg-info
 PYTHON_ARTIFACTS += ./.pytest_cache
-VENV_DIR = ./venv
+VENV_DIR = ./_venv
 
 # Build config Defaults
 DEBUG=1

build-windows/add_windows_libraries.py

@@ -0,0 +1,146 @@
+import os
+import re
+import sys
+import subprocess
+import winreg
+import zipfile as zf
+from os.path import dirname
+from os.path import join
+
+def get_bindependencies(lib, debug=False):
+    """Gets the pEp dependencies of a library"""
+
+    python_location = sys.exec_prefix
+    bindepend = join(python_location, 'Scripts', 'pyi-bindepend.exe')
+    output = subprocess.run([bindepend, lib], capture_output=True)
+    if os.path.isfile(lib):
+        yield lib
+    if len(output.stdout) > 0:
+        deps = parse_bindepend_output(output.stdout)
+        for dep in add_full_paths(deps, debug):
+            for d in get_bindependencies(dep):
+                yield d
+
+
+def parse_bindepend_output(stdout):
+    """Parses the output of pyi-bindepent and returns a list of file names"""
+    
+    deps = []
+    if stdout == None:
+        return None
+    p = re.compile('(.*){(.*)}(.*)')
+    str_val = bytes.decode(stdout)
+    raw_val = p.findall(str_val)
+    if len(raw_val) == 0:
+        return deps
+    else:
+        libs = raw_val[0][1].replace(' ', '').split(',')
+        for lib in libs:
+            lib = lib.strip("'")
+            if not 'api' in lib:
+                deps.append(lib)
+    return deps    
+
+
+def get_pEp_install_location(debug=False):
+    """Gets the location where pEp is installed"""
+
+    reg_path = "SOFTWARE\\Classes\\TypeLib\\{564A4350-419E-47F1-B0DF-6FCCF0CD0BBC}\\1.0\\0\\win32"
+    KeyName = None
+    regKey = None
+    try:
+        regKey = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, reg_path, 0, winreg.KEY_READ)
+        # Keys: Description, FileName, FriendlyName, LoadBehavior
+        com_server, _ = winreg.QueryValueEx(regKey, KeyName)
+    except WindowsError as error:
+        print('Error getting install location: ' + error)
+        com_server = None
+    finally:
+        if winreg:
+            winreg.CloseKey(regKey)
+    location = os.path.dirname(com_server)
+    if debug:
+        location = location.strip('Release') + 'Debug'
+    return location
+
+
+def get_boost_directories():
+    """Gets the location of the boost libraries"""
+    
+    for dir in [f.path for f in os.scandir(join(dirname(os.getcwd()), 'packages')) if f.is_dir()]:
+        if 'boost.' in dir or 'boost_python' in dir or 'boost_locale' in dir:
+            yield join(dir, 'lib', 'native')    
+
+
+def add_full_paths(libs, debug=False):
+    """Combines a list of libraries with a set of common directories. Returns a list of file names that exist on this machine"""
+    
+    paths = [get_pEp_install_location(debug)] + [dir for dir in get_boost_directories()] + [p for p in os.environ['PATH'].split(';')]
+    paths = [p for p in paths if not "system32" in p.lower()]
+    full_paths = [] 
+    for lib in libs:
+        for p in paths:
+            test_path = join(p, lib)
+            if os.path.isfile(test_path) and not test_path.casefold() in (f.casefold for f in full_paths):
+                full_paths.append(test_path)
+    return full_paths
+
+
+def main():  
+    
+    args = sys.argv
+    cwd = dirname(args[0])
+    del args[0]
+
+    # Check for debug build
+    debug = False
+    if len(args) > 0 and args[0] == '--debug':
+        debug = True
+        del args[0]
+
+    # Get the pEp wheel and extract the pEp Python library
+    dist_path = join(dirname(cwd), 'dist')
+    print('Dist path is: ' + dist_path)
+    pEp_python_library = None
+    if os.path.exists(dist_path):
+        for _, _, files in os.walk(dist_path):
+            for f in files:
+                if f[:3] == 'pEp' and f[-3:] == 'whl':
+                    wheel_name = join(dist_path, f)
+                    with zf.ZipFile(wheel_name, 'a') as wheel:
+                        for archive_member in wheel.namelist():
+                            if '_pEp' in archive_member and archive_member[-3:] == 'pyd':
+                                wheel.extract(archive_member, path=dist_path)                            
+                                pEp_python_library = join(dist_path, archive_member)
+                                print('pEp Python library found and extracted to ' + pEp_python_library)
+                                break
+                    break
+
+    # Get all dependencies for the pEp Python library
+    if pEp_python_library == None:
+        raise FileNotFoundError('pEp Python library not found in ' + dist_path)
+    pEp_python_library = pEp_python_library.replace('/', '\\')
+    libs = []
+    for lib in get_bindependencies(pEp_python_library, debug):
+        if not lib.casefold() in (l.casefold for l in libs) and lib.casefold() != pEp_python_library.casefold():
+            print('Dependency found: ' + lib)
+            libs.append(lib)
+
+    # Copy dependencies into the wheel
+    with zf.ZipFile(wheel_name, 'a') as wheel:
+        for lib in libs:
+            filename = os.path.basename(lib)
+            arcname = 'pEp/' + filename
+            if not arcname in wheel.namelist():
+                wheel.write(filename=lib, arcname=arcname, compress_type=zf.ZIP_DEFLATED)
+
+    # Delete the temporarily extracted pEp Python library
+    if os.path.isfile(pEp_python_library):
+        os.remove(pEp_python_library)
+        dir = os.path.dirname(pEp_python_library)
+        if dir[-3:].lower() == 'pep' and os.path.isdir(dir):
+            os.rmdir(dir)
+
+
+if __name__ == '__main__':
+    main()

build-windows/pEpPythonAdapter.sln

@@ -1,31 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.31005.135
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pEpPythonAdapter", "pEpPythonAdapter.vcxproj", "{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x64.ActiveCfg = Debug|x64
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x64.Build.0 = Debug|x64
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x86.ActiveCfg = Debug|Win32
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x86.Build.0 = Debug|Win32
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x64.ActiveCfg = Release|x64
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x64.Build.0 = Release|x64
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x86.ActiveCfg = Release|Win32
-		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x86.Build.0 = Release|Win32
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {B63BC9BA-EF76-4FB4-9126-29CBFAD9092C}
-	EndGlobalSection
-EndGlobal

build-windows/pEpPythonAdapter.vcxproj

@@ -95,18 +95,23 @@
     <None Include="..\setup.py" />
     <None Include="packages.config" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\pEpEngine\build-windows\pEpEngine.vcxproj">
+      <Project>{146e69f8-e1da-456a-b048-6dd29d9acf6b}</Project>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
-    <Import Project="packages\boost.1.72.0.0\build\boost.targets" Condition="Exists('packages\boost.1.72.0.0\build\boost.targets')" />
+    <Import Project="..\..\packages\boost.1.72.0.0\build\boost.targets" Condition="Exists('..\..\packages\boost.1.72.0.0\build\boost.targets')" />
-    <Import Project="packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets" Condition="Exists('packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" />
+    <Import Project="..\..\packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets" Condition="Exists('..\..\packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" />
-    <Import Project="packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets" Condition="Exists('packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" />
+    <Import Project="..\..\packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets" Condition="Exists('..\..\packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" />
   </ImportGroup>
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('packages\boost.1.72.0.0\build\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost.1.72.0.0\build\boost.targets'))" />
+    <Error Condition="!Exists('..\..\packages\boost.1.72.0.0\build\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\boost.1.72.0.0\build\boost.targets'))" />
-    <Error Condition="!Exists('packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets'))" />
+    <Error Condition="!Exists('..\..\packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets'))" />
-    <Error Condition="!Exists('packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets'))" />
+    <Error Condition="!Exists('..\..\packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets'))" />
   </Target>
 </Project>

make.mak

@@ -1,6 +1,6 @@
 # build dirs
-BUILD_DIR = $(ProjectDir)..\build
+BUILD_DIR = build
-DIST_DIR = $(ProjectDir)..\dist
+DIST_DIR = dist
 
 # create wheel and egg package in dist
 dist: dist-whl dist-egg
@@ -31,9 +31,13 @@ release: clean
     CD..
     PY -3.8-32 setup.py build_ext
     PY -3.8-32 setup.py bdist_wheel
+    PY -3.8-32 build-windows\add_windows_libraries.py
+    COPY dist\pEp* ..\Release\
 
 #debug build
 debug: clean
     CD..
     PY -3.8-32 setup.py build_ext --debug
-    PY -3.8-32 setup.py bdist_wheel
+    PY -3.8-32 setup.py bdist_wheel
+    PY -3.8-32 build-windows\add_windows_libraries.py --debug
+    COPY dist\pEp* ..\Debug\

pyproject.toml

@@ -1,19 +0,0 @@
-
-[build-system]
-# Preparing for PEP-517/PEP-518, but not in effect yet.
-# These requires are not effective yet, setup.cfg is.
-requires =[
-        "setuptools >=39.2.0",
-        "setuptools_scm >= 4.1.2",
-        "wheel",
-        ]
-
-build-backend = "setuptools.build_meta"
-
-[tool.pytest.ini_options]
-minversion = "6.0"
-addopts = "-rP --forked"
-testpaths = [
-    "tests",
-]
-

setup.cfg

@@ -27,14 +27,13 @@ classifiers =
 [options]
 zip_safe = false
 include_package_data = true
-python_requires = >=3.7,<3.9
+python_requires = >=3.9
 test_suite = tests
 install_requires =
 # deprecated/redundant with pyproject.toml, but lets keep both ways around for now
 setup_requires =
     setuptools >=39.2.0
     setuptools_scm >= 4.1.2
-    wheel
 [options.extras_require]
 # To install these dependencies, run pip install .[test]
 test =

setup.py

@@ -60,7 +60,7 @@ class BuildExtCommand(build_ext):
         return ret
 
     def windowsGetBoostDirs(self):
-        for dir in [f.path for f in os.scandir(join(os.getcwd(), 'build-windows', 'packages')) if f.is_dir()]:
+        for dir in [f.path for f in os.scandir(join(os.path.dirname(os.getcwd()), 'packages')) if f.is_dir()]:
             if 'boost.' in dir or 'boost_python' in dir or 'boost_locale' in dir:
                 yield join(dir, 'lib', 'native'), join(dir, 'lib', 'native', 'include')
 
@@ -97,7 +97,7 @@ class BuildExtCommand(build_ext):
         libs = [
             'pEpEngine',
             'pEpAdapter',
-            'boost_python38-mt',
+            'boost_python3-mt',
             'boost_locale-mt'
         ]
         compile_flags = ['-std=c++14', '-fpermissive']

src/pEp/__init__.py

@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 # pEp package
 #
 # The names that are in _pEp that do not begin with an underscore, will be be imported into, and "re-exported" from this module.
@@ -30,7 +30,7 @@ from ._pEp import *
 
 # import the native module into the current namespace because we also need to access the names beginning
 # with an underscore (of _pEp), but we dont want to import them into this module
-import pEp._pEp
+from . import _pEp
 
 # 3rd party imports
 from threading import Thread, Barrier

src/pEp/_pEp/message.cc

@@ -34,24 +34,8 @@ namespace pEp {
             if (!_bl)
                 throw bad_alloc();
 
-            Py_buffer src;
-            int result = PyObject_GetBuffer(data.ptr(), &src, PyBUF_CONTIG_RO);
-            if (result)
-                throw invalid_argument("need a contiguous buffer to read");
-
-            char *mem = (char *) malloc(src.len);
-            if (!mem) {
-                PyBuffer_Release(&src);
-                throw bad_alloc();
-            }
-
-            memcpy(mem, src.buf, src.len);
-            free(_bl->value);
-            _bl->size = src.len;
-            _bl->value = mem;
-
-            PyBuffer_Release(&src);
 
+            this->data(data);
             this->mime_type(mime_type);
             this->filename(filename);
         }
@@ -110,6 +94,30 @@ namespace pEp {
             return PyBuffer_FillInfo(view, self, bl->value, bl->size, 0, flags);
         }
 
+        object Message::Blob::data() {
+            return object(handle<>(PyBytes_FromStringAndSize(_bl->value, _bl->size)));
+        }
+
+        void Message::Blob::data(object data) {
+            Py_buffer src;
+            int result = PyObject_GetBuffer(data.ptr(), &src, PyBUF_CONTIG_RO);
+            if (result)
+                throw invalid_argument("need a contiguous buffer to read");
+
+            char *mem = (char *) malloc(src.len);
+            if (!mem) {
+                PyBuffer_Release(&src);
+                throw bad_alloc();
+            }
+
+            memcpy(mem, src.buf, src.len);
+            free(_bl->value);
+            _bl->size = src.len;
+            _bl->value = mem;
+
+            PyBuffer_Release(&src);
+        }
+
         string Message::Blob::decode(string encoding) {
             if (encoding == "") {
                 string _mime_type = _bl->mime_type ? _bl->mime_type : "";

src/pEp/_pEp/message.hh

@@ -59,6 +59,10 @@ namespace pEp {
 
                 size_t size() { return _bl->size; }
 
+                object data();
+
+                void data(object data);
+
                 string decode(string encoding);
 
                 string decode() { return decode(""); }
@@ -191,4 +195,4 @@ namespace pEp {
     } /* namespace PythonAdapter */
 } /* namespace pEp */
 
-#endif /* MESSAGE_HH */
+#endif /* MESSAGE_HH */

src/pEp/_pEp/pEpmodule.cc

@@ -14,6 +14,7 @@
 #include <pEp/message_api.h>
 #include <pEp/sync_api.h>
 #include <pEp/status_to_string.h>
+#include <pEp/media_key.h>
 
 // libpEpAdapter
 #include <pEp/Adapter.hh>
@@ -62,6 +63,14 @@ namespace pEp {
             ::config_unencrypted_subject(Adapter::session(), enable);
         }
 
+        void config_enable_echo_protocol(bool enable) {
+            ::config_enable_echo_protocol(Adapter::session(), enable);
+        }
+
+        void config_enable_echo_in_outgoing_message_rating_preview(bool enable) {
+            ::config_enable_echo_in_outgoing_message_rating_preview(Adapter::session(), enable);
+        }
+
         void key_reset_user(string user_id, string fpr) {
             if (user_id == "")
                 throw invalid_argument("user_id required");
@@ -185,6 +194,19 @@ namespace pEp {
             ::disable_all_sync_channels(Adapter::session());
         }
 
+        void sync_reinit() {
+            PEP_STATUS status = ::sync_reinit(Adapter::session());
+            _throw_status(status);
+        }
+
+        void config_media_keys(dict value) {
+            ::stringpair_list_t *_spl = nullptr;
+            strdict_attr(_spl, value);
+
+            PEP_STATUS status = ::config_media_keys(Adapter::session(), _spl);
+            _throw_status(status);
+        }
+
         void testfunc() {
             _messageToSend(NULL);
         }
@@ -269,6 +291,12 @@ namespace pEp {
                 def("unencrypted_subject", config_unencrypted_subject,
                 "do not encrypt the subject of messages");
 
+                def("config_enable_echo_protocol", config_enable_echo_protocol,
+                "enable or disable the Distribution.Echo");
+
+                def("config_enable_echo_in_outgoing_message_rating_preview", config_enable_echo_in_outgoing_message_rating_preview,
+                "enable or disable Distribution.Echo.Ping");
+
                 def("key_reset", key_reset_user,
                 "reset the default database status for the user / keypair provided\n"
                 "This will effectively perform key_reset on each identity\n"
@@ -399,7 +427,10 @@ namespace pEp {
                 "MIME type of object in Blob")
                 .add_property("filename", (string(Message::Blob::*)()) &Message::Blob::filename,
                 (void(Message::Blob::*)(string)) &Message::Blob::filename,
-                "filename of object in Blob");
+                "filename of object in Blob")
+                .add_property("data", (object(Message::Blob::*)()) &Message::Blob::data,
+                (void(Message::Blob::*)(object)) &Message::Blob::data,
+                "data of object in Blob");
 
                 ((PyTypeObject *)(void *)blob_class.ptr())->tp_as_buffer = &Message::Blob::bp;
 
@@ -649,7 +680,8 @@ namespace pEp {
                 .value("SYNC_NOTIFY_ACCEPTED_GROUP_CREATED", SYNC_NOTIFY_ACCEPTED_GROUP_CREATED)
                 .value("SYNC_NOTIFY_ACCEPTED_DEVICE_ACCEPTED", SYNC_NOTIFY_ACCEPTED_DEVICE_ACCEPTED)
                 .value("SYNC_NOTIFY_SOLE", SYNC_NOTIFY_SOLE)
-                .value("SYNC_NOTIFY_IN_GROUP", SYNC_NOTIFY_IN_GROUP);
+                .value("SYNC_NOTIFY_IN_GROUP", SYNC_NOTIFY_IN_GROUP)
+                .value("SYNC_NOTIFY_OUTGOING_RATING_CHANGE", SYNC_NOTIFY_OUTGOING_RATING_CHANGE);
 
                 def("deliver_handshake_result", &deliverHandshakeResult, boost::python::arg("identities")=object(),
                 "deliverHandshakeResult(self, result, identities=None)\n"
@@ -675,6 +707,15 @@ namespace pEp {
                 "Disable sync for all identities\n"
                 );
 
+                def("sync_reinit", &sync_reinit,
+                "Explicitly reinitialize Sync.  This is meant to be explicitly called\n"
+                "from the application upon user request\n"
+                );
+
+                def("config_media_keys", &config_media_keys,
+                "Replace the session map with the given map"
+                );
+
                 // codecs
                 call< object >(((object)(import("codecs").attr("register"))).ptr(), make_function(sync_search));
                 call< object >(((object)(import("codecs").attr("register"))).ptr(), make_function(distribution_search));

tests/test_blob.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# This file is under GNU Affero General Public License 3.0
+# see LICENSE.txt
+
+"""Blob unit tests."""
+
+import pytest
+
+from . import constants
+from . import model
+
+def test_blob_data_constructor(pEp):
+    bdata = b'this is binary \x00\x01\xbb\xa7\xa4\xab test data'
+    b = pEp.Blob(bdata)
+    assert b.data == bdata
+    assert not b.mime_type
+    assert not b.filename
+
+
+def test_blob_data_property(pEp):
+    bdata = b'this is binary \x00\x01\xbb\xa7\xa4\xab test data'
+    b = pEp.Blob(b'dummy')
+    b.data = bdata
+    assert b.data == bdata
+    assert not b.mime_type
+    assert not b.filename
+
+
+def test_blob_data_property_keeps_other_fields(pEp):
+    bdata = b'this is binary \x00\x01\xbb\xa7\xa4\xab test data'
+    b = pEp.Blob(b'dummy', 'application/x-mydata', 'myfile.dat')
+    b.data = bdata
+    assert b.data == bdata
+    assert b.mime_type == 'application/x-mydata'
+    assert b.filename == 'myfile.dat'

tests/test_identity.py

@@ -29,6 +29,18 @@ def test_identity_constructor(pEp, model):
     assert str(alice) == str(model.alice)
 
 
+# Covers PYADPT-124
+def test_identity_update_cpt(pEp,model):
+    bob = pEp.Identity(model.bob.addr, model.bob.name)
+    bob.update()
+    assert bob.address == model.bob.addr
+    assert bob.username == model.bob.name
+    assert bob.user_id == 'TOFU_bob@peptest.org'
+    assert bob.fpr == ''
+    assert bob.comm_type == 0
+    assert bob.lang == ''
+
+
 # TODO:
 # These here are actually plenty of individual tests
 # Identity.update