diff --git a/passphrase_cache.cc b/passphrase_cache.cc
index 196a352..6b372bf 100644
--- a/passphrase_cache.cc
+++ b/passphrase_cache.cc
@@ -4,6 +4,8 @@ namespace pEp {
     void PassphraseCache::add(std::string passphrase)
     {
         std::lock_guard<std::mutex> lock(_mtx);
+        while (_cache.size() >= _max_size)
+            _cache.pop_front();
         _cache.push_back({passphrase, clock::now()});
     }
 
@@ -26,8 +28,6 @@ namespace pEp {
     {
         while (!_cache.empty() && _cache.front().tp < clock::now() - _timeout)
             _cache.pop_front();
-        while (_cache.size() > _max_size)
-            _cache.pop_front();
     }
 
     void PassphraseCache::refresh(cache::iterator entry)