diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h
--- a/src/wallet/scriptpubkeyman.h
+++ b/src/wallet/scriptpubkeyman.h
@@ -204,6 +204,8 @@
     virtual size_t KeypoolCountExternalKeys() { return 0; }
     virtual unsigned int GetKeyPoolSize() const { return 0; }
 
+    virtual int64_t GetTimeFirstKey() const { return 0; }
+
     virtual const CKeyMetadata *GetMetadata(uint160 id) const {
         return nullptr;
     }
@@ -337,6 +339,8 @@
         EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
     unsigned int GetKeyPoolSize() const override;
 
+    int64_t GetTimeFirstKey() const override;
+
     const CKeyMetadata *GetMetadata(uint160 id) const override;
 
     bool CanGetAddresses(bool internal = false) override;
diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp
--- a/src/wallet/scriptpubkeyman.cpp
+++ b/src/wallet/scriptpubkeyman.cpp
@@ -424,6 +424,11 @@
     return setInternalKeyPool.size() + setExternalKeyPool.size();
 }
 
+int64_t LegacyScriptPubKeyMan::GetTimeFirstKey() const {
+    AssertLockHeld(cs_wallet);
+    return nTimeFirstKey;
+}
+
 const CKeyMetadata *LegacyScriptPubKeyMan::GetMetadata(uint160 id) const {
     AssertLockHeld(cs_wallet);
     auto it = mapKeyMetadata.find(CKeyID(id));
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -1362,7 +1362,6 @@
         mapWatchKeys GUARDED_BY(cs_KeyStore) = m_spk_man->mapWatchKeys;
     WalletBatch *&
         encrypted_batch GUARDED_BY(cs_wallet) = m_spk_man->encrypted_batch;
-    int64_t &nTimeFirstKey GUARDED_BY(cs_wallet) = m_spk_man->nTimeFirstKey;
     using CryptedKeyMap = LegacyScriptPubKeyMan::CryptedKeyMap;
 };
 
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -4255,11 +4255,18 @@
 
         // No need to read and scan block if block was created before our wallet
         // birthday (as adjusted for block time variability)
-        if (walletInstance->nTimeFirstKey) {
+        Optional<int64_t> time_first_key;
+        if (auto spk_man = walletInstance->m_spk_man.get()) {
+            int64_t time = spk_man->GetTimeFirstKey();
+            if (!time_first_key || time < *time_first_key) {
+                time_first_key = time;
+            }
+        }
+        if (time_first_key) {
             if (Optional<int> first_block =
                     locked_chain->findFirstBlockWithTimeAndHeight(
-                        walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW,
-                        rescan_height, nullptr)) {
+                        *time_first_key - TIMESTAMP_WINDOW, rescan_height,
+                        nullptr)) {
                 rescan_height = *first_block;
             }
         }