diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h
--- a/src/script/signingprovider.h
+++ b/src/script/signingprovider.h
@@ -78,17 +78,10 @@
     mutable RecursiveMutex cs_KeyStore;
 
     using KeyMap = std::map<CKeyID, CKey>;
-    using WatchKeyMap = std::map<CKeyID, CPubKey>;
     using ScriptMap = std::map<CScriptID, CScript>;
-    using WatchOnlySet = std::set<CScript>;
 
     KeyMap mapKeys GUARDED_BY(cs_KeyStore);
-    WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore);
     ScriptMap mapScripts GUARDED_BY(cs_KeyStore);
-    WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore);
-
-    void ImplicitlyLearnRelatedKeyScripts(const CPubKey &pubkey)
-        EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore);
 
 public:
     virtual bool AddKeyPubKey(const CKey &key, const CPubKey &pubkey);
@@ -105,11 +98,6 @@
     virtual std::set<CScriptID> GetCScripts() const;
     virtual bool GetCScript(const CScriptID &hash,
                             CScript &redeemScriptOut) const override;
-
-    virtual bool AddWatchOnly(const CScript &dest);
-    virtual bool RemoveWatchOnly(const CScript &dest);
-    virtual bool HaveWatchOnly(const CScript &dest) const;
-    virtual bool HaveWatchOnly() const;
 };
 
 /**
diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp
--- a/src/script/signingprovider.cpp
+++ b/src/script/signingprovider.cpp
@@ -83,33 +83,10 @@
     return ret;
 }
 
-void FillableSigningProvider::ImplicitlyLearnRelatedKeyScripts(
-    const CPubKey &pubkey) {
-    AssertLockHeld(cs_KeyStore);
-    CKeyID key_id = pubkey.GetID();
-    // We must actually know about this key already.
-    assert(HaveKey(key_id) || mapWatchKeys.count(key_id));
-    // This adds the redeemscripts necessary to detect alternative outputs using
-    // the same keys. Also note that having superfluous scripts in the keystore
-    // never hurts. They're only used to guide recursion in signing and IsMine
-    // logic - if a script is present but we can't do anything with it, it has
-    // no effect. "Implicitly" refers to fact that scripts are derived
-    // automatically from existing keys, and are present in memory, even without
-    // being explicitly loaded (e.g. from a file).
-
-    // Right now there are none so do nothing.
-}
-
 bool FillableSigningProvider::GetPubKey(const CKeyID &address,
                                         CPubKey &vchPubKeyOut) const {
     CKey key;
     if (!GetKey(address, key)) {
-        LOCK(cs_KeyStore);
-        WatchKeyMap::const_iterator it = mapWatchKeys.find(address);
-        if (it != mapWatchKeys.end()) {
-            vchPubKeyOut = it->second;
-            return true;
-        }
         return false;
     }
     vchPubKeyOut = key.GetPubKey();
@@ -120,7 +97,6 @@
                                            const CPubKey &pubkey) {
     LOCK(cs_KeyStore);
     mapKeys[pubkey.GetID()] = key;
-    ImplicitlyLearnRelatedKeyScripts(pubkey);
     return true;
 }
 
@@ -187,58 +163,6 @@
     return false;
 }
 
-static bool ExtractPubKey(const CScript &dest, CPubKey &pubKeyOut) {
-    // TODO: Use Solver to extract this?
-    CScript::const_iterator pc = dest.begin();
-    opcodetype opcode;
-    std::vector<uint8_t> vch;
-    if (!dest.GetOp(pc, opcode, vch) || !CPubKey::ValidSize(vch)) {
-        return false;
-    }
-    pubKeyOut = CPubKey(vch);
-    if (!pubKeyOut.IsFullyValid()) {
-        return false;
-    }
-    if (!dest.GetOp(pc, opcode, vch) || opcode != OP_CHECKSIG ||
-        dest.GetOp(pc, opcode, vch)) {
-        return false;
-    }
-    return true;
-}
-
-bool FillableSigningProvider::AddWatchOnly(const CScript &dest) {
-    LOCK(cs_KeyStore);
-    setWatchOnly.insert(dest);
-    CPubKey pubKey;
-    if (ExtractPubKey(dest, pubKey)) {
-        mapWatchKeys[pubKey.GetID()] = pubKey;
-        ImplicitlyLearnRelatedKeyScripts(pubKey);
-    }
-    return true;
-}
-
-bool FillableSigningProvider::RemoveWatchOnly(const CScript &dest) {
-    LOCK(cs_KeyStore);
-    setWatchOnly.erase(dest);
-    CPubKey pubKey;
-    if (ExtractPubKey(dest, pubKey)) {
-        mapWatchKeys.erase(pubKey.GetID());
-    }
-    // Related CScripts are not removed; having superfluous scripts around is
-    // harmless (see comment in ImplicitlyLearnRelatedKeyScripts).
-    return true;
-}
-
-bool FillableSigningProvider::HaveWatchOnly(const CScript &dest) const {
-    LOCK(cs_KeyStore);
-    return setWatchOnly.count(dest) > 0;
-}
-
-bool FillableSigningProvider::HaveWatchOnly() const {
-    LOCK(cs_KeyStore);
-    return (!setWatchOnly.empty());
-}
-
 CKeyID GetKeyForDestination(const SigningProvider &store,
                             const CTxDestination &dest) {
     // Only supports destinations which map to single public keys, i.e. P2PKH.
diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h
--- a/src/wallet/wallet.h
+++ b/src/wallet/wallet.h
@@ -772,6 +772,8 @@
 
     using CryptedKeyMap =
         std::map<CKeyID, std::pair<CPubKey, std::vector<uint8_t>>>;
+    using WatchOnlySet = std::set<CScript>;
+    using WatchKeyMap = std::map<CKeyID, CPubKey>;
 
     bool SetCrypted();
 
@@ -781,6 +783,8 @@
     bool Unlock(const CKeyingMaterial &vMasterKeyIn,
                 bool accept_no_keys = false);
     CryptedKeyMap mapCryptedKeys GUARDED_BY(cs_KeyStore);
+    WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore);
+    WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore);
 
     bool AddCryptedKeyInner(const CPubKey &vchPubKey,
                             const std::vector<uint8_t> &vchCryptedSecret);
@@ -891,10 +895,10 @@
      * AddWatchOnly which accepts a timestamp and sets nTimeFirstKey more
      * intelligently for more efficient rescans.
      */
-    bool AddWatchOnly(const CScript &dest) override
-        EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
+    bool AddWatchOnly(const CScript &dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
     bool AddWatchOnlyWithDB(WalletBatch &batch, const CScript &dest)
         EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
+    bool AddWatchOnlyInMem(const CScript &dest);
 
     /** Add a KeyOriginInfo to the wallet */
     bool AddKeyOriginWithDB(WalletBatch &batch, const CPubKey &pubkey,
@@ -1185,11 +1189,17 @@
     //! Adds a watch-only address to the store, and saves it to disk.
     bool AddWatchOnly(const CScript &dest, int64_t nCreateTime)
         EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
-    bool RemoveWatchOnly(const CScript &dest) override
+    bool RemoveWatchOnly(const CScript &dest)
         EXCLUSIVE_LOCKS_REQUIRED(cs_wallet);
     //! Adds a watch-only address to the store, without saving it to disk (used
     //! by LoadWallet)
     bool LoadWatchOnly(const CScript &dest);
+    //! Returns whether the watch-only script is in the wallet
+    bool HaveWatchOnly(const CScript &dest) const;
+    //! Returns whether there are any watch-only things in the wallet
+    bool HaveWatchOnly() const;
+    //! Fetches a pubkey from mapWatchKeys if it exists there
+    bool GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const;
 
     //! Holds a timestamp at which point the wallet is scheduled (externally) to
     //! be relocked. Caller must arrange for actual relocking to occur via
diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp
--- a/src/wallet/wallet.cpp
+++ b/src/wallet/wallet.cpp
@@ -585,8 +585,37 @@
     return FillableSigningProvider::AddCScript(redeemScript);
 }
 
+static bool ExtractPubKey(const CScript &dest, CPubKey &pubKeyOut) {
+    // TODO: Use Solver to extract this?
+    CScript::const_iterator pc = dest.begin();
+    opcodetype opcode;
+    std::vector<uint8_t> vch;
+    if (!dest.GetOp(pc, opcode, vch) || !CPubKey::ValidSize(vch)) {
+        return false;
+    }
+    pubKeyOut = CPubKey(vch);
+    if (!pubKeyOut.IsFullyValid()) {
+        return false;
+    }
+    if (!dest.GetOp(pc, opcode, vch) || opcode != OP_CHECKSIG ||
+        dest.GetOp(pc, opcode, vch)) {
+        return false;
+    }
+    return true;
+}
+
+bool CWallet::AddWatchOnlyInMem(const CScript &dest) {
+    LOCK(cs_KeyStore);
+    setWatchOnly.insert(dest);
+    CPubKey pubKey;
+    if (ExtractPubKey(dest, pubKey)) {
+        mapWatchKeys[pubKey.GetID()] = pubKey;
+    }
+    return true;
+}
+
 bool CWallet::AddWatchOnlyWithDB(WalletBatch &batch, const CScript &dest) {
-    if (!FillableSigningProvider::AddWatchOnly(dest)) {
+    if (!AddWatchOnlyInMem(dest)) {
         return false;
     }
 
@@ -618,8 +647,13 @@
 
 bool CWallet::RemoveWatchOnly(const CScript &dest) {
     AssertLockHeld(cs_wallet);
-    if (!FillableSigningProvider::RemoveWatchOnly(dest)) {
-        return false;
+    {
+        LOCK(cs_KeyStore);
+        setWatchOnly.erase(dest);
+        CPubKey pubKey;
+        if (ExtractPubKey(dest, pubKey)) {
+            mapWatchKeys.erase(pubKey.GetID());
+        }
     }
 
     if (!HaveWatchOnly()) {
@@ -630,7 +664,17 @@
 }
 
 bool CWallet::LoadWatchOnly(const CScript &dest) {
-    return FillableSigningProvider::AddWatchOnly(dest);
+    return AddWatchOnlyInMem(dest);
+}
+
+bool CWallet::HaveWatchOnly(const CScript &dest) const {
+    LOCK(cs_KeyStore);
+    return setWatchOnly.count(dest) > 0;
+}
+
+bool CWallet::HaveWatchOnly() const {
+    LOCK(cs_KeyStore);
+    return (!setWatchOnly.empty());
 }
 
 bool CWallet::Unlock(const SecureString &strWalletPassphrase,
@@ -5326,10 +5370,23 @@
     return false;
 }
 
+bool CWallet::GetWatchPubKey(const CKeyID &address, CPubKey &pubkey_out) const {
+    LOCK(cs_KeyStore);
+    WatchKeyMap::const_iterator it = mapWatchKeys.find(address);
+    if (it != mapWatchKeys.end()) {
+        pubkey_out = it->second;
+        return true;
+    }
+    return false;
+}
+
 bool CWallet::GetPubKey(const CKeyID &address, CPubKey &vchPubKeyOut) const {
     LOCK(cs_KeyStore);
     if (!IsCrypted()) {
-        return FillableSigningProvider::GetPubKey(address, vchPubKeyOut);
+        if (!FillableSigningProvider::GetPubKey(address, vchPubKeyOut)) {
+            return GetWatchPubKey(address, vchPubKeyOut);
+        }
+        return true;
     }
 
     CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address);
@@ -5339,7 +5396,7 @@
     }
 
     // Check for watch-only pubkeys
-    return FillableSigningProvider::GetPubKey(address, vchPubKeyOut);
+    return GetWatchPubKey(address, vchPubKeyOut);
 }
 
 std::set<CKeyID> CWallet::GetKeys() const {
@@ -5409,6 +5466,5 @@
     }
 
     mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret);
-    ImplicitlyLearnRelatedKeyScripts(vchPubKey);
     return true;
 }