diff --git a/src/net_processing.cpp b/src/net_processing.cpp
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -3752,16 +3752,18 @@
                  peer->m_starting_height, addrMe.ToString(), fRelay,
                  pfrom.GetId(), remoteAddr);
 
-        // Ignore time offsets that are improbable (before the Genesis block)
-        // and may underflow the nTimeOffset calculation.
         int64_t currentTime = GetTime();
-        if (nTime >= int64_t(m_chainparams.GenesisBlock().nTime)) {
-            int64_t nTimeOffset = nTime - currentTime;
-            pfrom.nTimeOffset = nTimeOffset;
-            AddTimeData(pfrom.addr, nTimeOffset);
-        } else {
+        int64_t nTimeOffset = nTime - currentTime;
+        pfrom.nTimeOffset = nTimeOffset;
+        if (nTime < int64_t(m_chainparams.GenesisBlock().nTime)) {
+            // Ignore time offsets that are improbable (before the Genesis
+            // block) and may underflow our adjusted time.
             Misbehaving(pfrom, 20,
                         "Ignoring invalid timestamp in version message");
+        } else if (!pfrom.IsInboundConn()) {
+            // Don't use timedata samples from inbound peers to make it
+            // harder for others to tamper with our adjusted time.
+            AddTimeData(pfrom.addr, nTimeOffset);
         }
 
         // Feeler connections exist only to verify if address is online.
diff --git a/test/functional/abc_p2p_version_timestamp.py b/test/functional/abc_p2p_version_timestamp.py
--- a/test/functional/abc_p2p_version_timestamp.py
+++ b/test/functional/abc_p2p_version_timestamp.py
@@ -5,14 +5,17 @@
 """Test time adjustment behavior when receiving a VERSION message.
 
 Messages with a timestamp too far in the past are ignored and discouraged.
-Messages with a timestamp more recent than the genesis block timestamp are used to
-adjust our local time.
+Messages from an outbound peer with a timestamp more recent than the genesis block
+timestamp are used to adjust our local time.
 """
 
 from test_framework.blocktools import TIME_GENESIS_BLOCK
 from test_framework.messages import NODE_NETWORK, msg_version
 from test_framework.p2p import P2P_SUBVERSION, P2P_VERSION, P2PInterface
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+NODE0_TIME = TIME_GENESIS_BLOCK + 1337
 
 
 class ModifiedVersionTimestampP2PInterface(P2PInterface):
@@ -44,9 +47,17 @@
 
 class VersionTimestampTest(BitcoinTestFramework):
     def set_test_params(self):
-        self.num_nodes = 1
+        self.num_nodes = 2
+
+    def setup_network(self):
+        self.setup_nodes()
+        # Don't connect the nodes
 
     def run_test(self):
+        # Set a deterministic time so that we can check the time offset with other
+        # nodes is as expected
+        self.nodes[0].setmocktime(NODE0_TIME)
+
         self.log.info("Check some invalid timestamp in the version message")
         with self.nodes[0].assert_debug_log(
                 expected_msgs=["Added connection peer=0",
@@ -64,9 +75,22 @@
                 send_version=False)
 
         self.log.info("Check valid side of the timestamp boundary (genesis timestamp)")
+        # Outbound connection: the timestamp is used for our adjusted time
+        self.nodes[1].setmocktime(TIME_GENESIS_BLOCK)
         with self.nodes[0].assert_debug_log(
                 expected_msgs=["Added connection peer=2", "added time data"],
                 unexpected_msgs=["Ignoring invalid timestamp in version message"]):
+            self.connect_nodes(0, 1)
+
+        # This check verifies that the mocktime was indeed used in the VERSION message
+        assert_equal(self.nodes[0].getpeerinfo()[2]["timeoffset"],
+                     TIME_GENESIS_BLOCK - NODE0_TIME)
+
+        # Inbound connection: the timestamp is ignored
+        with self.nodes[0].assert_debug_log(
+                expected_msgs=["Added connection peer=3"],
+                unexpected_msgs=["Ignoring invalid timestamp in version message",
+                                 "added time data"]):
             self.nodes[0].add_p2p_connection(
                 ModifiedVersionTimestampP2PInterface(TIME_GENESIS_BLOCK),
                 send_version=False)