diff --git a/src/net_processing.cpp b/src/net_processing.cpp
--- a/src/net_processing.cpp
+++ b/src/net_processing.cpp
@@ -3752,16 +3752,18 @@
                  peer->m_starting_height, addrMe.ToString(), fRelay,
                  pfrom.GetId(), remoteAddr);
 
-        // Ignore time offsets that are improbable (before the Genesis block)
-        // and may underflow the nTimeOffset calculation.
         int64_t currentTime = GetTime();
-        if (nTime >= int64_t(m_chainparams.GenesisBlock().nTime)) {
-            int64_t nTimeOffset = nTime - currentTime;
-            pfrom.nTimeOffset = nTimeOffset;
-            AddTimeData(pfrom.addr, nTimeOffset);
-        } else {
+        int64_t nTimeOffset = nTime - currentTime;
+        pfrom.nTimeOffset = nTimeOffset;
+        // Ignore time offsets that are improbable (before the Genesis block)
+        // and may underflow our adjusted time.
+        // Don't use timedata samples from inbound peers to make it
+        // harder for others to tamper with our adjusted time.
+        if (nTime < int64_t(m_chainparams.GenesisBlock().nTime)) {
             Misbehaving(pfrom, 20,
                         "Ignoring invalid timestamp in version message");
+        } else if (!pfrom.IsInboundConn()) {
+            AddTimeData(pfrom.addr, nTimeOffset);
         }
 
         // Feeler connections exist only to verify if address is online.
diff --git a/test/functional/abc-version-message.py b/test/functional/abc-version-message.py
--- a/test/functional/abc-version-message.py
+++ b/test/functional/abc-version-message.py
@@ -5,8 +5,8 @@
 """Test time adjustment behavior when receiving a VERSION message.
 
 Messages with a timestamp too far in the past are ignored and discouraged.
-Messages with a timestamp more recent than the genesis block timestamp are used to
-adjust our local time.
+Messages from an outbound peer with a timestamp more recent than the genesis block
+timestamp are used to adjust our local time.
 """
 
 from test_framework.blocktools import TIME_GENESIS_BLOCK
@@ -44,7 +44,11 @@
 
 class VersionMessageTest(BitcoinTestFramework):
     def set_test_params(self):
-        self.num_nodes = 1
+        self.num_nodes = 3
+
+    def setup_network(self):
+        self.setup_nodes()
+        # Don't connect the nodes
 
     def run_test(self):
         self.log.info("Check some invalid timestamp in the version message")
@@ -64,12 +68,20 @@
                 send_version=False)
 
         self.log.info("Check valid side of the timestamp boundary (genesis timestamp)")
+        # Outbound connection: the timestamp is used for our adjusted time
+        self.nodes[1].setmocktime(TIME_GENESIS_BLOCK)
         with self.nodes[0].assert_debug_log(
                 expected_msgs=["Added connection peer=2", "added time data"],
                 unexpected_msgs=["Ignoring invalid timestamp in version message"]):
-            self.nodes[0].add_p2p_connection(
-                ModifiedVersionTimestampP2PInterface(TIME_GENESIS_BLOCK),
-                send_version=False)
+            self.connect_nodes(0, 1)
+
+        # Inbound connection: the timestamp is ignored
+        self.nodes[2].setmocktime(TIME_GENESIS_BLOCK)
+        with self.nodes[0].assert_debug_log(
+                expected_msgs=["Added connection peer=3"],
+                unexpected_msgs=["Ignoring invalid timestamp in version message",
+                                 "added time data"]):
+            self.connect_nodes(2, 0)
 
 
 if __name__ == '__main__':