Page MenuHomePhabricator
Files F14864608

No OneTemporary
Actions

View Options

diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py
index 535d6e856..48605a11e 100755
--- a/contrib/seeds/makeseeds.py
+++ b/contrib/seeds/makeseeds.py
@@ -1,214 +1,270 @@
#!/usr/bin/env python3
# Copyright (c) 2013-2017 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Generate seeds.txt from Pieter's DNS seeder
#
import collections
import dns.resolver
import re
import sys
NSEEDS = 512
MAX_SEEDS_PER_ASN = 2
MIN_BLOCKS = 700000
# These are hosts that have been observed to be behaving strangely (e.g.
# aggressively connecting to every node).
SUSPICIOUS_HOSTS = {
"23.92.36.9", "72.36.89.11", "130.211.129.106", "178.63.107.226",
"83.81.130.26", "88.198.17.7", "148.251.238.178", "176.9.46.6",
"54.173.72.127", "54.174.10.182", "54.183.64.54", "54.194.231.211",
"54.66.214.167", "54.66.220.137", "54.67.33.14", "54.77.251.214",
"54.94.195.96", "54.94.200.247"
}
PATTERN_IPV4 = re.compile(
r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
PATTERN_ONION = re.compile(
r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")
# Used to only select nodes with a user agent string compatible with the
# eCash network.
PATTERN_AGENT = re.compile(r"^(/Bitcoin ABC:0.(26|25).(\d+)\(.+\)/)")
def parseline(line):
sline = line.split()
if len(sline) < 11:
return None
# The user agent is at the end of the line. It may contain space, so we
# concatenate.
for i in range(12, len(sline)):
sline[11] += ' ' + sline[i]
# Remove leftovers
del sline[12:]
m = PATTERN_IPV4.match(sline[0])
sortkey = None
ip = None
if m is None:
m = PATTERN_IPV6.match(sline[0])
if m is None:
m = PATTERN_ONION.match(sline[0])
if m is None:
return None
else:
net = 'onion'
ipstr = sortkey = m.group(1)
port = int(m.group(2))
else:
net = 'ipv6'
# Not interested in localhost
if m.group(1) in ['::']:
return None
ipstr = m.group(1)
# XXX parse IPv6 into number, could use name_to_ipv6 from
# generate-seeds
sortkey = ipstr
port = int(m.group(2))
else:
# Do IPv4 sanity check
ip = 0
for i in range(0, 4):
if int(m.group(i + 2)) < 0 or int(m.group(i + 2)) > 255:
return None
ip = ip + (int(m.group(i + 2)) << (8 * (3 - i)))
if ip == 0:
return None
net = 'ipv4'
sortkey = ip
ipstr = m.group(1)
port = int(m.group(6))
# Skip bad results.
if sline[1] == 0:
return None
# Extract uptime %.
uptime30 = float(sline[7][:-1])
# Extract Unix timestamp of last success.
lastsuccess = int(sline[2])
# Extract protocol version.
version = int(sline[10])
# Extract user agent.
agent = sline[11][1:-1]
# Extract service flags.
service = int(sline[9], 16)
# Extract blocks.
blocks = int(sline[8])
# Construct result.
return {
'net': net,
'ip': ipstr,
'port': port,
'ipnum': ip,
'uptime': uptime30,
'lastsuccess': lastsuccess,
'version': version,
'agent': agent,
'service': service,
'blocks': blocks,
'sortkey': sortkey,
}
+def dedup(ips):
+ '''deduplicate by address,port'''
+ d = {}
+ for ip in ips:
+ d[ip['ip'], ip['port']] = ip
+ return list(d.values())
+
+
def filtermultiport(ips):
'''Filter out hosts with more nodes per IP'''
hist = collections.defaultdict(list)
for ip in ips:
hist[ip['sortkey']].append(ip)
return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]
+
+def lookup_asn(net, ip):
+ '''
+ Look up the asn for an IP (4 or 6) address by querying cymru.com, or None
+ if it could not be found.
+ '''
+ try:
+ if net == 'ipv4':
+ ipaddr = ip
+ prefix = '.origin'
+ else:
+ # http://www.team-cymru.com/IP-ASN-mapping.html
+ # 2001:4860:b002:23::68
+ res = str()
+ # pick the first 4 nibbles
+ for nb in ip.split(':')[:4]:
+ # right padded with '0'
+ for c in nb.zfill(4):
+ # 2001 4860 b002 0023
+ res += c + '.'
+ # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
+ ipaddr = res.rstrip('.')
+ prefix = '.origin6'
+
+ asn = int([x.to_text() for x in dns.resolver.query('.'.join(
+ reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
+ 'TXT').response.answer][0].split('\"')[1].split(' ')[0])
+ return asn
+ except Exception:
+ sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n')
+ return None
+
# Based on Greg Maxwell's seed_filter.py
-def filterbyasn(ips, max_per_asn, max_total):
+def filterbyasn(ips, max_per_asn, max_per_net):
# Sift out ips by type
ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
ips_onion = [ip for ip in ips if ip['net'] == 'onion']
- # Filter IPv46 by ASN
+ # Filter IPv46 by ASN, and limit to max_per_net per network
result = []
- asn_count = {}
+ net_count = collections.defaultdict(int)
+ asn_count = collections.defaultdict(int)
for ip in ips_ipv46:
- if len(result) == max_total:
- break
- try:
- if ip['net'] == 'ipv4':
- ipaddr = ip['ip']
- prefix = '.origin'
- else:
- # http://www.team-cymru.com/IP-ASN-mapping.html
- # 2001:4860:b002:23::68
- res = str()
- # pick the first 4 nibbles
- for nb in ip['ip'].split(':')[:4]:
- # right padded with '0'
- for c in nb.zfill(4):
- # 2001 4860 b002 0023
- res += c + '.'
- # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
- ipaddr = res.rstrip('.')
- prefix = '.origin6'
-
- asn = int([x.to_text() for x in dns.resolver.query('.'.join(
- reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
- 'TXT').response.answer][0].split('\"')[1].split(' ')[0])
- if asn not in asn_count:
- asn_count[asn] = 0
- if asn_count[asn] == max_per_asn:
- continue
- asn_count[asn] += 1
- result.append(ip)
- except Exception:
- sys.stderr.write(
- 'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')
-
- # Add back Onions
- result.extend(ips_onion)
+ if net_count[ip['net']] == max_per_net:
+ continue
+ asn = lookup_asn(ip['net'], ip['ip'])
+ if asn is None or asn_count[asn] == max_per_asn:
+ continue
+ asn_count[asn] += 1
+ net_count[ip['net']] += 1
+ result.append(ip)
+
+ # Add back Onions (up to max_per_net)
+ result.extend(ips_onion[0:max_per_net])
return result
+def ip_stats(ips):
+ hist = collections.defaultdict(int)
+ for ip in ips:
+ if ip is not None:
+ hist[ip['net']] += 1
+
+ return '{:6d} {:6d} {:6d}'.format(
+ hist['ipv4'], hist['ipv6'], hist['onion'])
+
+
def main():
lines = sys.stdin.readlines()
ips = [parseline(line) for line in lines]
- # Skip entries with valid address.
+ print(
+ '\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m',
+ file=sys.stderr)
+ print('{} Initial'.format(ip_stats(ips)), file=sys.stderr)
+ # Skip entries with invalid address.
ips = [ip for ip in ips if ip is not None]
+ print(
+ '{} Skip entries with invalid address'.format(ip_stats(ips)), file=sys.stderr)
+ # Skip duplicates (in case multiple seeds files were concatenated)
+ ips = dedup(ips)
+ print(
+ '{} After removing duplicates'.format(
+ ip_stats(ips)),
+ file=sys.stderr)
# Skip entries from suspicious hosts.
ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
+ print(
+ '{} Skip entries from suspicious hosts'.format(ip_stats(ips)), file=sys.stderr)
# Enforce minimal number of blocks.
ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
+ print(
+ '{} Enforce minimal number of blocks'.format(
+ ip_stats(ips)),
+ file=sys.stderr)
# Require service bit 1.
ips = [ip for ip in ips if (ip['service'] & 1) == 1]
- # Require at least 50% 30-day uptime.
- ips = [ip for ip in ips if ip['uptime'] > 50]
+ print('{} Require service bit 1'.format(ip_stats(ips)), file=sys.stderr)
+ # Require at least 50% 30-day uptime for clearnet, 10% for onion.
+ req_uptime = {
+ 'ipv4': 50,
+ 'ipv6': 50,
+ 'onion': 10,
+ }
+ ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
+ print('{} Require minimum uptime'.format(ip_stats(ips)), file=sys.stderr)
# Require a known and recent user agent.
ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
-
+ print(
+ '{} Require a known and recent user agent'.format(ip_stats(ips)), file=sys.stderr)
# Sort by availability (and use last success as tie breaker)
ips.sort(key=lambda x:
(x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
# Filter out hosts with multiple bitcoin ports, these are likely abusive
ips = filtermultiport(ips)
+ print(
+ '{} Filter out hosts with multiple bitcoin ports'.format(ip_stats(ips)), file=sys.stderr)
# Look up ASNs and limit results, both per ASN and globally.
ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
+ print(
+ '{} Look up ASNs and limit results per ASN and per net'.format(ip_stats(ips)), file=sys.stderr)
# Sort the results by IP address (for deterministic output).
ips.sort(key=lambda x: (x['net'], x['sortkey']))
-
for ip in ips:
if ip['net'] == 'ipv6':
print('[{}]:{}'.format(ip['ip'], ip['port']))
else:
print('{}:{}'.format(ip['ip'], ip['port']))
if __name__ == '__main__':
main()

File Metadata

Mime Type
text/x-diff
Expires
Wed, May 21, 20:55 (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
5865923
Default Alt Text
(10 KB)

Event Timeline