Differential D12138 Diff 35416 contrib/seeds/makeseeds.py

Changeset View

Standalone View

contrib/seeds/makeseeds.py

Show First 20 Lines • Show All 113 Lines • ▼ Show 20 Lines	return {
'version': version,		'version': version,
'agent': agent,		'agent': agent,
'service': service,		'service': service,
'blocks': blocks,		'blocks': blocks,
'sortkey': sortkey,		'sortkey': sortkey,
}		}


		def dedup(ips):
		'''deduplicate by address,port'''
		d = {}
		for ip in ips:
		d[ip['ip'], ip['port']] = ip
		return list(d.values())


def filtermultiport(ips):		def filtermultiport(ips):
'''Filter out hosts with more nodes per IP'''		'''Filter out hosts with more nodes per IP'''
hist = collections.defaultdict(list)		hist = collections.defaultdict(list)
for ip in ips:		for ip in ips:
hist[ip['sortkey']].append(ip)		hist[ip['sortkey']].append(ip)
return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]		return [value[0] for (key, value) in list(hist.items()) if len(value) == 1]

# Based on Greg Maxwell's seed_filter.py


def filterbyasn(ips, max_per_asn, max_total):
# Sift out ips by type
ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
ips_onion = [ip for ip in ips if ip['net'] == 'onion']

# Filter IPv46 by ASN		def lookup_asn(net, ip):
result = []		'''
asn_count = {}		Look up the asn for an IP (4 or 6) address by querying cymru.com, or None
for ip in ips_ipv46:		if it could not be found.
if len(result) == max_total:		'''
break
try:		try:
if ip['net'] == 'ipv4':		if net == 'ipv4':
ipaddr = ip['ip']		ipaddr = ip
prefix = '.origin'		prefix = '.origin'
else:		else:
# http://www.team-cymru.com/IP-ASN-mapping.html		# http://www.team-cymru.com/IP-ASN-mapping.html
# 2001:4860:b002:23::68		# 2001:4860:b002:23::68
res = str()		res = str()
# pick the first 4 nibbles		# pick the first 4 nibbles
for nb in ip['ip'].split(':')[:4]:		for nb in ip.split(':')[:4]:
# right padded with '0'		# right padded with '0'
for c in nb.zfill(4):		for c in nb.zfill(4):
# 2001 4860 b002 0023		# 2001 4860 b002 0023
res += c + '.'		res += c + '.'
# 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3		# 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
ipaddr = res.rstrip('.')		ipaddr = res.rstrip('.')
prefix = '.origin6'		prefix = '.origin6'

asn = int([x.to_text() for x in dns.resolver.query('.'.join(		asn = int([x.to_text() for x in dns.resolver.query('.'.join(
reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',		reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
'TXT').response.answer][0].split('\"')[1].split(' ')[0])		'TXT').response.answer][0].split('\"')[1].split(' ')[0])
if asn not in asn_count:		return asn
asn_count[asn] = 0		except Exception:
if asn_count[asn] == max_per_asn:		sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n')
		return None

		# Based on Greg Maxwell's seed_filter.py


		def filterbyasn(ips, max_per_asn, max_per_net):
		# Sift out ips by type
		ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
		ips_onion = [ip for ip in ips if ip['net'] == 'onion']

		# Filter IPv46 by ASN, and limit to max_per_net per network
		result = []
		net_count = collections.defaultdict(int)
		asn_count = collections.defaultdict(int)
		for ip in ips_ipv46:
		if net_count[ip['net']] == max_per_net:
		continue
		asn = lookup_asn(ip['net'], ip['ip'])
		if asn is None or asn_count[asn] == max_per_asn:
continue		continue
asn_count[asn] += 1		asn_count[asn] += 1
		net_count[ip['net']] += 1
result.append(ip)		result.append(ip)
except Exception:
sys.stderr.write(
'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n')

# Add back Onions		# Add back Onions (up to max_per_net)
result.extend(ips_onion)		result.extend(ips_onion[0:max_per_net])
return result		return result


		def ip_stats(ips):
		hist = collections.defaultdict(int)
		for ip in ips:
		if ip is not None:
		hist[ip['net']] += 1

		return '{:6d} {:6d} {:6d}'.format(
		hist['ipv4'], hist['ipv6'], hist['onion'])


def main():		def main():
lines = sys.stdin.readlines()		lines = sys.stdin.readlines()
ips = [parseline(line) for line in lines]		ips = [parseline(line) for line in lines]

# Skip entries with valid address.		print(
		'\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m',
		file=sys.stderr)
		print('{} Initial'.format(ip_stats(ips)), file=sys.stderr)
		# Skip entries with invalid address.
ips = [ip for ip in ips if ip is not None]		ips = [ip for ip in ips if ip is not None]
		print(
		'{} Skip entries with invalid address'.format(ip_stats(ips)), file=sys.stderr)
		# Skip duplicates (in case multiple seeds files were concatenated)
		ips = dedup(ips)
		print(
		'{} After removing duplicates'.format(
		ip_stats(ips)),
		file=sys.stderr)
# Skip entries from suspicious hosts.		# Skip entries from suspicious hosts.
ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]		ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
		print(
		'{} Skip entries from suspicious hosts'.format(ip_stats(ips)), file=sys.stderr)
# Enforce minimal number of blocks.		# Enforce minimal number of blocks.
ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]		ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
		print(
		'{} Enforce minimal number of blocks'.format(
		ip_stats(ips)),
		file=sys.stderr)
# Require service bit 1.		# Require service bit 1.
ips = [ip for ip in ips if (ip['service'] & 1) == 1]		ips = [ip for ip in ips if (ip['service'] & 1) == 1]
# Require at least 50% 30-day uptime.		print('{} Require service bit 1'.format(ip_stats(ips)), file=sys.stderr)
ips = [ip for ip in ips if ip['uptime'] > 50]		# Require at least 50% 30-day uptime for clearnet, 10% for onion.
		req_uptime = {
		'ipv4': 50,
		'ipv6': 50,
		'onion': 10,
		}
		ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
		print('{} Require minimum uptime'.format(ip_stats(ips)), file=sys.stderr)
# Require a known and recent user agent.		# Require a known and recent user agent.
ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]		ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
		print(
		'{} Require a known and recent user agent'.format(ip_stats(ips)), file=sys.stderr)
# Sort by availability (and use last success as tie breaker)		# Sort by availability (and use last success as tie breaker)
ips.sort(key=lambda x:		ips.sort(key=lambda x:
(x['uptime'], x['lastsuccess'], x['ip']), reverse=True)		(x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
# Filter out hosts with multiple bitcoin ports, these are likely abusive		# Filter out hosts with multiple bitcoin ports, these are likely abusive
ips = filtermultiport(ips)		ips = filtermultiport(ips)
		print(
		'{} Filter out hosts with multiple bitcoin ports'.format(ip_stats(ips)), file=sys.stderr)
# Look up ASNs and limit results, both per ASN and globally.		# Look up ASNs and limit results, both per ASN and globally.
ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)		ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
		print(
		'{} Look up ASNs and limit results per ASN and per net'.format(ip_stats(ips)), file=sys.stderr)
# Sort the results by IP address (for deterministic output).		# Sort the results by IP address (for deterministic output).
ips.sort(key=lambda x: (x['net'], x['sortkey']))		ips.sort(key=lambda x: (x['net'], x['sortkey']))

for ip in ips:		for ip in ips:
if ip['net'] == 'ipv6':		if ip['net'] == 'ipv6':
print('[{}]:{}'.format(ip['ip'], ip['port']))		print('[{}]:{}'.format(ip['ip'], ip['port']))
else:		else:
print('{}:{}'.format(ip['ip'], ip['port']))		print('{}:{}'.format(ip['ip'], ip['port']))


if __name__ == '__main__':		if __name__ == '__main__':
main()		main()