diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -119,6 +119,14 @@ } +def dedup(ips): + '''deduplicate by address,port''' + d = {} + for ip in ips: + d[ip['ip'], ip['port']] = ip + return list(d.values()) + + def filtermultiport(ips): '''Filter out hosts with more nodes per IP''' hist = collections.defaultdict(list) @@ -126,83 +134,131 @@ hist[ip['sortkey']].append(ip) return [value[0] for (key, value) in list(hist.items()) if len(value) == 1] + +def lookup_asn(net, ip): + ''' + Look up the asn for an IP (4 or 6) address by querying cymru.com, or None + if it could not be found. + ''' + try: + if net == 'ipv4': + ipaddr = ip + prefix = '.origin' + else: + # http://www.team-cymru.com/IP-ASN-mapping.html + # 2001:4860:b002:23::68 + res = str() + # pick the first 4 nibbles + for nb in ip.split(':')[:4]: + # right padded with '0' + for c in nb.zfill(4): + # 2001 4860 b002 0023 + res += c + '.' + # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 + ipaddr = res.rstrip('.') + prefix = '.origin6' + + asn = int([x.to_text() for x in dns.resolver.query('.'.join( + reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', + 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) + return asn + except Exception: + sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n') + return None + # Based on Greg Maxwell's seed_filter.py -def filterbyasn(ips, max_per_asn, max_total): +def filterbyasn(ips, max_per_asn, max_per_net): # Sift out ips by type ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] - # Filter IPv46 by ASN + # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - asn_count = {} + net_count = collections.defaultdict(int) + asn_count = collections.defaultdict(int) for ip in ips_ipv46: - if len(result) == max_total: - break - try: - if ip['net'] == 'ipv4': - ipaddr = ip['ip'] - prefix = '.origin' - else: - # http://www.team-cymru.com/IP-ASN-mapping.html - # 2001:4860:b002:23::68 - res = str() - # pick the first 4 nibbles - for nb in ip['ip'].split(':')[:4]: - # right padded with '0' - for c in nb.zfill(4): - # 2001 4860 b002 0023 - res += c + '.' - # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 - ipaddr = res.rstrip('.') - prefix = '.origin6' - - asn = int([x.to_text() for x in dns.resolver.query('.'.join( - reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', - 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) - if asn not in asn_count: - asn_count[asn] = 0 - if asn_count[asn] == max_per_asn: - continue - asn_count[asn] += 1 - result.append(ip) - except Exception: - sys.stderr.write( - 'ERR: Could not resolve ASN for "' + ip['ip'] + '"\n') - - # Add back Onions - result.extend(ips_onion) + if net_count[ip['net']] == max_per_net: + continue + asn = lookup_asn(ip['net'], ip['ip']) + if asn is None or asn_count[asn] == max_per_asn: + continue + asn_count[asn] += 1 + net_count[ip['net']] += 1 + result.append(ip) + + # Add back Onions (up to max_per_net) + result.extend(ips_onion[0:max_per_net]) return result +def ip_stats(ips): + hist = collections.defaultdict(int) + for ip in ips: + if ip is not None: + hist[ip['net']] += 1 + + return '{:6d} {:6d} {:6d}'.format( + hist['ipv4'], hist['ipv6'], hist['onion']) + + def main(): lines = sys.stdin.readlines() ips = [parseline(line) for line in lines] - # Skip entries with valid address. + print( + '\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', + file=sys.stderr) + print('{} Initial'.format(ip_stats(ips)), file=sys.stderr) + # Skip entries with invalid address. ips = [ip for ip in ips if ip is not None] + print( + '{} Skip entries with invalid address'.format(ip_stats(ips)), file=sys.stderr) + # Skip duplicates (in case multiple seeds files were concatenated) + ips = dedup(ips) + print( + '{} After removing duplicates'.format( + ip_stats(ips)), + file=sys.stderr) # Skip entries from suspicious hosts. ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] + print( + '{} Skip entries from suspicious hosts'.format(ip_stats(ips)), file=sys.stderr) # Enforce minimal number of blocks. ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] + print( + '{} Enforce minimal number of blocks'.format( + ip_stats(ips)), + file=sys.stderr) # Require service bit 1. ips = [ip for ip in ips if (ip['service'] & 1) == 1] - # Require at least 50% 30-day uptime. - ips = [ip for ip in ips if ip['uptime'] > 50] + print('{} Require service bit 1'.format(ip_stats(ips)), file=sys.stderr) + # Require at least 50% 30-day uptime for clearnet, 10% for onion. + req_uptime = { + 'ipv4': 50, + 'ipv6': 50, + 'onion': 10, + } + ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] + print('{} Require minimum uptime'.format(ip_stats(ips)), file=sys.stderr) # Require a known and recent user agent. ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] - + print( + '{} Require a known and recent user agent'.format(ip_stats(ips)), file=sys.stderr) # Sort by availability (and use last success as tie breaker) ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) # Filter out hosts with multiple bitcoin ports, these are likely abusive ips = filtermultiport(ips) + print( + '{} Filter out hosts with multiple bitcoin ports'.format(ip_stats(ips)), file=sys.stderr) # Look up ASNs and limit results, both per ASN and globally. ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) + print( + '{} Look up ASNs and limit results per ASN and per net'.format(ip_stats(ips)), file=sys.stderr) # Sort the results by IP address (for deterministic output). ips.sort(key=lambda x: (x['net'], x['sortkey'])) - for ip in ips: if ip['net'] == 'ipv6': print('[{}]:{}'.format(ip['ip'], ip['port']))