Back to Article
Public IP enrichment: worked example
Download Notebook

Public IP enrichment: worked example

Companion notebook for the site page Public IP enrichment: what the flags actually mean. The implementation lives in openbotrisk.ip_enrichment; this notebook loads existing immutable raw snapshots, measures coverage, and demonstrates the stable output schema.

True means matched, False means checked and not matched, and NA means the source could not be applied. This notebook never downloads data: execution and site rendering use the named local snapshots only.

In [1]:
import json
from datetime import UTC, datetime
from ipaddress import ip_network
from pathlib import Path

import pandas as pd

from openbotrisk.ip_enrichment import (
    ASdbTable,
    IPinfoTable,
    IntervalSet,
    ListedIPs,
    PeeringDBTable,
    ProviderTable,
    RouteTable,
    artifact_provenance,
    enrich,
    matching_manifest_record,
)

REPO = Path.cwd().parent if Path.cwd().name == "notebooks" else Path.cwd()
RAW = REPO / "data/raw/ip-enrichment/2026-10-25"
IP_DIR = REPO / "data/IP"
IPTOASN_PATH = IP_DIR / "ip2asn-combined.tsv.gz"
IPINFO_PATH = IP_DIR / "ipinfo_lite.parquet"
ASDB_PATH = IP_DIR / "2026-03_categorized_ases.csv"
PEERINGDB_PATH = REPO / "notebooks/peeringdb-net.json"
SAMPLE_SEED = 20261026
In [2]:
# Execution is intentionally local-only. Missing inputs remain unavailable.
manifest_path = RAW / "manifest.jsonl"
manifest = (
    [json.loads(line) for line in manifest_path.read_text().splitlines()]
    if manifest_path.exists()
    else []
)

def peeringdb_generation_time(path):
    if not path.exists():
        return None
    generated = json.loads(path.read_text()).get("meta", {}).get("generated")
    return (
        datetime.fromtimestamp(generated, UTC).isoformat(timespec="seconds")
        if generated is not None
        else None
    )

source_specs = [
    {"key": "route", "source": "IPtoASN", "path": IPTOASN_PATH, "url": "https://iptoasn.com/data/ip2asn-combined.tsv.gz", "schema": "start, end, ASN, country code, description", "prior_use": "route origin/name/country", "source_updated_at": None},
    {"key": "peeringdb", "source": "PeeringDB", "path": PEERINGDB_PATH, "url": "https://www.peeringdb.com/api/net", "schema": "API network rows: ASN, name, info_types, website, scope, updated", "prior_use": "exploratory inspection only", "source_updated_at": peeringdb_generation_time(PEERINGDB_PATH)},
    {"key": "ipinfo", "source": "IPinfo Lite", "path": IPINFO_PATH, "url": "https://ipinfo.io/developers/ipinfo-lite-database", "schema": "network, country/code, continent/code, ASN, AS name/domain", "prior_use": "exploratory inspection only", "source_updated_at": None},
    {"key": "providers", "source": "rezmoss aggregate", "path": RAW / "rezmoss_all_providers.json", "url": "https://raw.githubusercontent.com/rezmoss/cloud-provider-ip-addresses/main/all_providers/all_providers.json", "schema": "CIDR, IP version, provider, service, region, last_updated", "prior_use": "provider name only", "source_updated_at": None},
    {"key": "asdb", "source": "Stanford ASdb", "path": ASDB_PATH, "url": "https://asdb.stanford.edu/about", "schema": "ASN plus numbered layer-1/layer-2 category pairs", "prior_use": "exploratory merge only", "source_updated_at": None},
    {"key": "vpn", "source": "X4BNet VPN IPv4", "path": RAW / "x4bnet_vpn_ipv4.txt", "url": "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/vpn/ipv4.txt", "schema": "newline-delimited IPv4 CIDRs", "prior_use": "VPN range flag", "source_updated_at": None},
    {"key": "datacentre", "source": "X4BNet datacentre IPv4", "path": RAW / "x4bnet_datacenter_ipv4.txt", "url": "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/datacenter/ipv4.txt", "schema": "newline-delimited IPv4 CIDRs", "prior_use": "datacentre range flag", "source_updated_at": None},
    {"key": "bad_asn", "source": "bad-asn-list", "path": RAW / "bad_asn_list.csv", "url": "https://raw.githubusercontent.com/brianhama/bad-asn-list/master/bad-asn-list.csv", "schema": "ASN and maintainer label CSV", "prior_use": "hosting-ASN heuristic", "source_updated_at": None},
    {"key": "tor", "source": "Tor bulk exit list", "path": RAW / "tor_bulk_exit.txt", "url": "https://check.torproject.org/torbulkexitlist", "schema": "newline-delimited exit IP addresses", "prior_use": "dated exact-address flag", "source_updated_at": None},
]

def source_record(spec):
    path = spec["path"]
    record = {**spec, "path": str(path.relative_to(REPO)), "local_path": path}
    if not path.exists():
        return {**record, "status": "unavailable", "reason": "local artifact not found", "retrieved_at": None, "file_mtime": None, "sha256": None}
    item = artifact_provenance(path.name, spec["url"], path, source_updated_at=spec["source_updated_at"])
    evidence = matching_manifest_record(manifest, path.name, item.sha256)
    return {
        **record,
        "status": "available",
        "reason": None,
        "retrieved_at": evidence.get("retrieved_at") if evidence else None,
        "source_updated_at": spec["source_updated_at"] or (evidence.get("source_updated_at") if evidence else None),
        "file_mtime": item.file_mtime,
        "sha256": item.sha256,
    }

source_records = [source_record(spec) for spec in source_specs]
records_by_key = {record["key"]: record for record in source_records}

def available_path(key):
    record = records_by_key[key]
    return record["local_path"] if record["status"] == "available" else None
In [3]:
vpn_path = available_path("vpn")
datacentre_path = available_path("datacentre")
providers_path = available_path("providers")
bad_asn_path = available_path("bad_asn")
tor_path = available_path("tor")
route_path = available_path("route")
peeringdb_path = available_path("peeringdb")
ipinfo_path = available_path("ipinfo")
asdb_path = available_path("asdb")

vpn = IntervalSet.from_file(vpn_path) if vpn_path is not None else None
datacentre = IntervalSet.from_file(datacentre_path) if datacentre_path is not None else None
providers = pd.read_json(providers_path, convert_dates=False) if providers_path is not None else None
cloud = ProviderTable(providers) if providers is not None else None

bad_asn = (
    pd.read_csv(bad_asn_path).drop_duplicates("ASN")
    if bad_asn_path is not None
    else None
)
bad_asns = set(bad_asn["ASN"]) if bad_asn is not None else None

tor = ListedIPs.from_file(tor_path) if tor_path is not None else None
route = RouteTable.from_tsv(route_path) if route_path is not None else None
peeringdb = PeeringDBTable.from_json(peeringdb_path) if peeringdb_path is not None else None
ipinfo = IPinfoTable.from_parquet(ipinfo_path) if ipinfo_path is not None else None
asdb = ASdbTable.from_csv(asdb_path) if asdb_path is not None else None
loaded_sources = {
    "route": route, "peeringdb": peeringdb, "ipinfo": ipinfo, "providers": cloud,
    "asdb": asdb, "vpn": vpn, "datacentre": datacentre, "bad_asn": bad_asns, "tor": tor,
}

Local source inventory

This inventory is generated from the same provenance records used to load the sources. Integrated means the source participates in enrich() in this run; it does not imply complete coverage. Retrieval, source publication/generation, and filesystem modification times are separate. A manifest retrieval time appears only when its recorded SHA-256 matches the current local bytes. Unknown dates remain unknown.

In [4]:
inventory_rows = []
for record in source_records:
    loaded = loaded_sources[record["key"]] is not None
    inventory_rows.append({
        "source": record["source"],
        "path": record["path"],
        "schema": record["schema"],
        "status": "integrated" if loaded else "unavailable",
        "reason": None if loaded else record["reason"] or "loader unavailable",
        "retrieved_at": record["retrieved_at"],
        "source_updated_at": record["source_updated_at"],
        "file_mtime": record["file_mtime"],
        "prior_use": record["prior_use"],
    })
source_inventory = pd.DataFrame(inventory_rows)
source_inventory
source path schema status reason retrieved_at source_updated_at file_mtime prior_use
0 IPtoASN data/IP/ip2asn-combined.tsv.gz start, end, ASN, country code, description integrated None None None 2026-10-24T12:38:06+00:00 route origin/name/country
1 PeeringDB notebooks/peeringdb-net.json API network rows: ASN, name, info_types, websi... integrated None None 2026-10-24T16:21:34+00:00 2026-10-24T19:41:12+00:00 exploratory inspection only
2 IPinfo Lite data/IP/ipinfo_lite.parquet network, country/code, continent/code, ASN, AS... integrated None None None 2026-10-24T13:02:50+00:00 exploratory inspection only
3 rezmoss aggregate data/raw/ip-enrichment/2026-10-25/rezmoss_all_... CIDR, IP version, provider, service, region, l... integrated None 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 provider name only
4 Stanford ASdb data/IP/2026-03_categorized_ases.csv ASN plus numbered layer-1/layer-2 category pairs integrated None None None 2026-10-24T12:39:51+00:00 exploratory merge only
5 X4BNet VPN IPv4 data/raw/ip-enrichment/2026-10-25/x4bnet_vpn_i... newline-delimited IPv4 CIDRs integrated None 2026-10-25T10:54:43+00:00 None 2026-10-25T10:54:43+00:00 VPN range flag
6 X4BNet datacentre IPv4 data/raw/ip-enrichment/2026-10-25/x4bnet_datac... newline-delimited IPv4 CIDRs integrated None 2026-10-25T10:54:43+00:00 None 2026-10-25T10:54:43+00:00 datacentre range flag
7 bad-asn-list data/raw/ip-enrichment/2026-10-25/bad_asn_list... ASN and maintainer label CSV integrated None 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 hosting-ASN heuristic
8 Tor bulk exit list data/raw/ip-enrichment/2026-10-25/tor_bulk_exi... newline-delimited exit IP addresses integrated None 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 dated exact-address flag

Coverage and dependence

In [5]:
cloud_any = None
if providers is not None:
    provider_v4 = providers.loc[providers["ip_version"] == "IPv4", "cidr"]
    cloud_any = IntervalSet(ip_network(cidr, strict=False) for cidr in provider_v4)

coverage_rows = []
for name, value, reason in [
    ("X4BNet VPN", vpn, "X4BNet VPN source unavailable"),
    ("X4BNet datacentre", datacentre, "X4BNet datacentre source unavailable"),
    ("rezmoss provider union (IPv4)", cloud_any, "rezmoss provider source unavailable"),
]:
    if value is None:
        coverage_rows.append({"source": name, "networks": pd.NA, "addresses": pd.NA, "pct_of_ipv4": pd.NA, "status": "unavailable", "reason": reason})
    else:
        coverage_rows.append({"source": name, "networks": len(value), "addresses": value.addresses, "pct_of_ipv4": round(value.addresses / 2**32 * 100, 2), "status": "available", "reason": None})
coverage = pd.DataFrame(coverage_rows).set_index("source")
coverage
networks addresses pct_of_ipv4 status reason
source
X4BNet VPN 10931 3170072 0.07 available None
X4BNet datacentre 44584 395597872 9.21 available None
rezmoss provider union (IPv4) 12842 255279702 5.94 available None 
In [6]:
def overlap_diagnostic(name, left, right, left_name, right_name):
    missing = [label for value, label in ((left, left_name), (right, right_name)) if value is None]
    if missing:
        return {"comparison": name, "overlap_addresses": pd.NA, "denominator_addresses": pd.NA, "overlap_share": pd.NA, "status": "unavailable", "reason": f"missing {', '.join(missing)}"}
    if left.addresses == 0:
        return {"comparison": name, "overlap_addresses": pd.NA, "denominator_addresses": 0, "overlap_share": pd.NA, "status": "unavailable", "reason": "zero-address denominator"}
    count = left.overlap_addresses(right)
    return {"comparison": name, "overlap_addresses": count, "denominator_addresses": left.addresses, "overlap_share": count / left.addresses, "status": "available", "reason": None}

overlap = pd.DataFrame([
    overlap_diagnostic("provider union also in X4BNet datacentre", cloud_any, datacentre, "rezmoss provider source", "X4BNet datacentre source"),
    overlap_diagnostic("X4BNet VPN also in X4BNet datacentre", vpn, datacentre, "X4BNet VPN source", "X4BNet datacentre source"),
]).set_index("comparison")
overlap
overlap_addresses denominator_addresses overlap_share status reason
comparison
provider union also in X4BNet datacentre 234191767 255279702 0.917393 available None
X4BNet VPN also in X4BNet datacentre 3145854 3170072 0.992360 available None

Enrich some addresses

The real snapshot examples include Janet/Jisc (AS786), whose PeeringDB record reports Educational/Research, Andrews & Arnold (AS20712), whose record reports Cable/DSL/ISP, and Amazon.com (AS16509), whose record reports Enterprise. These are network-level, self-reported classifications. Educational context does not show how many users share an IP, and Enterprise is not a reliable shared-office indicator—as Amazon’s classification illustrates. None establishes that a particular user is a student, employee, residential subscriber, or behind NAT. The examples also include IPv6, IPv4-mapped IPv6, malformed input, and null input so the distinction between False and NA is visible.

In [7]:
demo = [
    "45.88.80.1",       # Route AS786; PeeringDB reports Educational/Research
    "1.44.96.1",        # Route AS16509; PeeringDB reports Enterprise
    "52.95.110.1",      # AWS range with detailed provider metadata
    "8.8.8.8",          # Google resolver; infrastructure is not an abuse verdict
    "81.2.69.142",      # AS20712; PeeringDB reports Cable/DSL/ISP
    "2606:4700:4700::1111", # Real IPv6; IPinfo/provider support, route table does not
    "2001:db8::1",      # Documentation IPv6; no real lookup claim
    "::ffff:34.82.1.5", # Deliberately normalized to IPv4
    "1.2.3.4:443",      # Port attached -> unparseable -> NA
    "not an ip",
    None,
]
In [8]:
out = enrich(
    demo,
    route=route,
    cloud=cloud,
    peeringdb=peeringdb,
    ipinfo=ipinfo,
    asdb=asdb,
    tor=tor,
    vpn=vpn,
    datacentre=datacentre,
    bad_asns=bad_asns,
)
out[[
    "ip_input", "route_asn", "ipinfo_asn", "route_ipinfo_asn_agree",
    "peeringdb_name", "peeringdb_info_types",
    "education_network_reported", "enterprise_network_reported",
    "cable_dsl_isp_network_reported", "cloud_matches",
]]
ip_input route_asn ipinfo_asn route_ipinfo_asn_agree peeringdb_name peeringdb_info_types education_network_reported enterprise_network_reported cable_dsl_isp_network_reported cloud_matches
0 45.88.80.1 786 786 True Janet (Educational/Research,) True False False ()
1 1.44.96.1 16509 16509 True Amazon.com (Enterprise,) False True False ()
2 52.95.110.1 16509 16509 True Amazon.com (Enterprise,) False True False (aws,)
3 8.8.8.8 15169 15169 True Google LLC (Content,) False False False (googleservices,)
4 81.2.69.142 20712 20712 True Andrews & Arnold Ltd (Cable/DSL/ISP,) False False True ()
5 2606:4700:4700::1111 <NA> 13335 <NA> <NA> <NA> <NA> <NA> <NA> (cloudflare,)
6 2001:db8::1 <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> ()
7 ::ffff:34.82.1.5 396982 396982 True Google Cloud () False False False (googlecloud,)
8 1.2.3.4:443 <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>
9 not an ip <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>
10 None <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA> <NA>

cloud_matches remains a tuple of provider names for compatibility. cloud_match_details keeps CIDR, provider, service, region and source update together for every matching row, with exact duplicates removed and overlaps retained. Provider region describes the published infrastructure range, not necessarily the user’s location. No relay status is inferred from a generic cloud match. NA means the source could not be applied.

In [9]:
out[[
    "ip_input", "ipinfo_country", "ipinfo_continent", "ipinfo_as_name",
    "ipinfo_as_domain", "cloud_match_details", "asdb_category_pairs",
]]
ip_input ipinfo_country ipinfo_continent ipinfo_as_name ipinfo_as_domain cloud_match_details asdb_category_pairs
0 45.88.80.1 United Kingdom Europe Jisc Services Limited jisc.ac.uk () ({'layer1': 'Education and Research', 'layer2'...
1 1.44.96.1 Australia Oceania Amazon.com, Inc. amazon.com () ({'layer1': 'Computer and Information Technolo...
2 52.95.110.1 United States North America Amazon.com, Inc. amazon.com ({'cidr': '52.95.110.0/24', 'provider': 'aws',... ({'layer1': 'Computer and Information Technolo...
3 8.8.8.8 United States North America Google LLC google.com ({'cidr': '8.8.8.0/24', 'provider': 'googleser... ({'layer1': 'Service', 'layer2': None}, {'laye...
4 81.2.69.142 United Kingdom Europe Andrews & Arnold Ltd aa.net.uk () ({'layer1': 'Computer and Information Technolo...
5 2606:4700:4700::1111 United States North America Cloudflare, Inc. cloudflare.com ({'cidr': '2606:4700::/32', 'provider': 'cloud... <NA>
6 2001:db8::1 <NA> <NA> <NA> <NA> () <NA>
7 ::ffff:34.82.1.5 United States North America Google LLC google.com ({'cidr': '34.82.0.0/15', 'provider': 'googlec... ({'layer1': 'Service', 'layer2': None}, {'laye...
8 1.2.3.4:443 <NA> <NA> <NA> <NA> <NA> <NA>
9 not an ip <NA> <NA> <NA> <NA> <NA> <NA>
10 None <NA> <NA> <NA> <NA> <NA> <NA> 
In [10]:
# Report this share next to any flag rate.
out.drop(columns=["ip_input"]).isna().mean().rename("NA share").round(2)
ip_normalized                     0.27
ip_version                        0.27
route_asn                         0.45
asn_name                          0.45
asn_country                       0.45
cloud_matches                     0.27
cloud_match_details               0.27
ipinfo_country                    0.36
ipinfo_continent                  0.36
ipinfo_asn                        0.36
ipinfo_as_name                    0.36
ipinfo_as_domain                  0.36
route_ipinfo_asn_agree            0.45
peeringdb_name                    0.45
peeringdb_info_types              0.45
peeringdb_website                 0.45
peeringdb_info_scope              0.45
peeringdb_updated                 0.45
education_network_reported        0.45
enterprise_network_reported       0.45
cable_dsl_isp_network_reported    0.45
asdb_category_pairs               0.45
asdb_layer1_categories            0.45
asdb_layer2_categories            0.45
tor_exit_listed                   0.45
vpn_listed                        0.45
datacentre_listed                 0.45
hosting_asn_listed                0.45
Name: NA share, dtype: float64

Variable dictionary

Field Meaning Source Missing-value semantics Main limitation
ip_input Original submitted value Event input Preserved null input May not be a parseable address
ip_normalized, ip_version Canonical address and family Parser NA when parsing fails IPv4-mapped IPv6 is deliberately normalized to IPv4
route_asn, asn_name, asn_country Origin ASN and route-table descriptions IPtoASN snapshot NA for invalid, IPv6, ASN 0, no route, or absent source Route origin is not end-user identity; asn_country is not IP geolocation
ipinfo_country, ipinfo_continent IP range geolocation labels IPinfo Lite NA for no matching row or absent source Approximate IP-range geography, not user location or ASN country
ipinfo_asn, ipinfo_as_name, ipinfo_as_domain IPinfo’s ASN assignment and AS metadata IPinfo Lite NA when the row or field is missing Kept separate from route origin
route_ipinfo_asn_agree Whether normalized route and IPinfo ASNs agree Derived comparison NA unless both ASNs exist Disagreement identifies source difference, not which source is correct
peeringdb_name, peeringdb_website, peeringdb_info_scope, peeringdb_updated Network record metadata PeeringDB joined on route_asn NA for no active record or absent field/source Community-maintained organisational context
peeringdb_info_types Every self-reported network type PeeringDB NA for no record/unavailable type; empty tuple for known empty list Self-reported, non-exclusive and incomplete
education_network_reported, enterprise_network_reported, cable_dsl_isp_network_reported Membership in three reported PeeringDB categories PeeringDB NA if record/types unavailable; False only for a known type list lacking the category Not proof of users, purpose, NAT, safety or expected usage
cloud_matches Compatible tuple of all matching provider names rezmoss aggregate Empty tuple means checked/no match; NA means unsupported/unavailable Third-party aggregation; overlapping labels are retained
cloud_match_details Matching CIDR/provider/service/region/source-update records rezmoss aggregate Empty tuple means checked/no match; NA means unsupported/unavailable Region belongs to the published range, not necessarily the user; no generic relay inference
asdb_category_pairs, asdb_layer1_categories, asdb_layer2_categories All supplied industry category pairs and convenience projections Stanford ASdb joined on route_asn NA for no ASN row/source Inferred classifications; disagreements with PeeringDB are not reconciled
tor_exit_listed Exact address present in the dated Tor exit artifact Tor bulk list NA when family/source unavailable Dated shared-infrastructure observation, not abuse evidence
vpn_listed, datacentre_listed Match in separate community IPv4 range lists X4BNet NA for IPv6/invalid/absent source Broad heuristic coverage; not independent labels
hosting_asn_listed Route ASN present in curated hosting-ASN list bad-asn-list NA without route/source Maintainer heuristic, not ground truth

Classification completeness and ASN agreement

The ASN tables use distinct non-zero IPtoASN route ASNs as their denominator. The GB subset means ASNs attached to at least one route row whose IPtoASN country field is exactly GB; it is not a census of networks serving UK users. The IP-level tables use fixed-seed samples across all eligible IPinfo IPv4 rows and, separately, rows whose IPinfo country code is GB. IPinfo country is IP-range geolocation and remains distinct from IPtoASN’s ASN country. These are ASN-level and source-row diagnostics, not traffic-, user-, or address-space-weighted estimates.

In [11]:
def route_asn_subset(country=None):
    if route is None:
        return None
    mask = route.asn != 0
    if country is not None:
        mask &= route.country == country
    return pd.Series(pd.array(pd.unique(route.asn[mask]), dtype="Int64"))

def pct(count, denominator):
    return pd.NA if denominator == 0 or pd.isna(count) else count / denominator * 100

def classification_row(scope, asns, source, table, classification_field):
    empty = {"scope": scope, "source": source, "denominator_asns": pd.NA, "no_record": pd.NA, "no_record_pct": pd.NA, "classification_missing_or_empty": pd.NA, "classification_missing_or_empty_pct": pd.NA, "usable_classification": pd.NA, "usable_classification_pct": pd.NA}
    if asns is None:
        return {**empty, "status": "unavailable", "reason": "IPtoASN route source unavailable"}
    denominator = len(asns)
    if table is None:
        return {**empty, "denominator_asns": denominator, "status": "unavailable", "reason": f"{source} source unavailable"}
    if denominator == 0:
        return {**empty, "denominator_asns": 0, "status": "unavailable", "reason": "no eligible route ASNs"}
    present = table.record_presence(asns).eq(True)
    joined = table.lookup(asns)
    usable = joined[classification_field].map(lambda value: value is not pd.NA and value is not None and len(value) > 0)
    no_record = int((~present).sum())
    missing = int((present & ~usable).sum())
    usable_count = int((present & usable).sum())
    return {**empty, "denominator_asns": denominator, "no_record": no_record, "no_record_pct": pct(no_record, denominator), "classification_missing_or_empty": missing, "classification_missing_or_empty_pct": pct(missing, denominator), "usable_classification": usable_count, "usable_classification_pct": pct(usable_count, denominator), "status": "available", "reason": None}

route_asn_scopes = {
    "all distinct non-zero route ASNs": route_asn_subset(),
    "IPtoASN country=GB distinct non-zero route ASNs": route_asn_subset("GB"),
}
asn_classification_coverage = pd.DataFrame([
    classification_row(scope, asns, source, table, field)
    for scope, asns in route_asn_scopes.items()
    for source, table, field in [
        ("PeeringDB", peeringdb, "peeringdb_info_types"),
        ("Stanford ASdb", asdb, "asdb_category_pairs"),
    ]
])
asn_classification_coverage
scope source denominator_asns no_record no_record_pct classification_missing_or_empty classification_missing_or_empty_pct usable_classification usable_classification_pct status reason
0 all distinct non-zero route ASNs PeeringDB 79310 50027 63.077796 8039 10.136175 21244 26.786030 available None
1 all distinct non-zero route ASNs Stanford ASdb 79310 5745 7.243727 0 0.000000 73565 92.756273 available None
2 IPtoASN country=GB distinct non-zero route ASNs PeeringDB 2072 1118 53.957529 217 10.472973 737 35.569498 available None
3 IPtoASN country=GB distinct non-zero route ASNs Stanford ASdb 2072 114 5.501931 0 0.000000 1958 94.498069 available None 
In [12]:
ipinfo_source_rows = (
    pd.read_parquet(ipinfo_path, columns=["network", "country_code"])
    if ipinfo_path is not None
    else None
)

def sampled_output(scope, country_code=None):
    if ipinfo_source_rows is None:
        return None, 0, "IPinfo artifact unavailable"
    eligible = ipinfo_source_rows.loc[~ipinfo_source_rows["network"].str.contains(":", na=False)]
    if country_code is not None:
        eligible = eligible.loc[eligible["country_code"].eq(country_code)]
    if eligible.empty:
        return None, 0, f"no eligible {scope} IPv4 rows"
    selected = eligible.sample(n=min(5_000, len(eligible)), random_state=SAMPLE_SEED).sort_index()
    ips = selected["network"].str.split("/", n=1).str[0]
    result = enrich(ips, route=route, cloud=cloud, peeringdb=peeringdb, ipinfo=ipinfo, asdb=asdb, tor=tor, vpn=vpn, datacentre=datacentre, bad_asns=bad_asns)
    return result, len(eligible), None

sample_scopes = {
    "all IPinfo IPv4 source rows": sampled_output("overall"),
    "IPinfo country_code=GB IPv4 source rows": sampled_output("GB", "GB"),
}

def metric(scope, name, count, denominator, reason=None):
    available = reason is None and denominator is not None and denominator > 0 and count is not pd.NA
    return {"scope": scope, "metric": name, "count": count if available else pd.NA, "denominator": denominator, "percent": pct(count, denominator) if available else pd.NA, "status": "available" if available else "unavailable", "reason": reason or ("zero denominator" if not available else None)}

sample_metrics = []
for scope, (sample_out, eligible_count, sample_reason) in sample_scopes.items():
    if sample_out is None:
        sample_metrics.append(metric(scope, "sample rows", pd.NA, eligible_count, sample_reason))
        continue
    sample_size = len(sample_out)
    sample_metrics.append(metric(scope, "sample rows", sample_size, eligible_count))
    route_known = int(sample_out["route_asn"].notna().sum())
    ipinfo_known = int(sample_out["ipinfo_asn"].notna().sum())
    route_reason = None if route is not None else "IPtoASN route source unavailable"
    ipinfo_reason = None if ipinfo is not None else "IPinfo lookup source unavailable"
    sample_metrics.append(metric(scope, "route ASN supplied", route_known, sample_size, route_reason))
    sample_metrics.append(metric(scope, "IPinfo ASN supplied", ipinfo_known, sample_size, ipinfo_reason))
    both = sample_out["route_asn"].notna() & sample_out["ipinfo_asn"].notna()
    both_count = int(both.sum())
    comparison_reason = route_reason or ipinfo_reason
    sample_metrics.append(metric(scope, "both ASNs supplied", both_count, sample_size, comparison_reason))
    agreement_reason = comparison_reason or ("no rows with both ASNs" if both_count == 0 else None)
    sample_metrics.append(metric(scope, "ASN agreements", int(sample_out.loc[both, "route_ipinfo_asn_agree"].eq(True).sum()), both_count, agreement_reason))
    sample_metrics.append(metric(scope, "ASN disagreements", int(sample_out.loc[both, "route_ipinfo_asn_agree"].eq(False).sum()), both_count, agreement_reason))
ip_sample_summary = pd.DataFrame(sample_metrics)
ip_sample_summary
scope metric count denominator percent status reason
0 all IPinfo IPv4 source rows sample rows 5000 1337977 0.373699 available None
1 all IPinfo IPv4 source rows route ASN supplied 4398 5000 87.960000 available None
2 all IPinfo IPv4 source rows IPinfo ASN supplied 4421 5000 88.420000 available None
3 all IPinfo IPv4 source rows both ASNs supplied 4397 5000 87.940000 available None
4 all IPinfo IPv4 source rows ASN agreements 4352 4397 98.976575 available None
5 all IPinfo IPv4 source rows ASN disagreements 45 4397 1.023425 available None
6 IPinfo country_code=GB IPv4 source rows sample rows 5000 67290 7.430525 available None
7 IPinfo country_code=GB IPv4 source rows route ASN supplied 4369 5000 87.380000 available None
8 IPinfo country_code=GB IPv4 source rows IPinfo ASN supplied 4388 5000 87.760000 available None
9 IPinfo country_code=GB IPv4 source rows both ASNs supplied 4366 5000 87.320000 available None
10 IPinfo country_code=GB IPv4 source rows ASN agreements 4345 4366 99.519011 available None
11 IPinfo country_code=GB IPv4 source rows ASN disagreements 21 4366 0.480989 available None 
In [13]:
def sample_classification_row(scope, sample_out, source, table, classification_field, sample_reason):
    empty = {"scope": scope, "source": source, "denominator_sample_rows": pd.NA, "route_asn_unavailable": pd.NA, "route_asn_unavailable_pct": pd.NA, "no_source_record": pd.NA, "no_source_record_pct": pd.NA, "classification_missing_or_empty": pd.NA, "classification_missing_or_empty_pct": pd.NA, "usable_classification": pd.NA, "usable_classification_pct": pd.NA}
    if sample_out is None:
        return {**empty, "status": "unavailable", "reason": sample_reason}
    denominator = len(sample_out)
    if route is None:
        return {**empty, "denominator_sample_rows": denominator, "route_asn_unavailable": denominator, "route_asn_unavailable_pct": pct(denominator, denominator), "status": "unavailable", "reason": "IPtoASN route source unavailable"}
    if table is None:
        return {**empty, "denominator_sample_rows": denominator, "status": "unavailable", "reason": f"{source} source unavailable"}
    if denominator == 0:
        return {**empty, "denominator_sample_rows": 0, "status": "unavailable", "reason": "empty sample"}
    route_asns = sample_out["route_asn"]
    route_missing = route_asns.isna()
    present = table.record_presence(route_asns).eq(True)
    joined = table.lookup(route_asns)
    usable = joined[classification_field].map(lambda value: value is not pd.NA and value is not None and len(value) > 0)
    no_record = int((~route_missing & ~present).sum())
    missing = int((present & ~usable).sum())
    usable_count = int((present & usable).sum())
    route_missing_count = int(route_missing.sum())
    return {**empty, "denominator_sample_rows": denominator, "route_asn_unavailable": route_missing_count, "route_asn_unavailable_pct": pct(route_missing_count, denominator), "no_source_record": no_record, "no_source_record_pct": pct(no_record, denominator), "classification_missing_or_empty": missing, "classification_missing_or_empty_pct": pct(missing, denominator), "usable_classification": usable_count, "usable_classification_pct": pct(usable_count, denominator), "status": "available", "reason": None}

ip_sample_classification = pd.DataFrame([
    sample_classification_row(scope, sample_out, source, table, field, sample_reason)
    for scope, (sample_out, _, sample_reason) in sample_scopes.items()
    for source, table, field in [
        ("PeeringDB", peeringdb, "peeringdb_info_types"),
        ("Stanford ASdb", asdb, "asdb_category_pairs"),
    ]
])
ip_sample_classification
scope source denominator_sample_rows route_asn_unavailable route_asn_unavailable_pct no_source_record no_source_record_pct classification_missing_or_empty classification_missing_or_empty_pct usable_classification usable_classification_pct status reason
0 all IPinfo IPv4 source rows PeeringDB 5000 602 12.04 900 18.00 301 6.02 3197 63.94 available None
1 all IPinfo IPv4 source rows Stanford ASdb 5000 602 12.04 69 1.38 0 0.00 4329 86.58 available None
2 IPinfo country_code=GB IPv4 source rows PeeringDB 5000 631 12.62 525 10.50 157 3.14 3687 73.74 available None
3 IPinfo country_code=GB IPv4 source rows Stanford ASdb 5000 631 12.62 31 0.62 0 0.00 4338 86.76 available None

Provenance

In [14]:
provenance = pd.DataFrame([
    {
        "source": record["source"],
        "path": record["path"],
        "retrieved_at": record["retrieved_at"],
        "source_updated_at": record["source_updated_at"],
        "file_mtime": record["file_mtime"],
        "sha256": record["sha256"],
        "url": record["url"],
        "status": record["status"],
        "reason": record["reason"],
    }
    for record in source_records
])
provenance
source path retrieved_at source_updated_at file_mtime sha256 url status reason
0 IPtoASN data/IP/ip2asn-combined.tsv.gz None None 2026-10-24T12:38:06+00:00 7336abc7611351b0261023c6268293bc9d4a02fb32a3ea... https://iptoasn.com/data/ip2asn-combined.tsv.gz available None
1 PeeringDB notebooks/peeringdb-net.json None 2026-10-24T16:21:34+00:00 2026-10-24T19:41:12+00:00 ccdcaea36cef0b2752c1b53abafae8c06c3d4684ad39e2... https://www.peeringdb.com/api/net available None
2 IPinfo Lite data/IP/ipinfo_lite.parquet None None 2026-10-24T13:02:50+00:00 9972cc59727492fec87ddb1da55c14faeabb75362401d3... https://ipinfo.io/developers/ipinfo-lite-database available None
3 rezmoss aggregate data/raw/ip-enrichment/2026-10-25/rezmoss_all_... 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 fa5c03243d283b27af15bf7cad73f42cf768e21f9c0654... https://raw.githubusercontent.com/rezmoss/clou... available None
4 Stanford ASdb data/IP/2026-03_categorized_ases.csv None None 2026-10-24T12:39:51+00:00 00ba179031d6eac793638c9ad1dd2ff03f71424b53a2f4... https://asdb.stanford.edu/about available None
5 X4BNet VPN IPv4 data/raw/ip-enrichment/2026-10-25/x4bnet_vpn_i... 2026-10-25T10:54:43+00:00 None 2026-10-25T10:54:43+00:00 d5555f06a151141d08d79bbbcb96150f979860b48d8bfd... https://raw.githubusercontent.com/X4BNet/lists... available None
6 X4BNet datacentre IPv4 data/raw/ip-enrichment/2026-10-25/x4bnet_datac... 2026-10-25T10:54:43+00:00 None 2026-10-25T10:54:43+00:00 380eccc1e05d05b3535dff6c380a0fb9d8bd6978018c97... https://raw.githubusercontent.com/X4BNet/lists... available None
7 bad-asn-list data/raw/ip-enrichment/2026-10-25/bad_asn_list... 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 0af8e58dc07ba070a10c9083a1a9f5c9fc76be35fab32b... https://raw.githubusercontent.com/brianhama/ba... available None
8 Tor bulk exit list data/raw/ip-enrichment/2026-10-25/tor_bulk_exi... 2026-10-25T10:54:51+00:00 None 2026-10-25T10:54:51+00:00 6756af70704aa18c358c48563b54ec27c962b86f0dc2b5... https://check.torproject.org/torbulkexitlist available None