#!/bin/bash
# tools/asn-import <dbip-asn-lite-YYYY-MM.csv.gz> <site data dir> — The network table (telemetry ADR-0005): which
# organisation holds an address's network. Turns DB-IP ASN Lite's CSV (start,end,as_number,as_organization; CC BY 4.0,
# https://db-ip.com) into the addon's store, <data>/telemetry/asn/:
#   v4-index, v6-index   the first address of each chunk, one per line, in order
#   v4-<n>, v6-<n>       2000 ranges each, one per line: the range's first address in fixed-width hex (8 for IPv4, 32
#                        for IPv6) and its network's number (6 hex digits, 000000 for none). Consecutive ranges of the
#                        same network are kept as one; a gap between ranges is a range with none.
#   nets-<n>             2000 networks each, one per line: "AS<number> <organisation>"; network n is line n % 2000 of
#                        nets-<n / 2000>, and network 0 is none
#   meta                 source, licence and counts
# Only robots get their network looked up, and only the organisation's name is kept (ADR-0005). The table is built
# in a temporary folder and swapped in whole, so the site never reads half a table.
set -euo pipefail

[ $# -eq 2 ] || { echo "usage: asn-import <dbip-asn-lite-YYYY-MM.csv.gz> <site data dir>" >&2; exit 2; }
src="$1"; data="$2"
[ -f "$src" ] || { echo "asn-import: no file $src" >&2; exit 1; }
gzip -t -- "$src" || { echo "asn-import: $src isn't a gzip file" >&2; exit 1; }
dest="$data/telemetry/asn"
tmp="$data/telemetry/.asn-import-$$"
rm -rf -- "$tmp"; mkdir -p -- "$tmp"; chmod 700 -- "$tmp"

zcat -- "$src" | gawk -v FPAT='([^,]*)|("[^"]*")' -v out="$tmp" -v chunk=2000 -v source="$(basename -- "$src")" '
function v4hex(ip,   p) { split(ip, p, "."); return sprintf("%02x%02x%02x%02x", p[1], p[2], p[3], p[4]) }
function v6hex(ip,   ht, h, t, hn, tn, i, s, missing) {
  if (index(ip, "::")) {
    split(ip, ht, "::")
    hn = (ht[1] == "") ? 0 : split(ht[1], h, ":"); tn = (ht[2] == "") ? 0 : split(ht[2], t, ":")
    s = ""; for (i = 1; i <= hn; i++) s = s sprintf("%4s", h[i])
    missing = 8 - hn - tn; for (i = 0; i < missing; i++) s = s "0000"
    for (i = 1; i <= tn; i++) s = s sprintf("%4s", t[i])
  } else { hn = split(ip, h, ":"); s = ""; for (i = 1; i <= hn; i++) s = s sprintf("%4s", h[i]) }
  gsub(/ /, "0", s); return tolower(s)
}
function next_hex(x,   i, digits, c, out) {
  digits = "0123456789abcdef"; out = x
  for (i = length(x); i >= 1; i--) {
    c = index(digits, substr(x, i, 1)) - 1
    if (c < 15) { return substr(out, 1, i - 1) substr(digits, c + 2, 1) substr(out, i + 1) }
    out = substr(out, 1, i - 1) "0" substr(out, i + 1)
  }
  return ""
}
function emit(fam, start, value) {
  if (fam in last && value == last[fam]) return
  if (n[fam] % chunk == 0) { k = n[fam] / chunk; print start > (out "/" fam "-index"); file[fam] = out "/" fam "-" k }
  print start value > file[fam]; n[fam]++; last[fam] = value
}
function add(fam, start, end, value) {
  if (expect[fam] != "" && start != expect[fam]) {
    if (start < expect[fam]) { print "asn-import: ranges out of order near " start > "/dev/stderr"; bad = 1; exit 1 }
    emit(fam, expect[fam], "000000"); gaps++
  }
  if (expect[fam] == "" && start != zero[fam]) emit(fam, zero[fam], "000000")
  emit(fam, start, value); expect[fam] = next_hex(end); ranges++
}
function unquote(s) { if (s ~ /^".*"$/) s = substr(s, 2, length(s) - 2); gsub(/[\t\r\n]/, " ", s); return s }
# The id of a network, new ones numbered from 1.
function net(asn, org,   key) {
  if (asn !~ /^[0-9]+$/) return 0
  key = "AS" asn " " unquote(org)
  if (!(key in ids)) { ids[key] = ++nets; byid[nets] = key }
  return ids[key]
}
BEGIN { zero["v4"] = "00000000"; zero["v6"] = "00000000000000000000000000000000" }
NF == 4 && $1 ~ /:/   { add("v6", v6hex($1), v6hex($2), sprintf("%06x", net($3, $4))); next }
NF == 4 && $1 ~ /\./ { add("v4", v4hex($1), v4hex($2), sprintf("%06x", net($3, $4))); next }
NF > 0 { skipped++ }
END {
  if (bad) exit 1
  # After the last range, up to the end of the family: none.
  for (fam in expect) if (expect[fam] != "") emit(fam, expect[fam], "000000")
  print "" > (out "/nets-0")
  for (i = 1; i <= nets; i++) print byid[i] > (out "/nets-" int(i / 2000))
  printf "source: %s\nlicence: CC BY 4.0 (DB-IP, https://db-ip.com)\nformat: asn\nranges: %d\nv4: %d\nv6: %d\nnetworks: %d\ngaps: %d\nskipped: %d\n", source, ranges, n["v4"], n["v6"], nets, gaps, skipped > (out "/meta")
  printf "asn-import: %d ranges kept as %d IPv4 and %d IPv6 records, %d networks, %d gaps, %d lines skipped\n", ranges, n["v4"], n["v6"], nets, gaps, skipped
}'

chmod 600 -- "$tmp"/*
rm -rf -- "$dest.old"
[ -d "$dest" ] && mv -- "$dest" "$dest.old"
mv -- "$tmp" "$dest"
rm -rf -- "$dest.old"
echo "asn-import: installed in $dest ($(du -sh -- "$dest" | cut -f1))"
