#!/bin/bash
# tools/geoip-import <dbip-(country|city)-lite-YYYY-MM.csv.gz> <site data dir> — The GeoIP table (telemetry
# FR-GEO-001, FR-GEO-002). Turns DB-IP Lite's country CSV (start,end,country) or city CSV (start,end,continent,
# country,region,city,latitude,longitude), both CC BY 4.0 (https://db-ip.com), into the addon's store,
# <data>/telemetry/geo/:
#   v4-index, v6-index   the first address of each chunk, one per line, in order
#   v4-<n>, v6-<n>       2000 ranges each, one per line: the range's first address in fixed-width hex (8 for IPv4, 32
#                        for IPv6), its country code ("ZZ" for none) and its place (6 hex digits, 000000 for none).
#                        Consecutive ranges with the same country and place are kept as one.
#   places-<n>           2000 places each, one per line: region, city, latitude and longitude (two decimals), tab-
#                        separated; place n is line n % 2000 of places-<n / 2000>, and place 0 is none
#   meta                 source, licence and counts
# The ranges must cover each family without gaps (DB-IP's do); a gap is filled with ZZ. The table is built in a
# temporary folder and swapped in whole, so the site never reads half a table.
set -euo pipefail

[ $# -eq 2 ] || { echo "usage: geoip-import <dbip-(country|city)-lite-YYYY-MM.csv.gz> <site data dir>" >&2; exit 2; }
src="$1"; data="$2"
[ -f "$src" ] || { echo "geoip-import: no file $src" >&2; exit 1; }
gzip -t -- "$src" || { echo "geoip-import: $src isn't a gzip file" >&2; exit 1; }
dest="$data/telemetry/geo"
tmp="$data/telemetry/.geo-import-$$"
rm -rf -- "$tmp"; mkdir -p -- "$tmp"; chmod 700 -- "$tmp"

zcat -- "$src" | gawk -v FPAT='([^,]*)|("[^"]*")' -v out="$tmp" -v chunk=2000 -v source="$(basename -- "$src")" '
function v4hex(ip,   p) { split(ip, p, "."); return sprintf("%02x%02x%02x%02x", p[1], p[2], p[3], p[4]) }
function v6hex(ip,   ht, h, t, hn, tn, i, s, missing) {
  if (index(ip, "::")) {
    split(ip, ht, "::")
    hn = (ht[1] == "") ? 0 : split(ht[1], h, ":"); tn = (ht[2] == "") ? 0 : split(ht[2], t, ":")
    s = ""; for (i = 1; i <= hn; i++) s = s sprintf("%4s", h[i])
    missing = 8 - hn - tn; for (i = 0; i < missing; i++) s = s "0000"
    for (i = 1; i <= tn; i++) s = s sprintf("%4s", t[i])
  } else { hn = split(ip, h, ":"); s = ""; for (i = 1; i <= hn; i++) s = s sprintf("%4s", h[i]) }
  gsub(/ /, "0", s); return tolower(s)
}
# hex + 1, for the address after a range (to find gaps); "" past the end of the family.
function next_hex(x,   i, d, digits, c, out) {
  digits = "0123456789abcdef"; out = x
  for (i = length(x); i >= 1; i--) {
    c = index(digits, substr(x, i, 1)) - 1
    if (c < 15) { return substr(out, 1, i - 1) substr(digits, c + 2, 1) substr(out, i + 1) }
    out = substr(out, 1, i - 1) "0" substr(out, i + 1)
  }
  return ""
}
function emit(fam, start, value) {
  if (value == last[fam]) return
  if (n[fam] % chunk == 0) { k = n[fam] / chunk; print start > (out "/" fam "-index"); file[fam] = out "/" fam "-" k }
  print start value > file[fam]; n[fam]++; last[fam] = value
}
function add(fam, start, end, value) {
  if (expect[fam] != "" && start != expect[fam]) {
    if (start < expect[fam]) { print "geoip-import: ranges out of order near " start > "/dev/stderr"; bad = 1; exit 1 }
    emit(fam, expect[fam], "ZZ000000"); gaps++
  }
  emit(fam, start, value); expect[fam] = next_hex(end); ranges++
}
function unquote(s) { if (s ~ /^".*"$/) s = substr(s, 2, length(s) - 2); gsub(/[\t\r\n]/, " ", s); return s }
function coord(s) { return (s ~ /^-?[0-9.]+$/) ? sprintf("%.2f", s + 0) : "" }
# The id of a place (region, city, coordinates), new ones numbered from 1.
function place(region, city, lat, lon,   key) {
  if (region == "" && city == "") return 0
  key = region "\t" city "\t" coord(lat) "\t" coord(lon)
  if (!(key in ids)) { ids[key] = ++places; byid[places] = key }
  return ids[key]
}
function value(cc, p) { if (cc !~ /^[A-Z][A-Z]$/ || cc == "ZZ") return "ZZ000000"; return cc sprintf("%06x", p) }
NF == 3 && $1 ~ /:/   { add("v6", v6hex($1), v6hex($2), value($3, 0)); next }
NF == 3 && $1 ~ /\./ { add("v4", v4hex($1), v4hex($2), value($3, 0)); next }
NF == 8 && $1 ~ /:/   { add("v6", v6hex($1), v6hex($2), value($4, place(unquote($5), unquote($6), $7, $8))); next }
NF == 8 && $1 ~ /\./ { add("v4", v4hex($1), v4hex($2), value($4, place(unquote($5), unquote($6), $7, $8))); next }
NF > 0 { skipped++ }
END {
  if (bad) exit 1
  print "" > (out "/places-0")
  for (i = 1; i <= places; i++) print byid[i] > (out "/places-" int(i / 2000))
  printf "source: %s\nlicence: CC BY 4.0 (DB-IP, https://db-ip.com)\nformat: places\nranges: %d\nv4: %d\nv6: %d\nplaces: %d\ngaps: %d\nskipped: %d\n", source, ranges, n["v4"], n["v6"], places, gaps, skipped > (out "/meta")
  printf "geoip-import: %d ranges kept as %d IPv4 and %d IPv6 records, %d places, %d gaps filled, %d lines skipped\n", ranges, n["v4"], n["v6"], places, gaps, skipped
}'

chmod 600 -- "$tmp"/*
rm -rf -- "$dest.old"
[ -d "$dest" ] && mv -- "$dest" "$dest.old"
mv -- "$tmp" "$dest"
rm -rf -- "$dest.old"
echo "geoip-import: installed in $dest ($(du -sh -- "$dest" | cut -f1))"
