diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bb94f..46a21dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file. ## Unreleased +### New Features + +* Add collector-aware AS relationship output alongside classic aggregate (#13) + * `as2rel-index` now produces per-collector provenance files (`*-collector-latest.json.bz2`) + * Classic output unchanged — no breaking changes to existing data pipeline + * Single file pass produces both outputs for all three prefixes (as2rel, as2rel-v4, as2rel-v6) + * Memory-efficient flat-vector + sort approach (~1.7GB peak RSS for 21M records) + +### Performance + +* Switch to route-level BGP parser (`into_route_iter`) from bgpkit-parser v0.18.0 + * ~19% faster on RIPE RIS bview files; larger gains expected on full RIB dumps + * Eliminates per-path allocation by borrowing AS path via `Arc` + +### Dependencies + +* bgpkit-parser: 0.11.0 → 0.18.0 (route-level parser, AS4_PATH merge fix, new attribute parsers) +* bgpkit-broker: 0.7.5 → 0.11.0 (new collectors: locix.fra, ixpn.lagos, decix.fra, crix.sjo; SDK caching) + ### Code Refactoring * Refactored lib.rs into dedicated modules (as2rel, peer_stats, pfx2as) with processor pattern @@ -20,6 +39,8 @@ All notable changes to this project will be documented in this file. * Removed AS 1239 (Sprint) from tier-1 ASN list to match bgp.tools definition * Removed unnecessary ASN 0 placeholder from TIER1_V4 array +* Fix candidate tier-1 transit detection to stop at failing candidates (#12) +* Fix clippy warnings in bootstrap.rs (useless borrows in format! macros) ## v0.2.1 - 2025-04-09 @@ -40,4 +61,4 @@ All notable changes to this project will be documented in this file. * add new command to index information into sqlite database by @digizeph in https://github.com/bgpkit/peer-stats/pull/2 * Update dependencies by @digizeph in https://github.com/bgpkit/peer-stats/pull/5 -* Env vars by @digizeph in https://github.com/bgpkit/peer-stats/pull/6 \ No newline at end of file +* Env vars by @digizeph in https://github.com/bgpkit/peer-stats/pull/6 diff --git a/Cargo.lock b/Cargo.lock index 76cc6d3..8b427d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,29 @@ dependencies = [ "url", ] +[[package]] +name = "aws-lc-rs" +version = "1.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "aws-region" version = "0.25.5" @@ -202,9 +225,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bgpkit-broker" -version = "0.7.11" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae9ba695f79fdece29bdb67e9e85b8616ba2d5e7555970120b48553c1072f08" +checksum = "2a4c76ccd79e21d20b52006834b63e68793cb4538e6c6b001e22cde1aa13019b" dependencies = [ "chrono", "dotenvy", @@ -213,25 +236,28 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 1.0.69", + "sha2", + "thiserror 2.0.12", "tracing", ] [[package]] name = "bgpkit-parser" -version = "0.11.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6550ea4a70595cb2b2c971581b3f217f27a40774e523552d9c58ebacf103187" +checksum = "69d67aa5093677071e8e702b469dba28f20f47ce703f600357e3f01dbb82f67a" dependencies = [ "bitflags", "bytes", "chrono", "ipnet", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num_enum", - "oneio", + "oneio 0.23.0", "regex", + "smallvec", + "zerocopy 0.8.24", ] [[package]] @@ -274,6 +300,15 @@ dependencies = [ "libc", ] +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -286,10 +321,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.18" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -362,6 +398,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -523,6 +568,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" @@ -568,6 +619,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "flate2" version = "1.1.1" @@ -575,6 +632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -593,6 +651,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.31" @@ -1114,9 +1178,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -1176,6 +1240,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.171" @@ -1193,6 +1263,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" +dependencies = [ + "zlib-rs", +] + [[package]] name = "litemap" version = "0.7.5" @@ -1367,7 +1446,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e60b2501ea880ed754327e90bfc38045190040b46978ff75eefe980c560e972f" dependencies = [ - "bzip2", + "bzip2 0.4.4", "dotenvy", "flate2", "lz4", @@ -1375,12 +1454,27 @@ dependencies = [ "rust-s3", "serde", "serde_json", - "suppaftp", + "suppaftp 6.1.1", "thiserror 1.0.69", "xz2", "zstd", ] +[[package]] +name = "oneio" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ec2e492d1f3350fa4b5701dbac89544a521c755b99d1667dd98c3cdd5984a" +dependencies = [ + "bzip2 0.6.1", + "dotenvy", + "flate2", + "reqwest", + "rustls 0.23.25", + "suppaftp 7.1.0", + "thiserror 2.0.12", +] + [[package]] name = "openssl-probe" version = "0.1.6" @@ -1416,14 +1510,14 @@ dependencies = [ "anyhow", "bgpkit-broker", "bgpkit-parser", - "bzip2", + "bzip2 0.4.4", "chrono", "clap", "indicatif", "ipnet", "itertools 0.13.0", "num_cpus", - "oneio", + "oneio 0.17.0", "rayon", "rusqlite", "serde", @@ -1795,6 +1889,7 @@ version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -1850,6 +1945,7 @@ version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1984,9 +2080,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "slab" @@ -2002,6 +2098,9 @@ name = "smallvec" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -2045,6 +2144,20 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "suppaftp" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a15b325bbe0a1f85de3dbf988a3a14e9cd321537dffcbf6641381dd6d7586f" +dependencies = [ + "chrono", + "futures-lite", + "lazy-regex", + "log", + "rustls 0.23.25", + "thiserror 2.0.12", +] + [[package]] name = "syn" version = "2.0.100" @@ -2968,6 +3081,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index ade074a..f6dd984 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,8 @@ keywords = ["bgp", "bgpkit", "mrt"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bgpkit-parser = "0.11.0" -bgpkit-broker = "0.7.5" +bgpkit-parser = { version = "0.18.0", features = ["local"] } +bgpkit-broker = "0.11.0" ipnet = "2.7.2" serde = { version = "1", features = ["derive"] } diff --git a/src/lib.rs b/src/lib.rs index 06f4014..58d8bf8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,10 +38,10 @@ pub fn parse_rib_file( let mut pfx2as_collector = Pfx2AsProcessor::new(); let mut as2rel_collector = As2RelProcessor::new(); - for elem in BgpkitParser::new(file_url)? { - let peer_ip = elem.peer_ip; - let peer_asn = elem.peer_asn.to_u32(); - let prefix = elem.prefix.prefix; + for route in BgpkitParser::new(file_url)?.into_route_iter() { + let peer_ip = route.peer_ip; + let peer_asn = route.peer_asn.to_u32(); + let prefix = route.prefix.prefix; // Extract prefix info let (prefix_v4, prefix_v6) = match prefix { @@ -49,9 +49,9 @@ pub fn parse_rib_file( IpNet::V6(net) => (None, Some(net)), }; - // Process AS path data + // Process AS path data (borrowed via Arc, no clone needed) let mut connected_asn = None; - if let Some(as_path) = elem.as_path.clone() { + if let Some(ref as_path) = route.as_path { if let Some(u32_path) = as_path.to_u32_vec_opt(true) { // Get connected ASN (second hop in path) connected_asn = u32_path.get(1).copied();