From af213d4365ed72b7d9450e999edb492a847e5441 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 5 Feb 2020 02:21:46 -0600 Subject: [PATCH] Trace using Jaeger. --- Cargo.lock | 399 ++++++++++++++++++++++++++++++++++++++++++------ Cargo.toml | 34 +++-- src/handlers.rs | 85 +++++------ src/main.rs | 48 ++++++ src/models.rs | 12 +- src/types.rs | 2 +- src/utils.rs | 7 +- 7 files changed, 466 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d621879..1cc2efd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,13 +8,22 @@ checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" [[package]] name = "aho-corasick" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f56c476256dc249def911d6f7580b5fc7e875895b5d7ee88f5d602208035744" +checksum = "743ad5a418686aad3b87fd14c43badd828cf26e214a00f92a384291cf22e1811" dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "arc-swap" version = "0.4.4" @@ -23,9 +32,9 @@ checksum = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" [[package]] name = "async-trait" -version = "0.1.22" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df72488e87761e772f14ae0c2480396810e51b2c2ade912f97f0f7e5b95e3c" +checksum = "750b1c38a1dfadd108da0f01c08f4cdc7ff1bb39b325f9c82cc972361780a6e1" dependencies = [ "proc-macro2 1.0.8", "quote 1.0.2", @@ -55,6 +64,28 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +[[package]] +name = "backtrace" +version = "0.3.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "base64" version = "0.10.1" @@ -168,6 +199,12 @@ dependencies = [ "ppv-lite86", ] +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" + [[package]] name = "cfg-if" version = "0.1.10" @@ -209,6 +246,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa" +dependencies = [ + "crossbeam-utils 0.6.6", +] + [[package]] name = "crossbeam-deque" version = "0.7.2" @@ -216,7 +262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" dependencies = [ "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.7.0", ] [[package]] @@ -227,7 +273,7 @@ checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" dependencies = [ "autocfg 0.1.7", "cfg-if", - "crossbeam-utils", + "crossbeam-utils 0.7.0", "lazy_static", "memoffset", "scopeguard", @@ -240,7 +286,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" dependencies = [ "cfg-if", - "crossbeam-utils", + "crossbeam-utils 0.7.0", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if", + "lazy_static", ] [[package]] @@ -297,9 +353,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" [[package]] name = "env_logger" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", "humantime", @@ -350,9 +406,9 @@ checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" [[package]] name = "futures" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" +checksum = "ad6636318d07abeb4656157ef1936c64485f066c7f9ce5d7c5b879fcb6dd5ccb" dependencies = [ "futures-channel", "futures-core", @@ -365,9 +421,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" +checksum = "7264eb65b194d2fa6ec31b898ead7c332854bfa42521659226e72a585fca5b85" dependencies = [ "futures-core", "futures-sink", @@ -375,15 +431,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" +checksum = "b597b16aa1a19ce2dfde5128a7c656d75346b35601a640be2d9efd4e9c83609d" [[package]] name = "futures-executor" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" +checksum = "46a5e593d77bee52393c7f3b16b8b413214096d3f7dc4f5f4c57dee01ad2bdaf" dependencies = [ "futures-core", "futures-task", @@ -392,15 +448,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" +checksum = "3d429f824b5e5dbd45fc8e54e1005a37e1f8c6d570cd64d0b59b24d3a80b8b8e" [[package]] name = "futures-macro" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" +checksum = "a1d75b72904b78044e0091355fc49d29f48bff07a68a719a41cf059711e071b4" dependencies = [ "proc-macro-hack", "proc-macro2 1.0.8", @@ -410,21 +466,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" +checksum = "04299e123547ea7c56f3e1b376703142f5fc0b6700433eed549e9d0b8a75a66c" [[package]] name = "futures-task" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" +checksum = "86f9ceab4bce46555ee608b1ec7c414d6b2e76e196ef46fa5a8d4815a8571398" [[package]] name = "futures-util" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" +checksum = "7d2f1296f7644d2cd908ebb2fa74645608e39f117c72bac251d40418c6d74c4f" dependencies = [ "futures-channel", "futures-core", @@ -452,11 +508,15 @@ dependencies = [ "hamming", "image", "img_hash", - "log 0.4.8", + "opentelemetry", "pretty_env_logger", "serde", "tokio", "tokio-postgres", + "tracing", + "tracing-futures", + "tracing-opentelemetry", + "tracing-subscriber", "warp", ] @@ -568,6 +628,17 @@ dependencies = [ "digest", ] +[[package]] +name = "hostname" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5260e6c63877196b6fca5a7fb4eaff751134045ad3415716192baa36f5b9a0" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.8", +] + [[package]] name = "http" version = "0.1.21" @@ -787,6 +858,21 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1022,6 +1108,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "opentelemetry" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aafc7bbd789148b428ff67ecd4379b18d997f982eedf4f8d5481aa3e56487" +dependencies = [ + "crossbeam-channel", + "lazy_static", + "prometheus", + "rand 0.7.3", + "rustracing", + "rustracing_jaeger", +] + [[package]] name = "parking_lot" version = "0.10.0" @@ -1046,6 +1146,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1191,11 +1297,10 @@ checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" [[package]] name = "pretty_env_logger" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "717ee476b1690853d222af4634056d830b5197ffd747726a9a1eee6da9f49074" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ - "chrono", "env_logger", "log 0.4.8", ] @@ -1235,6 +1340,26 @@ dependencies = [ "unicode-xid 0.2.0", ] +[[package]] +name = "prometheus" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "protobuf", + "quick-error", + "spin", +] + +[[package]] +name = "protobuf" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6686ddd96a8dbe2687b5f2a687b2cfb520854010ec480f2d74c32e7c9873d3c5" + [[package]] name = "quick-error" version = "1.2.3" @@ -1425,7 +1550,7 @@ checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" dependencies = [ "crossbeam-deque", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.7.0", "lazy_static", "num_cpus", ] @@ -1447,9 +1572,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "regex" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5508c1941e4e7cb19965abef075d35a9a8b5cdf0846f30b4050e9b55dc55e87" +checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" dependencies = [ "aho-corasick", "memchr", @@ -1458,10 +1583,21 @@ dependencies = [ ] [[package]] -name = "regex-syntax" -version = "0.6.13" +name = "regex-automata" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e734e891f5b408a29efbf8309e656876276f49ab6a6ac208600b4419bd893d90" +checksum = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "regex-syntax" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" [[package]] name = "remove_dir_all" @@ -1472,6 +1608,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + [[package]] name = "rustc-serialize" version = "0.3.24" @@ -1507,6 +1649,33 @@ dependencies = [ "num-traits", ] +[[package]] +name = "rustracing" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033dbbcfe140c3f3bec3b5e2165e11583a1eb0bc08056faaeb2b3bfaa753ddc" +dependencies = [ + "backtrace", + "crossbeam-channel", + "rand 0.7.3", + "trackable", +] + +[[package]] +name = "rustracing_jaeger" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13ad9f8324f6a35bf0d15facccdad627be352b737a4d15c1a15398909352fe61" +dependencies = [ + "crossbeam-channel", + "hostname", + "percent-encoding 1.0.1", + "rand 0.7.3", + "rustracing", + "thrift_codec", + "trackable", +] + [[package]] name = "ryu" version = "1.0.2" @@ -1574,9 +1743,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b" +checksum = "21b01d7f0288608a01dca632cf1df859df6fd6ffa885300fc275ce2ba6221953" dependencies = [ "itoa", "ryu", @@ -1619,6 +1788,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "sharded-slab" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae75d0445b5d3778c9da3d1f840faa16d0627c8607f78a74daf69e5b988c39a1" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.2.0" @@ -1665,6 +1843,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stringprep" version = "0.1.2" @@ -1735,6 +1919,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "thrift_codec" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb61fb3d0a0af14949f3a6949b2639112e13226647112824f4d081533f9b1a8" +dependencies = [ + "byteorder", + "trackable", +] + [[package]] name = "tiff" version = "0.3.1" @@ -1795,17 +1989,18 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03cb0c66092269a9b280e9e4956cb23ce00b8a6b1b393f7700f7732ac4bf133" +checksum = "b4794270a16f0095a02a752e6cbe9618ef440397af5eb409fa9eb826650ccbdb" dependencies = [ + "async-trait", "byteorder", "bytes 0.5.4", "fallible-iterator", "futures", "log 0.4.8", "parking_lot", - "percent-encoding", + "percent-encoding 2.1.0", "phf 0.8.0", "pin-project-lite", "postgres-protocol", @@ -1834,6 +2029,118 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +[[package]] +name = "tracing" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e213bd24252abeb86a0b7060e02df677d367ce6cb772cef17e9214b8390a8d3" +dependencies = [ + "cfg-if", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cfd395def5a60236e187e1ff905cb55668a59f29928dec05e6e1b1fd2ac1f3" +dependencies = [ + "quote 1.0.2", + "syn 1.0.14", +] + +[[package]] +name = "tracing-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a46f11e372b8bd4b4398ea54353412fdd7fd42a8370c7e543e218cf7661978" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33848db47a7c848ab48b66aab3293cb9c61ea879a3586ecfcd17302fcea0baf1" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log 0.4.8", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94651cc1896a2d9e2264e5fc95651b041f4d67245087dcb763523db3bc48185a" +dependencies = [ + "opentelemetry", + "rand 0.7.3", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-serde" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65dea8255e378ab7db9db2077a90cb3e051515e18eaa819a405c4eb129b9beb" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dc36e47794112347ccb9ae8a05b5ec4fab45f01545e4929323cdb1a543a8f4" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "trackable" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11475c3c53b075360eac9794965822cb053996046545f91cf61d90e00b72efa5" +dependencies = [ + "trackable_derive", +] + +[[package]] +name = "trackable_derive" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcf0b9b2caa5f4804ef77aeee1b929629853d806117c48258f402b69737e65c" +dependencies = [ + "quote 1.0.2", + "syn 1.0.14", +] + [[package]] name = "transpose" version = "0.1.0" @@ -1936,7 +2243,7 @@ checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" dependencies = [ "idna", "matches", - "percent-encoding", + "percent-encoding 2.1.0", ] [[package]] @@ -1951,6 +2258,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" +[[package]] +name = "utf8-ranges" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba" + [[package]] name = "version_check" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index d233b8f..c158649 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,26 +5,32 @@ authors = ["Syfaro "] edition = "2018" [dependencies] -log = "0.4.8" -pretty_env_logger = "0.3.1" +pretty_env_logger = "0.4" -tokio = { version = "0.2.9", features = ["full"] } -futures = "0.3.1" -futures-util = "0.3.1" +tracing = "0.1" +tracing-subscriber = "0.2" +tracing-futures = "0.2" -chrono = "0.4.10" -bytes = "0.5.3" +opentelemetry = "0.1" +tracing-opentelemetry = "0.1" -serde = { version = "1.0.104", features = ["derive"] } -warp = "0.2.0" +tokio = { version = "0.2", features = ["full"] } +futures = "0.3" +futures-util = "0.3" -tokio-postgres = "0.5.1" -bb8 = "0.4.0" -bb8-postgres = "0.4.0" +chrono = "0.4" +bytes = "0.5" -img_hash = "3.0.0" +serde = { version = "1", features = ["derive"] } +warp = "0.2" + +tokio-postgres = "0.5" +bb8 = "0.4" +bb8-postgres = "0.4" + +img_hash = "3" image = "0.22" -hamming = "0.1.3" +hamming = "0.1" [profile.release] lto = true diff --git a/src/handlers.rs b/src/handlers.rs index ad62b0e..680d9a1 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1,7 +1,8 @@ use crate::models::{image_query, image_query_sync}; use crate::types::*; use crate::{rate_limit, Pool}; -use log::{debug, info}; +use tracing::{span, warn}; +use tracing_futures::Instrument; use warp::{reject, Rejection, Reply}; fn map_bb8_err(err: bb8::RunError) -> Rejection { @@ -35,18 +36,11 @@ impl From for Error { impl warp::reject::Reject for Error {} -pub async fn search_image( - form: warp::multipart::FormData, - opts: ImageSearchOpts, - pool: Pool, - api_key: String, -) -> Result { - let db = pool.get().await.map_err(map_bb8_err)?; - - rate_limit!(&api_key, &db, image_limit, "image"); - +#[tracing::instrument(skip(form))] +async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHash<[u8; 8]>) { use bytes::BufMut; use futures_util::StreamExt; + let parts: Vec<_> = form.collect().await; let mut parts = parts .into_iter() @@ -65,18 +59,35 @@ pub async fn search_image( }) .await; - let hash = { + let len = bytes.len(); + + let hash = tokio::task::spawn_blocking(move || { let hasher = crate::get_hasher(); let image = image::load_from_memory(&bytes).unwrap(); hasher.hash_image(&image) - }; + }) + .instrument(span!(tracing::Level::TRACE, "hashing image", len)) + .await + .unwrap(); let mut buf: [u8; 8] = [0; 8]; buf.copy_from_slice(&hash.as_bytes()); - let num = i64::from_be_bytes(buf); + (i64::from_be_bytes(buf), hash) +} - debug!("Matching hash {}", num); +#[tracing::instrument(skip(form, pool, api_key))] +pub async fn search_image( + form: warp::multipart::FormData, + opts: ImageSearchOpts, + pool: Pool, + api_key: String, +) -> Result { + let db = pool.get().await.map_err(map_bb8_err)?; + + rate_limit!(&api_key, &db, image_limit, "image"); + + let (num, hash) = hash_input(form).await; let mut items = { if opts.search_type == Some(ImageSearchType::Force) { @@ -112,47 +123,19 @@ pub async fn search_image( Ok(warp::reply::json(&similarity)) } +#[tracing::instrument(skip(form, pool, api_key))] pub async fn stream_image( form: warp::multipart::FormData, pool: Pool, api_key: String, ) -> Result { + use futures_util::StreamExt; + let db = pool.get().await.map_err(map_bb8_err)?; rate_limit!(&api_key, &db, image_limit, "image", 2); - use bytes::BufMut; - use futures_util::StreamExt; - let parts: Vec<_> = form.collect().await; - let mut parts = parts - .into_iter() - .map(|part| { - let part = part.unwrap(); - (part.name().to_string(), part) - }) - .collect::>(); - let image = parts.remove("image").unwrap(); - - let bytes = image - .stream() - .fold(bytes::BytesMut::new(), |mut b, data| { - b.put(data.unwrap()); - async move { b } - }) - .await; - - let hash = { - let hasher = crate::get_hasher(); - let image = image::load_from_memory(&bytes).unwrap(); - hasher.hash_image(&image) - }; - - let mut buf: [u8; 8] = [0; 8]; - buf.copy_from_slice(&hash.as_bytes()); - - let num = i64::from_be_bytes(buf); - - debug!("Stream matching hash {}", num); + let (num, hash) = hash_input(form).await; let exact_event_stream = image_query_sync(pool.clone(), vec![num], 0, Some(hash.as_bytes().to_vec())) @@ -175,6 +158,7 @@ fn sse_matches( Ok(warp::sse::json(items)) } +#[tracing::instrument(skip(form, db, api_key))] pub async fn search_hashes( opts: HashSearchOpts, db: Pool, @@ -206,6 +190,7 @@ pub async fn search_hashes( Ok(warp::reply::json(&matches)) } +#[tracing::instrument(skip(db, api_key))] pub async fn search_file( opts: FileSearchOpts, db: Pool, @@ -226,8 +211,6 @@ pub async fn search_file( return Err(warp::reject::custom(Error::InvalidData)); }; - debug!("Searching for {:?}", opts); - let query = format!( "SELECT submission.id, @@ -250,6 +233,7 @@ pub async fn search_file( let matches: Vec<_> = db .query::(&*query, &[val]) + .instrument(span!(tracing::Level::TRACE, "waiting for db")) .await .map_err(map_postgres_err)? .into_iter() @@ -273,8 +257,9 @@ pub async fn search_file( Ok(warp::reply::json(&matches)) } +#[tracing::instrument] pub async fn handle_rejection(err: Rejection) -> Result { - info!("Had rejection: {:?}", err); + warn!("had rejection"); let (code, message) = if err.is_not_found() { ( diff --git a/src/main.rs b/src/main.rs index c136ef2..acf8389 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,10 +10,58 @@ mod utils; use warp::Filter; +fn configure_tracing() { + use opentelemetry::{ + api::{KeyValue, Provider, Sampler}, + exporter::trace::jaeger, + sdk::Config, + }; + use tracing_subscriber::layer::SubscriberExt; + + let env = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + + let exporter = jaeger::Exporter::builder() + .with_collector_endpoint(std::env::var("JAEGER_COLLECTOR").unwrap().parse().unwrap()) + .with_process(jaeger::Process { + service_name: "fuzzysearch", + tags: vec![ + KeyValue::new("environment", env), + KeyValue::new("version", env!("CARGO_PKG_VERSION")), + ], + }) + .init(); + + let provider = opentelemetry::sdk::Provider::builder() + .with_exporter(exporter) + .with_config(Config { + default_sampler: Sampler::Always, + ..Default::default() + }) + .build(); + + let tracer = provider.get_tracer("api"); + + let telem_layer = tracing_opentelemetry::OpentelemetryLayer::with_tracer(tracer); + let fmt_layer = tracing_subscriber::fmt::Layer::default(); + + let subscriber = tracing_subscriber::Registry::default() + .with(telem_layer) + .with(fmt_layer); + + tracing::subscriber::set_global_default(subscriber) + .expect("Unable to set default tracing subscriber"); +} + #[tokio::main] async fn main() { pretty_env_logger::init(); + configure_tracing(); + let s = std::env::var("POSTGRES_DSN").expect("Missing POSTGRES_DSN"); let manager = bb8_postgres::PostgresConnectionManager::new( diff --git a/src/models.rs b/src/models.rs index 38320bb..1864908 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,10 +1,12 @@ use crate::types::*; use crate::utils::extract_rows; use crate::Pool; +use tracing_futures::Instrument; pub type DB<'a> = &'a bb8::PooledConnection<'a, bb8_postgres::PostgresConnectionManager>; +#[tracing::instrument(skip(db))] pub async fn lookup_api_key(key: &str, db: DB<'_>) -> Option { let rows = db .query( @@ -37,6 +39,7 @@ pub async fn lookup_api_key(key: &str, db: DB<'_>) -> Option { } } +#[tracing::instrument(skip(pool))] pub async fn image_query( pool: Pool, hashes: Vec, @@ -53,18 +56,13 @@ pub async fn image_query( Ok(matches) } +#[tracing::instrument(skip(pool))] pub fn image_query_sync( pool: Pool, hashes: Vec, distance: i64, hash: Option>, ) -> tokio::sync::mpsc::Receiver, tokio_postgres::Error>> { - log::trace!( - "Running image query on {} hashes with distance {}", - hashes.len(), - distance - ); - let (mut tx, rx) = tokio::sync::mpsc::channel(1); tokio::spawn(async move { @@ -140,7 +138,7 @@ pub fn image_query_sync( let query = db.query::(&*hash_query, ¶ms).await; let rows = query.map(|rows| extract_rows(rows, hash.as_deref()).into_iter().collect()); tx.send(rows).await.unwrap(); - }); + }.in_current_span()); rx } diff --git a/src/types.rs b/src/types.rs index 0637f9a..ad9de53 100644 --- a/src/types.rs +++ b/src/types.rs @@ -98,7 +98,7 @@ pub struct ErrorMessage { pub message: String, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub struct HashSearchOpts { pub hashes: String, } diff --git a/src/utils.rs b/src/utils.rs index 51b3439..6e663ad 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,6 +1,5 @@ use crate::models::DB; use crate::types::*; -use log::debug; #[macro_export] macro_rules! rate_limit { @@ -30,6 +29,7 @@ macro_rules! rate_limit { /// the specified group, the name of the group we're incrementing, and the /// amount to increment for this request. This should remain as 1 except for /// joined requests. +#[tracing::instrument(skip(db))] pub async fn update_rate_limit( db: DB<'_>, key_id: i32, @@ -41,11 +41,6 @@ pub async fn update_rate_limit( let timestamp = now.timestamp(); let time_window = timestamp - (timestamp % 60); - debug!( - "Incrementing rate limit for: {}-{} by {}", - key_id, group_name, incr_by - ); - let rows = db .query( "INSERT INTO