Update versions, some cleanup.

This commit is contained in:
Syfaro 2023-02-10 22:00:42 -05:00
parent 510da555e6
commit e1f5b02464
9 changed files with 997 additions and 419 deletions

View File

@ -2,61 +2,37 @@ name: Release
on: on:
push: push:
pull_request: tags:
- "*"
env: env:
CARGO_TERM_COLOR: always CARGO_TERM_COLOR: always
jobs: jobs:
release: release:
runs-on: ubuntu-20.04 runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@stable
- name: Cache target - uses: Swatinem/rust-cache@v2
uses: actions/cache@v2
with:
path: |
target/
key: ${{ runner.os }}-debug-${{ hashFiles('Cargo.lock') }}
- name: Get stable toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
- name: Build binaries - name: Build binaries
uses: actions-rs/cargo@v1
with:
command: build
args: --release
- name: Move binaries
run: | run: |
cargo build --release --bin bkapi
mv target/release/bkapi bkapi/bkapi mv target/release/bkapi bkapi/bkapi
- name: Upload binaries
uses: actions/upload-artifact@v2
with:
name: bkapi
path: bkapi/bkapi
- name: Login to GitHub Container Registry - name: Login to GitHub Container Registry
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata for Docker - name: Extract metadata for Docker
id: meta id: meta
uses: docker/metadata-action@v3 uses: docker/metadata-action@v4
with: with:
images: ghcr.io/syfaro/bkapi images: ghcr.io/syfaro/bkapi
- name: Build and push Docker image - name: Build and push Docker image
uses: docker/build-push-action@v2 uses: docker/build-push-action@v4
with: with:
context: . context: .
push: true push: true

1265
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -3,3 +3,8 @@ members = [
"bkapi", "bkapi",
"bkapi-client", "bkapi-client",
] ]
[workspace.metadata.release]
sign-tag = true
shared-version = true
tag-name = "v{{version}}"

View File

@ -3,6 +3,7 @@ name = "bkapi-client"
version = "0.1.0" version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"] authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018" edition = "2018"
publish = false
[dependencies] [dependencies]
tracing = "0.1" tracing = "0.1"
@ -14,7 +15,7 @@ opentelemetry-http = "0.7"
futures = "0.3" futures = "0.3"
reqwest = { version = "0.11", features = ["json"] } reqwest = { version = "0.11", features = ["json"] }
async-nats = "0.21" async-nats = "0.27"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"

View File

@ -77,7 +77,6 @@ impl BKApiClient {
/// Search for multiple hashes given a single maximum distance. /// Search for multiple hashes given a single maximum distance.
/// ///
/// Results are returned in the same order as given hashes. /// Results are returned in the same order as given hashes.
#[tracing::instrument(err, skip(self))]
pub async fn search_many( pub async fn search_many(
&self, &self,
hashes: &[i64], hashes: &[i64],

View File

@ -3,6 +3,7 @@ name = "bkapi"
version = "0.1.0" version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"] authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018" edition = "2018"
publish = false
[dependencies] [dependencies]
dotenvy = "0.15" dotenvy = "0.15"
@ -11,7 +12,7 @@ thiserror = "1"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tracing-unwrap = "0.9" tracing-unwrap = "0.10"
tracing-opentelemetry = "0.18" tracing-opentelemetry = "0.18"
opentelemetry = { version = "0.18", features = ["rt-tokio"] } opentelemetry = { version = "0.18", features = ["rt-tokio"] }
@ -32,9 +33,11 @@ serde_json = "1"
actix-web = "4" actix-web = "4"
actix-http = "3" actix-http = "3"
actix-service = "2" actix-service = "2"
tracing-actix-web = { version = "0.6", features = ["opentelemetry_0_18"] } tracing-actix-web = { version = "0.7", features = ["opentelemetry_0_18"] }
async-nats = "0.21" async-nats = "0.27"
foxlib = { git = "https://github.com/Syfaro/foxlib.git" }
[dependencies.sqlx] [dependencies.sqlx]
version = "0.6" version = "0.6"

View File

@ -1,4 +1,4 @@
FROM ubuntu:20.04 FROM ubuntu:22.04
EXPOSE 3000 EXPOSE 3000
COPY ./bkapi/bkapi /bin/bkapi COPY ./bkapi/bkapi /bin/bkapi
CMD ["/bin/bkapi"] CMD ["/bin/bkapi"]

View File

@ -1,18 +1,15 @@
use std::sync::Arc; use std::{net::SocketAddr, sync::Arc};
use actix_service::Service; use actix_service::Service;
use actix_web::{ use actix_web::{
get, get,
web::{self, Data}, web::{self, Data},
App, HttpResponse, HttpServer, Responder, App, HttpResponse, HttpServer,
}; };
use clap::Parser; use clap::Parser;
use futures::StreamExt; use futures::StreamExt;
use opentelemetry::KeyValue;
use prometheus::{Encoder, TextEncoder};
use sqlx::postgres::PgPoolOptions; use sqlx::postgres::PgPoolOptions;
use tracing::Instrument; use tracing::Instrument;
use tracing_subscriber::layer::SubscriberExt;
use tracing_unwrap::ResultExt; use tracing_unwrap::ResultExt;
mod tree; mod tree;
@ -42,12 +39,12 @@ struct Config {
#[clap(long, env, default_value = "127.0.0.1:3000")] #[clap(long, env, default_value = "127.0.0.1:3000")]
http_listen: String, http_listen: String,
/// Jaeger agent endpoint for span collection. /// Host to listen for metrics requests.
#[clap(long, env, default_value = "127.0.0.1:6831")] #[clap(long, env, default_value = "127.0.0.1:3001")]
jaeger_agent: String, metrics_host: SocketAddr,
/// Service name for spans. /// If logs should be output in JSON format and sent to otlp collector.
#[clap(long, env, default_value = "bkapi")] #[clap(long, env)]
service_name: String, json_logs: bool,
/// Database URL for fetching data. /// Database URL for fetching data.
#[clap(long, env)] #[clap(long, env)]
@ -77,10 +74,18 @@ async fn main() {
let _ = dotenvy::dotenv(); let _ = dotenvy::dotenv();
let config = Config::parse(); let config = Config::parse();
configure_tracing(&config);
foxlib::trace_init(foxlib::TracingConfig {
namespace: "bkapi",
name: "bkapi",
version: env!("CARGO_PKG_VERSION"),
otlp: config.json_logs,
});
tracing::info!("starting bkapi"); tracing::info!("starting bkapi");
let metrics_server = foxlib::MetricsServer::serve(config.metrics_host, false).await;
let tree = tree::Tree::new(); let tree = tree::Tree::new();
tracing::trace!("connecting to postgres"); tracing::trace!("connecting to postgres");
@ -136,7 +141,9 @@ async fn main() {
receiver receiver
.await .await
.expect_or_log("tree loading was dropped before completing"); .expect_or_log("tree loading was dropped before completing");
tracing::info!("initial tree loaded, starting server"); tracing::info!("initial tree loaded, starting server");
metrics_server.set_ready(true);
if let Some(client) = client { if let Some(client) = client {
let tree_clone = tree.clone(); let tree_clone = tree.clone();
@ -151,40 +158,6 @@ async fn main() {
start_server(config, tree).await.unwrap_or_log(); start_server(config, tree).await.unwrap_or_log();
} }
fn configure_tracing(config: &Config) {
opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
let env = std::env::var("ENVIRONMENT");
let env = if let Ok(env) = env.as_ref() {
env.as_str()
} else if cfg!(debug_assertions) {
"debug"
} else {
"release"
};
let tracer = opentelemetry_jaeger::new_agent_pipeline()
.with_endpoint(&config.jaeger_agent)
.with_service_name(&config.service_name)
.with_trace_config(opentelemetry::sdk::trace::config().with_resource(
opentelemetry::sdk::Resource::new(vec![
KeyValue::new("environment", env.to_owned()),
KeyValue::new("version", env!("CARGO_PKG_VERSION")),
]),
))
.install_batch(opentelemetry::runtime::Tokio)
.expect("otel jaeger pipeline could not be created");
let trace = tracing_opentelemetry::layer().with_tracer(tracer);
tracing::subscriber::set_global_default(
tracing_subscriber::Registry::default()
.with(tracing_subscriber::EnvFilter::from_default_env())
.with(trace)
.with(tracing_subscriber::fmt::layer()),
)
.expect("tracing could not be configured");
}
async fn start_server(config: Config, tree: tree::Tree) -> Result<(), Error> { async fn start_server(config: Config, tree: tree::Tree) -> Result<(), Error> {
let tree = Data::new(tree); let tree = Data::new(tree);
let config_data = Data::new(config.clone()); let config_data = Data::new(config.clone());
@ -217,8 +190,6 @@ async fn start_server(config: Config, tree: tree::Tree) -> Result<(), Error> {
.app_data(tree.clone()) .app_data(tree.clone())
.app_data(config_data.clone()) .app_data(config_data.clone())
.service(search) .service(search)
.service(health)
.service(metrics)
}) })
.bind(&config.http_listen) .bind(&config.http_listen)
.expect_or_log("bind failed") .expect_or_log("bind failed")
@ -227,22 +198,6 @@ async fn start_server(config: Config, tree: tree::Tree) -> Result<(), Error> {
.map_err(Error::Io) .map_err(Error::Io)
} }
#[get("/health")]
async fn health() -> impl Responder {
"OK"
}
#[get("/metrics")]
async fn metrics() -> HttpResponse {
let mut buffer = Vec::new();
let encoder = TextEncoder::new();
let metric_families = prometheus::gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
HttpResponse::Ok().body(buffer)
}
#[derive(Debug, serde::Deserialize)] #[derive(Debug, serde::Deserialize)]
struct Query { struct Query {
hash: i64, hash: i64,

View File

@ -166,7 +166,7 @@ struct Payload {
} }
/// Listen for incoming payloads from Postgres. /// Listen for incoming payloads from Postgres.
#[tracing::instrument(skip(conn, subscription, query, tree, initial))] #[tracing::instrument(skip_all)]
pub(crate) async fn listen_for_payloads_db( pub(crate) async fn listen_for_payloads_db(
conn: Pool<Postgres>, conn: Pool<Postgres>,
subscription: String, subscription: String,
@ -204,7 +204,7 @@ pub(crate) async fn listen_for_payloads_db(
} }
/// Listen for incoming payloads from NATS. /// Listen for incoming payloads from NATS.
#[tracing::instrument(skip(config, pool, client, tree, initial))] #[tracing::instrument(skip_all)]
pub(crate) async fn listen_for_payloads_nats( pub(crate) async fn listen_for_payloads_nats(
config: Config, config: Config,
pool: sqlx::PgPool, pool: sqlx::PgPool,