From 3d12877e2718103aedde5d8e47d2fa28901d10da Mon Sep 17 00:00:00 2001 From: aleidk Date: Thu, 8 May 2025 16:52:01 -0400 Subject: [PATCH] wip: add logging capabilities --- Cargo.lock | 244 ++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 + src/config.rs | 115 ++++++++++++++++++++++-- src/main.rs | 26 ++++-- src/sql.rs | 12 ++- 5 files changed, 384 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b073fd0..87f2582 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -97,6 +106,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -154,6 +172,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytemuck" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" + [[package]] name = "byteorder" version = "1.5.0" @@ -426,6 +450,19 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "uncased", + "version_check", +] + [[package]] name = "flume" version = "0.11.1" @@ -795,6 +832,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -887,6 +930,15 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "md-5" version = "0.10.6" @@ -923,6 +975,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -991,6 +1053,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking" version = "2.2.1" @@ -1020,6 +1088,29 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1092,6 +1183,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "version_check", + "yansi", +] + [[package]] name = "quote" version = "1.0.40" @@ -1144,11 +1248,15 @@ dependencies = [ "chrono", "clap", "directories", + "figment", "serde", "serde_json", "sqlx", "thiserror", "tokio", + "tracing", + "tracing-core", + "tracing-subscriber", ] [[package]] @@ -1171,6 +1279,50 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "rsa" version = "0.9.8" @@ -1294,6 +1446,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1634,6 +1795,16 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -1727,6 +1898,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1735,6 +1936,15 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -1791,6 +2001,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1892,6 +2108,28 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.61.0" @@ -2120,6 +2358,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index 4c7dbb1..da95ddc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,7 @@ clap = { version = "4.5.37", features = ["derive"] } serde = { version = "1.0.219", features = ["derive"] } chrono = {version = "0.4.41", features = ["serde"]} serde_json = "1.0.140" +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19" , features = ["env-filter"]} +figment = { version = "0.10.19", features = ["env"] } +tracing-core = "0.1.33" diff --git a/src/config.rs b/src/config.rs index f1206aa..e2c047e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,13 +1,116 @@ -use std::path::PathBuf; use clap::Parser; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use tracing_core::{Level, LevelFilter}; -#[derive(Debug, Parser)] -pub struct Args { - path: PathBuf, +pub enum VerbosityFilter { + Off, + Error, + Warn, + Info, + Debug, + Trace, } -impl Args { +impl VerbosityFilter { + fn with_offset(&self, offset: i16) -> VerbosityFilter { + let value = match self { + Self::Off => 0_i16, + Self::Error => 1, + Self::Warn => 2, + Self::Info => 3, + Self::Debug => 4, + Self::Trace => 5, + }; + match value.saturating_add(offset) { + i16::MIN..=0 => Self::Off, + 1 => Self::Error, + 2 => Self::Warn, + 3 => Self::Info, + 4 => Self::Debug, + 5..=i16::MAX => Self::Trace, + } + } + +} +impl From for VerbosityFilter { + fn from(level: LevelFilter) -> Self { + match level { + LevelFilter::OFF => Self::Off, + LevelFilter::ERROR => Self::Error, + LevelFilter::WARN => Self::Warn, + LevelFilter::INFO => Self::Info, + LevelFilter::DEBUG => Self::Debug, + LevelFilter::TRACE => Self::Trace, + } + } +} + +impl From for Option { + fn from(filter: VerbosityFilter) -> Self { + match filter { + VerbosityFilter::Off => None, + VerbosityFilter::Error => Some(Level::ERROR), + VerbosityFilter::Warn => Some(Level::WARN), + VerbosityFilter::Info => Some(Level::INFO), + VerbosityFilter::Debug => Some(Level::DEBUG), + VerbosityFilter::Trace => Some(Level::TRACE), + } + } +} +impl From> for VerbosityFilter { + fn from(level: Option) -> Self { + match level { + None => Self::Off, + Some(Level::ERROR) => Self::Error, + Some(Level::WARN) => Self::Warn, + Some(Level::INFO) => Self::Info, + Some(Level::DEBUG) => Self::Debug, + Some(Level::TRACE) => Self::Trace, + } + } +} + +#[derive(Debug, Parser, Serialize, Deserialize)] +pub struct Config { + path: PathBuf, + #[arg( + long, + short = 'v', + action = clap::ArgAction::Count, + global = true, + help = "Increase logging verbosity", + )] + verbose: u8, + + #[arg( + long, + short = 'q', + action = clap::ArgAction::Count, + global = true, + help = "Decrease logging verbosity", + conflicts_with = "verbose", + )] + quiet: u8, +} + +impl Config { pub fn path(&self) -> &PathBuf { &self.path } -} \ No newline at end of file + /// Gets the filter that should be applied to the logger. + /// + pub fn filter(&self) -> VerbosityFilter { + let offset = self.verbose as i16 - self.quiet as i16; + VerbosityFilter::Error.with_offset(offset) + } + + pub fn tracing_level(&self) -> Option { + self.filter().into() + } + + /// Get the tracing level filter. + pub fn tracing_level_filter(&self) -> tracing_core::LevelFilter { + self.filter().into() + } +} diff --git a/src/main.rs b/src/main.rs index 2760c1c..6c1fe5e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,24 +1,34 @@ -use std::fs::File; use clap::Parser; -use readwise_bulk_upload::config::Args; +use readwise_bulk_upload::config::Config; use readwise_bulk_upload::readwise::DocumentPayload; -use readwise_bulk_upload::sql::{TaskManager}; +use readwise_bulk_upload::sql::TaskManager; use readwise_bulk_upload::{Error, Result}; +use std::fs::File; +use tracing_subscriber; +use figment::{Figment, providers::{Serialized, Env, Format}}; #[tokio::main] async fn main() -> Result<()> { - let args = Args::parse(); + let args: Config = Figment::new() + .merge(Serialized::defaults(Config::parse())) + .merge(Env::prefixed("APP_")) + .extract()?; - let file = File::open(args.path()) - .map_err(|_| Error::Runtime(format!( + tracing_subscriber::fmt() + .with_max_level(args.verbose) + .init(); + + let file = File::open(args.path()).map_err(|_| { + Error::Runtime(format!( r#"The file "{}" could not be open"#, args.path().display() - )))?; + )) + })?; let documents: Vec = serde_json::from_reader(file)?; let task_manager = TaskManager::new().await?; - + task_manager.load_tasks(documents).await?; Ok(()) diff --git a/src/sql.rs b/src/sql.rs index c4b26da..7d2a442 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -4,6 +4,7 @@ use serde::Serialize; use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode}; use sqlx::{QueryBuilder, Sqlite, SqlitePool}; use tokio::fs; +use tracing::{info, instrument}; static SQLITE_BIND_LIMIT: usize = 32766; @@ -20,6 +21,7 @@ pub trait TaskPayload { fn get_key(&self) -> String; } +#[derive(Debug)] pub struct TaskManager { pool: SqlitePool, } @@ -51,9 +53,10 @@ impl TaskManager { Ok(pool) } + #[instrument(skip(self, values))] pub async fn load_tasks(&self, values: Vec) -> crate::Result<()> where - T: TaskPayload + Serialize, + T: TaskPayload + Serialize + std::fmt::Debug, { let mut tx = self.pool.begin().await?; let mut builder: QueryBuilder<'_, Sqlite> = @@ -65,6 +68,7 @@ impl TaskManager { .collect(); + let mut affected_rows = 0; // Chunk the query by the size limit of bind params for chunk in args?.chunks(SQLITE_BIND_LIMIT / 3) { builder.push_values(chunk, |mut builder, item| { @@ -77,12 +81,14 @@ impl TaskManager { let query = builder.build(); - query.execute(&mut *tx).await?; + affected_rows += query.execute(&mut *tx).await?.rows_affected(); builder.reset(); } - + tx.commit().await?; + info!("{} rows inserted.", affected_rows); + Ok(()) } }