feature: add command to query tasks
This commit is contained in:
parent
71c11eaa84
commit
63c20cfc87
7 changed files with 184 additions and 8 deletions
92
Cargo.lock
generated
92
Cargo.lock
generated
|
|
@ -17,6 +17,19 @@ version = "2.0.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
|
checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ahash"
|
||||||
|
version = "0.8.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"getrandom 0.3.2",
|
||||||
|
"once_cell",
|
||||||
|
"version_check",
|
||||||
|
"zerocopy",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aho-corasick"
|
name = "aho-corasick"
|
||||||
version = "1.1.3"
|
version = "1.1.3"
|
||||||
|
|
@ -172,6 +185,12 @@ version = "3.17.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
|
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bytecount"
|
||||||
|
version = "0.6.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytemuck"
|
name = "bytemuck"
|
||||||
version = "1.23.0"
|
version = "1.23.0"
|
||||||
|
|
@ -1059,6 +1078,17 @@ version = "0.1.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "papergrid"
|
||||||
|
version = "0.15.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "30268a8d20c2c0d126b2b6610ab405f16517f6ba9f244d8c59ac2c512a8a1ce7"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
"bytecount",
|
||||||
|
"unicode-width",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking"
|
name = "parking"
|
||||||
version = "2.2.1"
|
version = "2.2.1"
|
||||||
|
|
@ -1174,6 +1204,28 @@ dependencies = [
|
||||||
"zerocopy",
|
"zerocopy",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error-attr2"
|
||||||
|
version = "2.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error2"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-error-attr2",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro2"
|
name = "proc-macro2"
|
||||||
version = "1.0.95"
|
version = "1.0.95"
|
||||||
|
|
@ -1252,6 +1304,7 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
|
"tabled",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
@ -1762,6 +1815,30 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tabled"
|
||||||
|
version = "0.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "228d124371171cd39f0f454b58f73ddebeeef3cef3207a82ffea1c29465aea43"
|
||||||
|
dependencies = [
|
||||||
|
"papergrid",
|
||||||
|
"tabled_derive",
|
||||||
|
"testing_table",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tabled_derive"
|
||||||
|
version = "0.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846"
|
||||||
|
dependencies = [
|
||||||
|
"heck",
|
||||||
|
"proc-macro-error2",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tempfile"
|
name = "tempfile"
|
||||||
version = "3.19.1"
|
version = "3.19.1"
|
||||||
|
|
@ -1775,6 +1852,15 @@ dependencies = [
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "testing_table"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0f8daae29995a24f65619e19d8d31dea5b389f3d853d8bf297bbf607cd0014cc"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-width",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror"
|
name = "thiserror"
|
||||||
version = "2.0.12"
|
version = "2.0.12"
|
||||||
|
|
@ -1972,6 +2058,12 @@ version = "0.1.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
|
checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-width"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "url"
|
name = "url"
|
||||||
version = "2.5.4"
|
version = "2.5.4"
|
||||||
|
|
|
||||||
|
|
@ -16,3 +16,4 @@ tracing = "0.1.41"
|
||||||
tracing-subscriber = { version = "0.3.19" , features = ["env-filter"]}
|
tracing-subscriber = { version = "0.3.19" , features = ["env-filter"]}
|
||||||
figment = { version = "0.10.19", features = ["env"] }
|
figment = { version = "0.10.19", features = ["env"] }
|
||||||
tracing-core = "0.1.33"
|
tracing-core = "0.1.33"
|
||||||
|
tabled = "0.19.0"
|
||||||
|
|
@ -65,6 +65,7 @@ pub enum Command {
|
||||||
/// Path to the file
|
/// Path to the file
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
},
|
},
|
||||||
|
Query,
|
||||||
#[clap(skip)]
|
#[clap(skip)]
|
||||||
None,
|
None,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
mod error;
|
mod error;
|
||||||
pub mod sql;
|
pub mod task_manager;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod readwise;
|
pub mod readwise;
|
||||||
|
|
||||||
|
|
|
||||||
15
src/main.rs
15
src/main.rs
|
|
@ -1,11 +1,15 @@
|
||||||
use clap::{CommandFactory, Parser};
|
use clap::{CommandFactory, Parser};
|
||||||
|
use figment::{
|
||||||
|
providers::{Env, Serialized},
|
||||||
|
Figment,
|
||||||
|
};
|
||||||
use readwise_bulk_upload::config::{Command, Config};
|
use readwise_bulk_upload::config::{Command, Config};
|
||||||
use readwise_bulk_upload::readwise::DocumentPayload;
|
use readwise_bulk_upload::readwise::DocumentPayload;
|
||||||
use readwise_bulk_upload::sql::TaskManager;
|
use readwise_bulk_upload::task_manager::TaskManager;
|
||||||
use readwise_bulk_upload::{Error, Result};
|
use readwise_bulk_upload::{Error, Result};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
|
use tabled::Table;
|
||||||
use tracing_subscriber;
|
use tracing_subscriber;
|
||||||
use figment::{Figment, providers::{Serialized, Env}};
|
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
|
|
@ -25,7 +29,6 @@ async fn main() -> Result<()> {
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run(command: &Command) -> Result<()> {
|
async fn run(command: &Command) -> Result<()> {
|
||||||
|
|
||||||
match command {
|
match command {
|
||||||
Command::LoadTasks { path } => {
|
Command::LoadTasks { path } => {
|
||||||
let file = File::open(path).map_err(|_| {
|
let file = File::open(path).map_err(|_| {
|
||||||
|
|
@ -41,6 +44,12 @@ async fn run(command: &Command) -> Result<()> {
|
||||||
|
|
||||||
task_manager.load_tasks(documents).await?;
|
task_manager.load_tasks(documents).await?;
|
||||||
}
|
}
|
||||||
|
Command::Query => {
|
||||||
|
let task_manager = TaskManager::new().await?;
|
||||||
|
let tasks = task_manager.get_tasks::<DocumentPayload>(None, 25).await?;
|
||||||
|
|
||||||
|
println!("{}", Table::new(tasks));
|
||||||
|
}
|
||||||
Command::None => {
|
Command::None => {
|
||||||
Config::command().print_help()?;
|
Config::command().print_help()?;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
|
use crate::task_manager::TaskPayload;
|
||||||
use chrono::{DateTime, Local};
|
use chrono::{DateTime, Local};
|
||||||
use serde::{Deserialize, Deserializer, de, Serialize};
|
use serde::{de, Deserialize, Deserializer, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use crate::sql::TaskPayload;
|
use std::fmt::Display;
|
||||||
|
|
||||||
#[derive(Deserialize, Serialize, Debug)]
|
#[derive(Deserialize, Serialize, Debug)]
|
||||||
pub struct DocumentPayload {
|
pub struct DocumentPayload {
|
||||||
|
|
@ -14,6 +15,16 @@ pub struct DocumentPayload {
|
||||||
location: String,
|
location: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Display for DocumentPayload {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(self).map_err(|_| std::fmt::Error)?
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl TaskPayload for DocumentPayload {
|
impl TaskPayload for DocumentPayload {
|
||||||
fn get_key(&self) -> String {
|
fn get_key(&self) -> String {
|
||||||
self.url.clone()
|
self.url.clone()
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,18 @@
|
||||||
use crate::Error;
|
use crate::Error;
|
||||||
|
use chrono::Utc;
|
||||||
use directories::ProjectDirs;
|
use directories::ProjectDirs;
|
||||||
|
use serde::de::DeserializeOwned;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode};
|
use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode};
|
||||||
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
|
use sqlx::{QueryBuilder, Sqlite, SqlitePool};
|
||||||
|
use std::fmt::Display;
|
||||||
|
use tabled::Tabled;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tracing::{info, instrument};
|
use tracing::{info, instrument};
|
||||||
|
|
||||||
static SQLITE_BIND_LIMIT: usize = 32766;
|
static SQLITE_BIND_LIMIT: usize = 32766;
|
||||||
|
|
||||||
#[derive(sqlx::Type)]
|
#[derive(sqlx::Type, Debug)]
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
pub enum TaskStatus {
|
pub enum TaskStatus {
|
||||||
Pending = 1,
|
Pending = 1,
|
||||||
|
|
@ -17,10 +21,50 @@ pub enum TaskStatus {
|
||||||
Failed = 4,
|
Failed = 4,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Display for TaskStatus {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
TaskStatus::Pending => {
|
||||||
|
write!(f, "Pending")
|
||||||
|
}
|
||||||
|
TaskStatus::InProgress => {
|
||||||
|
write!(f, "In Progress")
|
||||||
|
}
|
||||||
|
TaskStatus::Completed => {
|
||||||
|
write!(f, "Completed")
|
||||||
|
}
|
||||||
|
TaskStatus::Failed => {
|
||||||
|
write!(f, "Failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub trait TaskPayload {
|
pub trait TaskPayload {
|
||||||
fn get_key(&self) -> String;
|
fn get_key(&self) -> String;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(sqlx::FromRow, Tabled, Debug)]
|
||||||
|
pub struct Task<T: DeserializeOwned + std::fmt::Display> {
|
||||||
|
id: u32,
|
||||||
|
payload_key: String,
|
||||||
|
#[sqlx(json)]
|
||||||
|
#[tabled(skip)]
|
||||||
|
payload: T,
|
||||||
|
#[sqlx(rename = "status_id")]
|
||||||
|
status: TaskStatus,
|
||||||
|
created_at: chrono::DateTime<Utc>,
|
||||||
|
#[tabled(display = "display_option_date")]
|
||||||
|
updated_at: Option<chrono::DateTime<Utc>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn display_option_date(o: &Option<chrono::DateTime<Utc>>) -> String {
|
||||||
|
match o {
|
||||||
|
Some(s) => format!("{}", s),
|
||||||
|
None => String::from(""),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct TaskManager {
|
pub struct TaskManager {
|
||||||
pool: SqlitePool,
|
pool: SqlitePool,
|
||||||
|
|
@ -53,6 +97,25 @@ impl TaskManager {
|
||||||
Ok(pool)
|
Ok(pool)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn get_tasks<
|
||||||
|
T: DeserializeOwned + Send + Unpin + 'static + Display,
|
||||||
|
>(
|
||||||
|
&self,
|
||||||
|
status: Option<TaskStatus>,
|
||||||
|
limit: u16,
|
||||||
|
) -> crate::Result<Vec<Task<T>>> {
|
||||||
|
let tasks: Vec<Task<T>> = sqlx::query_as(
|
||||||
|
"select id, payload_key, payload, status_id, created_at, updated_at from tasks where status_id = ? order by ? limit ?",
|
||||||
|
)
|
||||||
|
.bind(status.unwrap_or(TaskStatus::Pending))
|
||||||
|
.bind("created_at DESC")
|
||||||
|
.bind(limit)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(tasks)
|
||||||
|
}
|
||||||
|
|
||||||
#[instrument(skip(self, values))]
|
#[instrument(skip(self, values))]
|
||||||
pub async fn load_tasks<T>(&self, values: Vec<T>) -> crate::Result<()>
|
pub async fn load_tasks<T>(&self, values: Vec<T>) -> crate::Result<()>
|
||||||
where
|
where
|
||||||
|
|
@ -67,7 +130,6 @@ impl TaskManager {
|
||||||
.map(|value| Ok((value.get_key(), serde_json::to_string(value)?)))
|
.map(|value| Ok((value.get_key(), serde_json::to_string(value)?)))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|
||||||
let mut affected_rows = 0;
|
let mut affected_rows = 0;
|
||||||
// Chunk the query by the size limit of bind params
|
// Chunk the query by the size limit of bind params
|
||||||
for chunk in args?.chunks(SQLITE_BIND_LIMIT / 3) {
|
for chunk in args?.chunks(SQLITE_BIND_LIMIT / 3) {
|
||||||
Loading…
Add table
Add a link
Reference in a new issue