From afddcb1dad883a0467c5b5e80fef11a637b952cb Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 3 Aug 2022 20:20:12 +0200 Subject: [PATCH 01/39] Initial draft of using tracing as logging library --- .gitignore | 1 + src/bin/conserve/log.rs | 34 ++++++++ src/bin/{conserve.rs => conserve/main.rs} | 96 +++++++++++++++++------ src/errors.rs | 2 +- src/restore.rs | 2 +- 5 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 src/bin/conserve/log.rs rename src/bin/{conserve.rs => conserve/main.rs} (87%) diff --git a/.gitignore b/.gitignore index 7ad99065..50e57e23 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ perf.old mutants.out mutants.out.old .cargo/config.toml +test/ \ No newline at end of file diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs new file mode 100644 index 00000000..8fc4ac0f --- /dev/null +++ b/src/bin/conserve/log.rs @@ -0,0 +1,34 @@ +use std::path::PathBuf; + +use tracing::metadata::LevelFilter; +use tracing_subscriber::prelude::*; +use tracing_subscriber::Registry; +use tracing_subscriber::fmt; + +pub struct LoggingOptions { + pub file: Option, + pub level: tracing::Level, +} + +pub fn init(options: LoggingOptions) -> std::result::Result { + let subscriber = Registry::default() + .with( + fmt::Layer::default() + .with_target(false) + // FIXME: Don't pipe directly into stdout if we got a progress bar. + .with_writer(std::io::stdout) + .with_filter(LevelFilter::from(options.level)) + ); + + tracing::subscriber::set_global_default(subscriber) + .map_err(|_| "Failed to update global default logger".to_string())?; + + Ok(LogGuard{ }) +} + +/// Guards all logging activity. +/// When dropping the pending logs will be written synchronously +/// and all open handles closed. +pub struct LogGuard { + +} \ No newline at end of file diff --git a/src/bin/conserve.rs b/src/bin/conserve/main.rs similarity index 87% rename from src/bin/conserve.rs rename to src/bin/conserve/main.rs index 596a7a7e..89d59a58 100644 --- a/src/bin/conserve.rs +++ b/src/bin/conserve/main.rs @@ -13,17 +13,23 @@ //! Command-line entry point for Conserve backups. +use std::error::Error; use std::io::{BufWriter, Write}; use std::path::PathBuf; +use std::process::ExitCode; +use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; -use tracing::trace; +use log::{LoggingOptions, LogGuard}; +use tracing::{ trace, error, info, warn }; use conserve::backup::BackupOptions; use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; +mod log; + #[derive(Debug, Parser)] #[clap( name = "conserve", @@ -40,8 +46,13 @@ struct Args { no_progress: bool, /// Show debug trace to stdout. + // TODO: Allow specifying a log level instead of only a debug flag. #[clap(long, short = 'D', global = true)] debug: bool, + + /// Path to the output log file + #[clap(long, short = 'L', global = true)] + log_file: Option, } #[derive(Subcommand, Debug)] @@ -242,14 +253,15 @@ enum Debug { Unreferenced { archive: String }, } -enum ExitCode { +#[repr(u8)] +enum CommandExitCode { Ok = 0, Failed = 1, PartialCorruption = 2, } impl Command { - fn run(&self) -> Result { + fn run(&self) -> Result { let mut stdout = std::io::stdout(); match self { Command::Backup { @@ -269,7 +281,8 @@ impl Command { }; let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options)?; if !no_stats { - ui::println(&format!("Backup complete.\n{}", stats)); + info!("Backup complete."); + info!("{}", stats); } } Command::Debug(Debug::Blocks { archive }) => { @@ -313,7 +326,7 @@ impl Command { }, )?; if !no_stats { - ui::println(&format!("{}", stats)); + info!("{}", stats); } } Command::Diff { @@ -348,12 +361,12 @@ impl Command { }, )?; if !no_stats { - ui::println(&format!("{}", stats)); + info!("{}", stats); } } Command::Init { archive } => { Archive::create(open_transport(archive)?)?; - ui::println(&format!("Created new archive in {:?}", &archive)); + info!("Created new archive in {:?}", &archive); } Command::Ls { stos, @@ -400,7 +413,8 @@ impl Command { let stats = restore(&archive, destination, &options)?; if !no_stats { - ui::println(&format!("Restore complete.\n{}", stats)); + info!("Restore complete."); + info!("{}", stats); } } Command::Size { @@ -420,9 +434,9 @@ impl Command { .file_bytes }; if *bytes { - ui::println(&format!("{}", size)); + info!("{}", size); } else { - ui::println(&conserve::bytes_to_human_mb(size)); + info!("{}", &conserve::bytes_to_human_mb(size)); } } Command::Validate { @@ -438,10 +452,10 @@ impl Command { println!("{}", stats); } if stats.has_problems() { - ui::problem("Archive has some problems."); - return Ok(ExitCode::PartialCorruption); + warn!("Archive has some problems."); + return Ok(CommandExitCode::PartialCorruption); } else { - ui::println("Archive is OK."); + info!("Archive is OK."); } } Command::Versions { @@ -463,7 +477,7 @@ impl Command { conserve::show_versions(&archive, &options, &mut stdout)?; } } - Ok(ExitCode::Ok) + Ok(CommandExitCode::Ok) } } @@ -481,30 +495,62 @@ fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolic } } -fn main() { - let args = Args::parse(); - ui::enable_progress(!args.no_progress && !args.debug); +fn initialize_log(args: &Args) -> std::result::Result { + let file = args.log_file + .as_ref() + .map(|file| PathBuf::from_str(&file)) + .transpose() + .map_err(|_| "Unparseable log file path".to_string())?; + + let guard = log::init(LoggingOptions{ + file, + level: if args.debug { tracing::Level::TRACE } else { tracing::Level::INFO } + })?; + if args.debug { - tracing_subscriber::fmt::Subscriber::builder() - .with_max_level(tracing::Level::TRACE) - .init(); trace!("tracing enabled"); } + + Ok(guard) +} + +fn main() -> ExitCode { + let args = Args::parse(); + let _log_guard = match initialize_log(&args) { + Ok(guard) => guard, + Err(message) => { + eprintln!("Failed to initialize log system:"); + eprintln!("{}", message); + return ExitCode::from(4); + } + }; + + ui::enable_progress(!args.no_progress && !args.debug); let result = args.command.run(); - match result { + let exit_code = match result { Err(ref e) => { - ui::show_error(e); + error!("{}", e.to_string()); + + let mut cause: &dyn Error = e; + while let Some(c) = cause.source() { + error!(" caused by: {}", c); + cause = c; + } + // // TODO: Perhaps always log the traceback to a log file. + // // NOTE(WolverinDEV): May always log this as trace level? // if let Some(bt) = e.backtrace() { // if std::env::var("RUST_BACKTRACE") == Ok("1".to_string()) { // println!("{}", bt); // } // } // Avoid Rust redundantly printing the error. - std::process::exit(ExitCode::Failed as i32) + ExitCode::FAILURE } - Ok(code) => std::process::exit(code as i32), - } + Ok(code) => ExitCode::from(code as u8), + }; + + exit_code } #[test] diff --git a/src/errors.rs b/src/errors.rs index 16d45acd..c6b14c0e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -22,7 +22,7 @@ use crate::*; type IOError = std::io::Error; -/// Conserve specific error. +/// Conserve library specific error. #[derive(Debug, Error)] pub enum Error { #[error("Block file {hash:?} corrupt; actual hash {actual_hash:?}")] diff --git a/src/restore.rs b/src/restore.rs index 34dbdd78..e10c5e62 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -18,7 +18,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; -use filetime::{set_file_handle_times, set_symlink_file_times}; +use filetime::{set_file_handle_times}; use crate::band::BandSelectionPolicy; use crate::entry::Entry; From fd493886001dd4b50cb403f8d325b1e2d46122c7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 3 Aug 2022 20:59:24 +0200 Subject: [PATCH 02/39] Moved display functions in `show.rs` to the binary and removed all outputs to stdout --- src/bin/conserve/main.rs | 27 +++++++++---------- src/{ => bin/conserve}/show.rs | 49 +++++++++++++++++++--------------- src/lib.rs | 5 ---- src/ui.rs | 3 ++- 4 files changed, 42 insertions(+), 42 deletions(-) rename src/{ => bin/conserve}/show.rs (74%) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 89d59a58..2f9870ba 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -21,6 +21,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; +use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn }; use conserve::backup::BackupOptions; @@ -29,6 +30,7 @@ use conserve::RestoreOptions; use conserve::*; mod log; +mod show; #[derive(Debug, Parser)] #[clap( @@ -45,7 +47,7 @@ struct Args { #[clap(long, short = 'P', global = true)] no_progress: bool, - /// Show debug trace to stdout. + /// Set the log level to trace // TODO: Allow specifying a log level instead of only a debug flag. #[clap(long, short = 'D', global = true)] debug: bool, @@ -262,7 +264,6 @@ enum CommandExitCode { impl Command { fn run(&self) -> Result { - let mut stdout = std::io::stdout(); match self { Command::Backup { archive, @@ -282,33 +283,32 @@ impl Command { let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options)?; if !no_stats { info!("Backup complete."); - info!("{}", stats); + for line in format!("{}", stats).lines() { + info!("{}", line); + } } } Command::Debug(Debug::Blocks { archive }) => { - let mut bw = BufWriter::new(stdout); for hash in Archive::open(open_transport(archive)?)? .block_dir() .block_names()? { - writeln!(bw, "{}", hash)?; + info!("{}", hash); } } Command::Debug(Debug::Index { archive, backup }) => { let st = stored_tree_from_opt(archive, backup)?; - show::show_index_json(st.band(), &mut stdout)?; + show::show_index_json(st.band())?; } Command::Debug(Debug::Referenced { archive }) => { - let mut bw = BufWriter::new(stdout); let archive = Archive::open(open_transport(archive)?)?; for hash in archive.referenced_blocks(&archive.list_band_ids()?)? { - writeln!(bw, "{}", hash)?; + info!("{}", hash); } } Command::Debug(Debug::Unreferenced { archive }) => { - let mut bw = BufWriter::new(stdout); for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks()? { - writeln!(bw, "{}", hash)?; + info!("{}", hash); } } Command::Delete { @@ -344,7 +344,8 @@ impl Command { exclude, include_unchanged: *include_unchanged, }; - show_diff(diff(&st, <, &options)?, &mut stdout)?; + + show_diff(diff(&st, <, &options)?)?; } Command::Gc { archive, @@ -379,13 +380,11 @@ impl Command { show::show_entry_names( stored_tree_from_opt(archive, &stos.backup)? .iter_entries(Apath::root(), exclude)?, - &mut stdout, )?; } else { show::show_entry_names( LiveTree::open(stos.source.clone().unwrap())? .iter_entries(Apath::root(), exclude)?, - &mut stdout, )?; } } @@ -474,7 +473,7 @@ impl Command { start_time: !*short, backup_duration: !*short, }; - conserve::show_versions(&archive, &options, &mut stdout)?; + show_versions(&archive, &options)?; } } Ok(CommandExitCode::Ok) diff --git a/src/show.rs b/src/bin/conserve/show.rs similarity index 74% rename from src/show.rs rename to src/bin/conserve/show.rs index 3bc3b3d5..6dbc5207 100644 --- a/src/show.rs +++ b/src/bin/conserve/show.rs @@ -17,9 +17,13 @@ //! file (typically stdout). use std::borrow::Cow; -use std::io::{BufWriter, Write}; -use crate::*; +use conserve::ui::duration_to_hms; +use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree}; +use tracing::{warn, info}; + +/// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. +const TIMESTAMP_FORMAT: &str = "%F %T"; /// Options controlling the behavior of `show_versions`. #[derive(Default, Clone, Eq, PartialEq)] @@ -37,11 +41,10 @@ pub struct ShowVersionsOptions { pub utc: bool, } -/// Print a list of versions, one per line. +/// Prinat all available versions to the `tracing`. pub fn show_versions( archive: &Archive, options: &ShowVersionsOptions, - w: &mut dyn Write, ) -> Result<()> { let mut band_ids = archive.list_band_ids()?; if options.newest_first { @@ -49,7 +52,7 @@ pub fn show_versions( } for band_id in band_ids { if !(options.tree_size || options.start_time || options.backup_duration) { - writeln!(w, "{}", band_id)?; + info!("{}", band_id); continue; } let mut l: Vec = Vec::new(); @@ -57,14 +60,14 @@ pub fn show_versions( let band = match Band::open(archive, &band_id) { Ok(band) => band, Err(e) => { - ui::problem(&format!("Failed to open band {:?}: {:?}", band_id, e)); + warn!("Failed to open band {:?}: {:?}", band_id, e); continue; } }; let info = match band.get_info() { Ok(info) => info, Err(e) => { - ui::problem(&format!("Failed to read band tail {:?}: {:?}", band_id, e)); + warn!("Failed to read band tail {:?}: {:?}", band_id, e); continue; } }; @@ -72,11 +75,11 @@ pub fn show_versions( if options.start_time { let start_time = info.start_time; let start_time_str = if options.utc { - start_time.format(crate::TIMESTAMP_FORMAT) + start_time.format(TIMESTAMP_FORMAT) } else { start_time .with_timezone(&chrono::Local) - .format(crate::TIMESTAMP_FORMAT) + .format(TIMESTAMP_FORMAT) }; l.push(format!("{:<10}", start_time_str)); } @@ -85,7 +88,7 @@ pub fn show_versions( let duration_str: Cow = if info.is_closed { info.end_time .and_then(|et| (et - info.start_time).to_std().ok()) - .map(crate::ui::duration_to_hms) + .map(duration_to_hms) .map(Cow::Owned) .unwrap_or(Cow::Borrowed("unknown")) } else { @@ -95,7 +98,7 @@ pub fn show_versions( } if options.tree_size { - let tree_mb_str = crate::misc::bytes_to_human_mb( + let tree_mb_str = bytes_to_human_mb( archive .open_stored_tree(BandSelectionPolicy::Specified(band_id.clone()))? .size(Exclude::nothing())? @@ -104,34 +107,36 @@ pub fn show_versions( l.push(format!("{:>14}", tree_mb_str,)); } - writeln!(w, "{}", l.join(" "))?; + info!("{}", l.join(" ")); } Ok(()) } -pub fn show_index_json(band: &Band, w: &mut dyn Write) -> Result<()> { +pub fn show_index_json(band: &Band) -> Result<()> { // TODO: Maybe use https://docs.serde.rs/serde/ser/trait.Serializer.html#method.collect_seq. - let bw = BufWriter::new(w); let index_entries: Vec = band.index().iter_entries().collect(); - serde_json::ser::to_writer_pretty(bw, &index_entries) - .map_err(|source| Error::SerializeIndex { source }) + let json = serde_json::to_string_pretty(&index_entries) + .map_err(|source| conserve::Error::SerializeIndex { source })?; + for line in json.lines() { + info!("{}", line); + } + Ok(()) } -pub fn show_entry_names>(it: I, w: &mut dyn Write) -> Result<()> { - let mut bw = BufWriter::new(w); +pub fn show_entry_names>(it: I) -> Result<()> { for entry in it { - writeln!(bw, "{}", entry.apath())?; + info!("{}", entry.apath()); } Ok(()) } -pub fn show_diff>(diff: D, w: &mut dyn Write) -> Result<()> { +pub fn show_diff>(diff: D) -> Result<()> { // TODO: Consider whether the actual files have changed. // TODO: Summarize diff. // TODO: Optionally include unchanged files. - let mut bw = BufWriter::new(w); for de in diff { - writeln!(bw, "{}", de)?; + info!("{}", de); } + Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 47610c9e..635c2801 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,6 @@ pub mod live_tree; mod merge; pub(crate) mod misc; pub mod restore; -pub mod show; pub mod stats; mod stitch; mod stored_file; @@ -66,7 +65,6 @@ pub use crate::live_tree::{LiveEntry, LiveTree}; pub use crate::merge::{MergeTrees, MergedEntryKind}; pub use crate::misc::bytes_to_human_mb; pub use crate::restore::{restore, RestoreOptions, RestoreTree}; -pub use crate::show::{show_diff, show_versions, ShowVersionsOptions}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; @@ -97,9 +95,6 @@ const SMALL_FILE_CAP: u64 = 100_000; /// Target maximum uncompressed size for combined blocks. const TARGET_COMBINED_BLOCK_SIZE: usize = MAX_BLOCK_SIZE; -/// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. -const TIMESTAMP_FORMAT: &str = "%F %T"; - /// Temporary files in the archive have this prefix. const TMP_PREFIX: &str = "tmp"; diff --git a/src/ui.rs b/src/ui.rs index a301aded..b82e8944 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -93,7 +93,8 @@ pub(crate) fn compression_percent(s: &Sizes) -> i64 { } } -pub(crate) fn duration_to_hms(d: Duration) -> String { +// FIXME: Move into conserve binary +pub fn duration_to_hms(d: Duration) -> String { let elapsed_secs = d.as_secs(); if elapsed_secs >= 3600 { format!( From deb9d3ab16d1ff23248b6d56c86de930c5d08ab1 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 3 Aug 2022 21:05:36 +0200 Subject: [PATCH 03/39] Using log levels instead of a debug flag --- src/bin/conserve/main.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 2f9870ba..b0964762 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -14,7 +14,6 @@ //! Command-line entry point for Conserve backups. use std::error::Error; -use std::io::{BufWriter, Write}; use std::path::PathBuf; use std::process::ExitCode; use std::str::FromStr; @@ -48,12 +47,11 @@ struct Args { no_progress: bool, /// Set the log level to trace - // TODO: Allow specifying a log level instead of only a debug flag. - #[clap(long, short = 'D', global = true)] - debug: bool, - - /// Path to the output log file #[clap(long, short = 'L', global = true)] + log_level: Option, + + /// Path to the output log file + #[clap(long, short = 'F', global = true)] log_file: Option, } @@ -503,10 +501,10 @@ fn initialize_log(args: &Args) -> std::result::Result { let guard = log::init(LoggingOptions{ file, - level: if args.debug { tracing::Level::TRACE } else { tracing::Level::INFO } + level: args.log_level.unwrap_or(tracing::Level::INFO) })?; - if args.debug { + if args.log_level == Some(tracing::Level::TRACE) { trace!("tracing enabled"); } @@ -524,7 +522,7 @@ fn main() -> ExitCode { } }; - ui::enable_progress(!args.no_progress && !args.debug); + ui::enable_progress(!args.no_progress); let result = args.command.run(); let exit_code = match result { Err(ref e) => { From 2ef2b4bf8d00c90f36185cfe5ff7082f9bab9ef1 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 3 Aug 2022 21:07:19 +0200 Subject: [PATCH 04/39] Using own exit code enum for exit codes --- src/bin/conserve/main.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index b0964762..d5f1916c 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -15,7 +15,7 @@ use std::error::Error; use std::path::PathBuf; -use std::process::ExitCode; +use std::process::Termination; use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; @@ -254,14 +254,20 @@ enum Debug { } #[repr(u8)] -enum CommandExitCode { +enum ExitCode { Ok = 0, Failed = 1, PartialCorruption = 2, } +impl Termination for ExitCode { + fn report(self) -> std::process::ExitCode { + std::process::ExitCode::from(self as u8) + } +} + impl Command { - fn run(&self) -> Result { + fn run(&self) -> Result { match self { Command::Backup { archive, @@ -450,7 +456,7 @@ impl Command { } if stats.has_problems() { warn!("Archive has some problems."); - return Ok(CommandExitCode::PartialCorruption); + return Ok(ExitCode::PartialCorruption); } else { info!("Archive is OK."); } @@ -474,7 +480,7 @@ impl Command { show_versions(&archive, &options)?; } } - Ok(CommandExitCode::Ok) + Ok(ExitCode::Ok) } } @@ -518,7 +524,7 @@ fn main() -> ExitCode { Err(message) => { eprintln!("Failed to initialize log system:"); eprintln!("{}", message); - return ExitCode::from(4); + return ExitCode::Failed; } }; @@ -542,9 +548,9 @@ fn main() -> ExitCode { // } // } // Avoid Rust redundantly printing the error. - ExitCode::FAILURE + ExitCode::Failed } - Ok(code) => ExitCode::from(code as u8), + Ok(code) => code, }; exit_code From 2392651299a91183ddd1bd0d2f11b51d5a3ada14 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 5 Aug 2022 22:19:14 +0200 Subject: [PATCH 05/39] Removed personal test folder from .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 50e57e23..f19e623d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,4 @@ perf.data* perf.old mutants.out mutants.out.old -.cargo/config.toml -test/ \ No newline at end of file +.cargo/config.toml \ No newline at end of file From aba2603315ccde974d4b36e8ba22f55f0b8f9398 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 5 Aug 2022 22:20:50 +0200 Subject: [PATCH 06/39] Using a monitor to monitor the backup progress --- src/backup.rs | 89 ++++++++++++-------------------------- src/bin/conserve/main.rs | 9 +++- src/bin/conserve/show.rs | 87 ++++++++++++++++++++++++++++++++++++- src/gc_lock.rs | 4 +- src/test_fixtures.rs | 4 +- tests/api/backup.rs | 35 ++++++++------- tests/api/diff.rs | 2 +- tests/api/gc.rs | 6 +-- tests/api/old_archives.rs | 1 + tests/api/restore.rs | 2 +- tests/expensive/changes.rs | 2 +- 11 files changed, 152 insertions(+), 89 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index b250a26f..45e95869 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -18,6 +18,7 @@ use std::io::prelude::*; use std::{convert::TryInto, time::Instant}; use itertools::Itertools; +use tracing::{debug, Level}; use crate::blockdir::Address; use crate::io::read_with_retries; @@ -48,42 +49,18 @@ impl Default for BackupOptions { } } -// This causes us to walk the source tree twice, which is probably an acceptable option -// since it's nice to see realistic overall progress. We could keep all the entries -// in memory, and maybe we should, but it might get unreasonably big. -// if options.measure_first { -// progress_bar.set_phase("Measure source tree".to_owned()); -// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it -// // again a second time? But, that'll potentially use memory proportional to tree size, which -// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were -// // deleted or changed while this is running. -// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); -// } - -#[derive(Default)] -struct ProgressModel { - filename: String, - scanned_file_bytes: u64, - scanned_dirs: usize, - scanned_files: usize, - entries_new: usize, - entries_changed: usize, - entries_unchanged: usize, - entries_deleted: usize, -} +/// Monitor the backup progress. +pub trait BackupMonitor { + /// Will be called before the entry will be backupped + fn copy(&mut self, _entry: &LiveEntry) {} + fn copy_error(&mut self, _entry: &LiveEntry, _error: &Error) {} + fn copy_result(&mut self, _entry: &LiveEntry, _result: &Option) {} -impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", - self.scanned_dirs, - self.scanned_files, - self.scanned_file_bytes / 1_000_000, - self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, - self.filename - ) - } -} + fn finished(&mut self, _stats: &BackupStats) {} +} + +struct DefaultMonitor {} +impl BackupMonitor for DefaultMonitor {} /// Backup a source directory into a new band in the archive. /// @@ -92,53 +69,43 @@ pub fn backup( archive: &Archive, source: &LiveTree, options: &BackupOptions, + monitor: Option<&mut dyn BackupMonitor>, ) -> Result { + let _span = tracing::span!(Level::DEBUG, "backup"); + + let mut default_monitor = DefaultMonitor{}; + let monitor = monitor.unwrap_or(&mut default_monitor); + let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; let mut stats = BackupStats::default(); - let mut view = nutmeg::View::new(ProgressModel::default(), ui::nutmeg_options()); let entry_iter = source.iter_entries(Apath::root(), options.exclude.clone())?; for entry_group in entry_iter.chunks(options.max_entries_per_hunk).into_iter() { for entry in entry_group { - view.update(|model| { - model.filename = entry.apath().to_string(); - match entry.kind() { - Kind::Dir => model.scanned_dirs += 1, - Kind::File => model.scanned_files += 1, - _ => (), - } - }); + monitor.copy(&entry); + match writer.copy_entry(&entry, source) { Err(e) => { - writeln!(view, "{}", ui::format_error_causes(&e))?; + debug!("{}", ui::format_error_causes(&e)); + monitor.copy_error(&entry, &e); stats.errors += 1; continue; } - Ok(Some(diff_kind)) => { - if options.print_filenames && diff_kind != DiffKind::Unchanged { - writeln!(view, "{} {}", diff_kind.as_sigil(), entry.apath())?; - } - view.update(|model| match diff_kind { - DiffKind::Changed => model.entries_changed += 1, - DiffKind::New => model.entries_new += 1, - DiffKind::Unchanged => model.entries_unchanged += 1, - DiffKind::Deleted => model.entries_deleted += 1, - }) - } - Ok(_) => {} - } - if let Some(bytes) = entry.size() { - if bytes > 0 { - view.update(|model| model.scanned_file_bytes += bytes) + Ok(result) => { + monitor.copy_result(&entry, &result); } } } writer.flush_group()?; } + stats += writer.finish()?; stats.elapsed = start.elapsed(); + // TODO: Merge in stats from the source tree? + + monitor.finished(&stats); Ok(stats) } diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index d5f1916c..0b3d3b11 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,6 +20,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; +use show::{BackupProgressModel, NutmegBackupMonitor}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn }; @@ -284,7 +285,13 @@ impl Command { exclude, ..Default::default() }; - let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options)?; + + // FIXME: Sync stdout with this view + // FIXME: Use cli flag! + let view = nutmeg::View::new(BackupProgressModel::default(), nutmeg::Options::default().progress_enabled(true)); + let mut monitor = NutmegBackupMonitor::new(&view); + + let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options, Some(&mut monitor))?; if !no_stats { info!("Backup complete."); for line in format!("{}", stats).lines() { diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 6dbc5207..e33eb798 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -18,9 +18,11 @@ use std::borrow::Cow; +use conserve::backup::BackupMonitor; use conserve::ui::duration_to_hms; -use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree}; +use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree, Kind, Entry, DiffKind}; use tracing::{warn, info}; +use nutmeg::View; /// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. const TIMESTAMP_FORMAT: &str = "%F %T"; @@ -140,3 +142,86 @@ pub fn show_diff>(diff: D) -> Result<()> { Ok(()) } + +// Considerations if we're trying properly extimate the remaining progress. +// +// This causes us to walk the source tree twice, which is probably an acceptable option +// since it's nice to see realistic overall progress. We could keep all the entries +// in memory, and maybe we should, but it might get unreasonably big. +// if options.measure_first { +// progress_bar.set_phase("Measure source tree".to_owned()); +// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it +// // again a second time? But, that'll potentially use memory proportional to tree size, which +// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were +// // deleted or changed while this is running. +// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); +// } + +#[derive(Default)] +pub struct BackupProgressModel { + filename: String, + scanned_file_bytes: u64, + scanned_dirs: usize, + scanned_files: usize, + entries_new: usize, + entries_changed: usize, + entries_unchanged: usize, + entries_deleted: usize, +} + +impl nutmeg::Model for BackupProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", + self.scanned_dirs, + self.scanned_files, + self.scanned_file_bytes / 1_000_000, + self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, + self.filename + ) + } +} + +pub struct NutmegBackupMonitor<'a> { + view: &'a View, +} + +impl<'a> NutmegBackupMonitor<'a> { + pub fn new(view: &'a View) -> Self { + Self { view } + } +} + +impl BackupMonitor for NutmegBackupMonitor<'_> { + fn copy(&mut self, entry: &conserve::LiveEntry) { + self.view.update(|model| { + model.filename = entry.apath().to_string(); + match entry.kind() { + Kind::Dir => model.scanned_dirs += 1, + Kind::File => model.scanned_files += 1, + _ => (), + } + }); + } + + fn copy_result(&mut self, entry: &conserve::LiveEntry, result: &Option) { + if let Some(diff_kind) = result.as_ref() { + self.view.update(|model| match diff_kind { + &DiffKind::Changed => model.entries_changed += 1, + &DiffKind::New => model.entries_new += 1, + &DiffKind::Unchanged => model.entries_unchanged += 1, + &DiffKind::Deleted => model.entries_deleted += 1, + }) + } + + if let Some(size) = entry.size() { + self.view.update(|model| model.scanned_file_bytes += size); + } + } + + fn copy_error(&mut self, entry: &conserve::LiveEntry, _error: &conserve::Error) { + if let Some(size) = entry.size() { + self.view.update(|model| model.scanned_file_bytes += size); + } + } +} \ No newline at end of file diff --git a/src/gc_lock.rs b/src/gc_lock.rs index b6c253a1..b2219ed3 100644 --- a/src/gc_lock.rs +++ b/src/gc_lock.rs @@ -126,7 +126,7 @@ mod test { fn completed_backup_ok() { let archive = ScratchArchive::new(); let source = TreeFixture::new(); - backup(&archive, &source.live_tree(), &BackupOptions::default()).unwrap(); + backup(&archive, &source.live_tree(), &BackupOptions::default(), None).unwrap(); let delete_guard = GarbageCollectionLock::new(&archive).unwrap(); delete_guard.check().unwrap(); } @@ -136,7 +136,7 @@ mod test { let archive = ScratchArchive::new(); let source = TreeFixture::new(); let _delete_guard = GarbageCollectionLock::new(&archive).unwrap(); - let backup_result = backup(&archive, &source.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, &source.live_tree(), &BackupOptions::default(), None); assert_eq!( backup_result.err().expect("backup fails").to_string(), "Archive is locked for garbage collection" diff --git a/src/test_fixtures.rs b/src/test_fixtures.rs index 4f792462..05f7440d 100644 --- a/src/test_fixtures.rs +++ b/src/test_fixtures.rs @@ -65,10 +65,10 @@ impl ScratchArchive { } let options = &BackupOptions::default(); - backup(&self.archive, &srcdir.live_tree(), options).unwrap(); + backup(&self.archive, &srcdir.live_tree(), options, None).unwrap(); srcdir.create_file("hello2"); - backup(&self.archive, &srcdir.live_tree(), options).unwrap(); + backup(&self.archive, &srcdir.live_tree(), options, None).unwrap(); } pub fn transport(&self) -> &dyn Transport { diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 9c5a29f3..8333c015 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -31,7 +31,7 @@ pub fn simple_backup() { let srcdir = TreeFixture::new(); srcdir.create_file("hello"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(copy_stats.index_builder_stats.index_hunks, 1); assert_eq!(copy_stats.files, 1); assert_eq!(copy_stats.deduplicated_blocks, 0); @@ -67,7 +67,7 @@ pub fn simple_backup_with_excludes() -> Result<()> { exclude, ..BackupOptions::default() }; - let copy_stats = backup(&af, &source, &options).expect("backup"); + let copy_stats = backup(&af, &source, &options, None).expect("backup"); check_backup(&af); @@ -124,7 +124,7 @@ pub fn backup_more_excludes() { print_filenames: false, ..Default::default() }; - let stats = backup(&af, &source, &options).expect("backup"); + let stats = backup(&af, &source, &options, None).expect("backup"); assert_eq!(1, stats.written_blocks); assert_eq!(1, stats.files); @@ -186,7 +186,7 @@ fn large_file() { let large_content = vec![b'a'; 4 << 20]; tf.create_file_with_contents("large", &large_content); - let backup_stats = backup(&af, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let backup_stats = backup(&af, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(backup_stats.new_files, 1); // First 1MB should be new; remainder should be deduplicated. assert_eq!(backup_stats.uncompressed_bytes, 1 << 20); @@ -220,7 +220,7 @@ fn source_unreadable() { tf.make_file_unreadable("b_unreadable"); - let stats = backup(&af, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(stats.errors, 1); assert_eq!(stats.new_files, 3); assert_eq!(stats.files, 3); @@ -260,7 +260,7 @@ pub fn symlink() { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_symlink("symlink", "/a/broken/destination"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(0, copy_stats.files); assert_eq!(1, copy_stats.symlinks); @@ -289,7 +289,7 @@ pub fn empty_file_uses_zero_blocks() { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_file_with_contents("empty", &[]); - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).unwrap(); assert_eq!(1, stats.files); assert_eq!(stats.written_blocks, 0); @@ -321,7 +321,7 @@ pub fn detect_unmodified() { srcdir.create_file("bbb"); let options = BackupOptions::default(); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &options, None).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 2); @@ -329,7 +329,7 @@ pub fn detect_unmodified() { // Make a second backup from the same tree, and we should see that // both files are unmodified. - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &options, None).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 0); @@ -339,7 +339,7 @@ pub fn detect_unmodified() { // as unmodified. srcdir.create_file_with_contents("bbb", b"longer content for bbb"); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &options, None).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 0); @@ -355,7 +355,7 @@ pub fn detect_minimal_mtime_change() { srcdir.create_file_with_contents("bbb", b"longer content for bbb"); let options = BackupOptions::default(); - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &options, None).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.new_files, 2); @@ -377,7 +377,7 @@ pub fn detect_minimal_mtime_change() { } } - let stats = backup(&af, &srcdir.live_tree(), &options).unwrap(); + let stats = backup(&af, &srcdir.live_tree(), &options, None).unwrap(); assert_eq!(stats.files, 2); assert_eq!(stats.unmodified_files, 1); } @@ -389,7 +389,7 @@ fn small_files_combined_two_backups() { srcdir.create_file("file1"); srcdir.create_file("file2"); - let stats1 = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats1 = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).unwrap(); // Although the two files have the same content, we do not yet dedupe them // within a combined block, so the block is different to when one identical // file is stored alone. This could be fixed. @@ -401,7 +401,7 @@ fn small_files_combined_two_backups() { // Add one more file, also identical, but it is not combined with the previous blocks. // This is a shortcoming of the current dedupe approach. srcdir.create_file("file3"); - let stats2 = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).unwrap(); + let stats2 = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).unwrap(); assert_eq!(stats2.new_files, 1); assert_eq!(stats2.unmodified_files, 2); assert_eq!(stats2.written_blocks, 1); @@ -423,7 +423,7 @@ fn many_small_files_combined_to_one_block() { format!("something about {}", i).as_bytes(), ); } - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!( stats.index_builder_stats.index_hunks, 2, "expect exactly 2 hunks" @@ -469,7 +469,7 @@ pub fn mixed_medium_small_files_two_hunks() { srcdir.create_file(&name); } } - let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default()).expect("backup"); + let stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!( stats.index_builder_stats.index_hunks, 2, "expect exactly 2 hunks" @@ -515,6 +515,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, + None ) .unwrap(); assert_eq!(stats.new_files, 2); @@ -530,6 +531,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, + None ) .unwrap(); assert_eq!(stats.unmodified_files, 1); @@ -551,6 +553,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, + None ) .unwrap(); assert_eq!(stats.unmodified_files, 2, "both files are unmodified"); diff --git a/tests/api/diff.rs b/tests/api/diff.rs index 5c0409d8..c7da6d3e 100644 --- a/tests/api/diff.rs +++ b/tests/api/diff.rs @@ -22,7 +22,7 @@ fn diff_unchanged() { tf.create_file_with_contents("thing", b"contents of thing"); let lt = tf.live_tree(); - let stats = backup(&a, <, &BackupOptions::default()).unwrap(); + let stats = backup(&a, <, &BackupOptions::default(), None).unwrap(); assert_eq!(stats.new_files, 1); let st = a.open_stored_tree(BandSelectionPolicy::Latest).unwrap(); diff --git a/tests/api/gc.rs b/tests/api/gc.rs index 8a80c653..cc0d18b3 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -26,7 +26,7 @@ fn unreferenced_blocks() { .parse() .unwrap(); - let _copy_stats = backup(&archive, &tf.live_tree(), &BackupOptions::default()).expect("backup"); + let _copy_stats = backup(&archive, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); // Delete the band and index std::fs::remove_dir_all(archive.path().join("b0000")).unwrap(); @@ -98,7 +98,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { let lock1 = GarbageCollectionLock::new(&archive)?; // Backup should fail while gc lock is held. - let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default(), None); match backup_result { Err(Error::GarbageCollectionLockHeld) => (), other => panic!("unexpected result {:?}", other), @@ -115,7 +115,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { )?; // Backup should now succeed. - let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default()); + let backup_result = backup(&archive, &tf.live_tree(), &BackupOptions::default(), None); assert!(backup_result.is_ok()); Ok(()) diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index c7a02abb..ef4e6b64 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -165,6 +165,7 @@ fn restore_modify_backup() { print_filenames: true, ..Default::default() }, + None ) .expect("Backup modified tree"); diff --git a/tests/api/restore.rs b/tests/api/restore.rs index 21d3e4b0..ca7504ea 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -130,7 +130,7 @@ fn restore_symlink() { let mtime: FileTime = years_ago.into(); set_symlink_file_times(&srcdir.path().join("symlink"), mtime, mtime).unwrap(); - backup(&af, &srcdir.live_tree(), &Default::default()).unwrap(); + backup(&af, &srcdir.live_tree(), &Default::default(), None).unwrap(); let restore_dir = TempDir::new().unwrap(); restore(&af, restore_dir.path(), &Default::default()).unwrap(); diff --git a/tests/expensive/changes.rs b/tests/expensive/changes.rs index 02fa28e8..7dd14ac2 100644 --- a/tests/expensive/changes.rs +++ b/tests/expensive/changes.rs @@ -93,7 +93,7 @@ fn backup_sequential_changes(changes: &[TreeChange]) { max_entries_per_hunk: 3, ..BackupOptions::default() }; - backup(&archive, &tf.live_tree(), &options).unwrap(); + backup(&archive, &tf.live_tree(), &options, None).unwrap(); let snapshot = TempDir::new().unwrap(); cp_r::CopyOptions::default() .copy_tree(&tf.path(), snapshot.path()) From 1b9f1b031f79db743a3fae777e50b713e05276d7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 00:56:38 +0200 Subject: [PATCH 07/39] Adding an option to pipe logging output through nutmeg when showing a process bar --- src/bin/conserve/log.rs | 39 ++++++++++++++++++++++++++++++++++++++- src/bin/conserve/main.rs | 22 +++++++++++++++++++--- src/bin/conserve/show.rs | 22 +++++++++++++--------- src/ui.rs | 7 +++++-- 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 8fc4ac0f..2564f855 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -1,10 +1,47 @@ +use std::io::Write; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use lazy_static::__Deref; +use lazy_static::lazy_static; use tracing::metadata::LevelFilter; use tracing_subscriber::prelude::*; use tracing_subscriber::Registry; use tracing_subscriber::fmt; +struct TerminalWriter { } + +impl TerminalWriter { } + +lazy_static!{ + pub static ref TERMINAL_OUTPUT: Mutex>>> = Mutex::new( + Some(Arc::new(Mutex::new(std::io::stdout()))) + ); +} + +impl Write for TerminalWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let current_target = TERMINAL_OUTPUT.lock().expect("lock() should not fail"); + if let Some(target) = current_target.deref() { + let mut target = target.lock().expect("lock() should not fail"); + target.write(buf) + } else { + Ok(buf.len()) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + let output = TERMINAL_OUTPUT.lock().expect("lock() should not fail"); + if let Some(target) = output.deref() { + let mut target = target.lock().expect("lock() should not fail"); + target.flush() + } else { + Ok(()) + } + } +} + pub struct LoggingOptions { pub file: Option, pub level: tracing::Level, @@ -16,7 +53,7 @@ pub fn init(options: LoggingOptions) -> std::result::Result { fmt::Layer::default() .with_target(false) // FIXME: Don't pipe directly into stdout if we got a progress bar. - .with_writer(std::io::stdout) + .with_writer(|| TerminalWriter{}) .with_filter(LevelFilter::from(options.level)) ); diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 0b3d3b11..0659cb58 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -17,9 +17,10 @@ use std::error::Error; use std::path::PathBuf; use std::process::Termination; use std::str::FromStr; +use std::sync::{Arc, Mutex}; use clap::{Parser, StructOpt, Subcommand}; -use log::{LoggingOptions, LogGuard}; +use log::{LoggingOptions, LogGuard, TERMINAL_OUTPUT}; use show::{BackupProgressModel, NutmegBackupMonitor}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn }; @@ -288,10 +289,25 @@ impl Command { // FIXME: Sync stdout with this view // FIXME: Use cli flag! - let view = nutmeg::View::new(BackupProgressModel::default(), nutmeg::Options::default().progress_enabled(true)); - let mut monitor = NutmegBackupMonitor::new(&view); + let view = Arc::new( + Mutex::new( + nutmeg::View::new(BackupProgressModel::default(), nutmeg::Options::default().progress_enabled(true)) + ) + ); + let mut monitor = NutmegBackupMonitor::new(view.clone()); + let old_output = { + let mut output = TERMINAL_OUTPUT.lock().unwrap(); + output.replace(view.clone()) + }; let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options, Some(&mut monitor))?; + { + let mut output = TERMINAL_OUTPUT.lock().unwrap(); + *output = old_output; + }; + drop(monitor); + drop(view); + if !no_stats { info!("Backup complete."); for line in format!("{}", stats).lines() { diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index e33eb798..67d81f36 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -17,6 +17,7 @@ //! file (typically stdout). use std::borrow::Cow; +use std::sync::{Arc, Mutex}; use conserve::backup::BackupMonitor; use conserve::ui::duration_to_hms; @@ -182,19 +183,20 @@ impl nutmeg::Model for BackupProgressModel { } } -pub struct NutmegBackupMonitor<'a> { - view: &'a View, +pub struct NutmegBackupMonitor { + view: Arc>>, } -impl<'a> NutmegBackupMonitor<'a> { - pub fn new(view: &'a View) -> Self { +impl NutmegBackupMonitor { + pub fn new(view: Arc>>) -> Self { Self { view } } } -impl BackupMonitor for NutmegBackupMonitor<'_> { +impl BackupMonitor for NutmegBackupMonitor { fn copy(&mut self, entry: &conserve::LiveEntry) { - self.view.update(|model| { + let view = self.view.lock().expect("lock() should not fail"); + view.update(|model| { model.filename = entry.apath().to_string(); match entry.kind() { Kind::Dir => model.scanned_dirs += 1, @@ -205,8 +207,9 @@ impl BackupMonitor for NutmegBackupMonitor<'_> { } fn copy_result(&mut self, entry: &conserve::LiveEntry, result: &Option) { + let view = self.view.lock().expect("lock() should not fail"); if let Some(diff_kind) = result.as_ref() { - self.view.update(|model| match diff_kind { + view.update(|model| match diff_kind { &DiffKind::Changed => model.entries_changed += 1, &DiffKind::New => model.entries_new += 1, &DiffKind::Unchanged => model.entries_unchanged += 1, @@ -215,13 +218,14 @@ impl BackupMonitor for NutmegBackupMonitor<'_> { } if let Some(size) = entry.size() { - self.view.update(|model| model.scanned_file_bytes += size); + view.update(|model| model.scanned_file_bytes += size); } } fn copy_error(&mut self, entry: &conserve::LiveEntry, _error: &conserve::Error) { + let view = self.view.lock().expect("lock() should not fail"); if let Some(size) = entry.size() { - self.view.update(|model| model.scanned_file_bytes += size); + view.update(|model| model.scanned_file_bytes += size); } } } \ No newline at end of file diff --git a/src/ui.rs b/src/ui.rs index b82e8944..7f15b05a 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -18,6 +18,7 @@ use std::sync::Mutex; use std::time::Duration; use lazy_static::lazy_static; +use tracing::{debug, error}; use crate::stats::Sizes; @@ -128,17 +129,19 @@ pub(crate) fn compression_ratio(s: &Sizes) -> f64 { } } +// FIXME: Don't use these functions any more. +// Directly log to tracing or to the monitor. impl UIState { pub(crate) fn println(&mut self, s: &str) { // TODO: Go through Nutmeg instead... // self.clear_progress(); - println!("{}", s); + debug!("{}", s); } fn problem(&mut self, s: &str) { // TODO: Go through Nutmeg instead... // self.clear_progress(); - println!("conserve error: {}", s); + error!("{}", s); // Drawing this way makes messages leak from tests, for unclear reasons. // queue!( From 6e13c6ae0201454f26b85df55f9fdca780119cc6 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 16:07:07 +0200 Subject: [PATCH 08/39] Improved monitor handling and nutmeg logging --- src/backup.rs | 5 ++++- src/bin/conserve/log.rs | 34 ++++++++++++++++++++++++++++++++-- src/bin/conserve/main.rs | 40 ++++++++++++++++------------------------ src/bin/conserve/show.rs | 40 ++++++++++++++++++++++++++++++---------- 4 files changed, 82 insertions(+), 37 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 45e95869..88d7d0eb 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -87,7 +87,10 @@ pub fn backup( match writer.copy_entry(&entry, source) { Err(e) => { - debug!("{}", ui::format_error_causes(&e)); + for line in ui::format_error_causes(&e).lines() { + debug!("{}", line); + } + monitor.copy_error(&entry, &e); stats.errors += 1; continue; diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 2564f855..3fe0f450 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -2,8 +2,8 @@ use std::io::Write; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use std::ops::Deref; -use lazy_static::__Deref; use lazy_static::lazy_static; use tracing::metadata::LevelFilter; use tracing_subscriber::prelude::*; @@ -52,7 +52,6 @@ pub fn init(options: LoggingOptions) -> std::result::Result { .with( fmt::Layer::default() .with_target(false) - // FIXME: Don't pipe directly into stdout if we got a progress bar. .with_writer(|| TerminalWriter{}) .with_filter(LevelFilter::from(options.level)) ); @@ -68,4 +67,35 @@ pub fn init(options: LoggingOptions) -> std::result::Result { /// and all open handles closed. pub struct LogGuard { +} + +pub struct ViewLogGuard { + released: bool, + previous_logger: Option>>, +} + +impl ViewLogGuard { + fn restore_previous_(&mut self) { + if self.released { + return; + } + + self.released = true; + + let mut output = TERMINAL_OUTPUT.lock().unwrap(); + *output = self.previous_logger.take(); + } +} + +impl Drop for ViewLogGuard { + fn drop(&mut self) { + self.restore_previous_(); + } +} + +pub fn update_terminal_target(target: Arc>) -> ViewLogGuard { + let mut output = TERMINAL_OUTPUT.lock().unwrap(); + let previous_logger = output.replace(target); + + ViewLogGuard { previous_logger, released: false } } \ No newline at end of file diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 0659cb58..76643cdb 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -17,15 +17,14 @@ use std::error::Error; use std::path::PathBuf; use std::process::Termination; use std::str::FromStr; -use std::sync::{Arc, Mutex}; use clap::{Parser, StructOpt, Subcommand}; -use log::{LoggingOptions, LogGuard, TERMINAL_OUTPUT}; -use show::{BackupProgressModel, NutmegBackupMonitor}; +use log::{LoggingOptions, LogGuard}; +use show::{NutmegMonitor, BackupProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn }; -use conserve::backup::BackupOptions; +use conserve::backup::{BackupOptions, BackupMonitor}; use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; @@ -269,7 +268,7 @@ impl Termination for ExitCode { } impl Command { - fn run(&self) -> Result { + fn run(&self, args: &Args) -> Result { match self { Command::Backup { archive, @@ -287,27 +286,20 @@ impl Command { ..Default::default() }; - // FIXME: Sync stdout with this view - // FIXME: Use cli flag! - let view = Arc::new( - Mutex::new( - nutmeg::View::new(BackupProgressModel::default(), nutmeg::Options::default().progress_enabled(true)) - ) - ); - let mut monitor = NutmegBackupMonitor::new(view.clone()); - let old_output = { - let mut output = TERMINAL_OUTPUT.lock().unwrap(); - output.replace(view.clone()) + let mut monitor = if args.no_progress { + None + } else { + Some(NutmegMonitor::::new()) }; - let stats = backup(&Archive::open(open_transport(archive)?)?, source, &options, Some(&mut monitor))?; - { - let mut output = TERMINAL_OUTPUT.lock().unwrap(); - *output = old_output; - }; + let stats = backup( + &Archive::open(open_transport(archive)?)?, + source, + &options, + monitor.as_mut().map(|v| v as &mut dyn BackupMonitor) + )?; drop(monitor); - drop(view); - + if !no_stats { info!("Backup complete."); for line in format!("{}", stats).lines() { @@ -552,7 +544,7 @@ fn main() -> ExitCode { }; ui::enable_progress(!args.no_progress); - let result = args.command.run(); + let result = args.command.run(&args); let exit_code = match result { Err(ref e) => { error!("{}", e.to_string()); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 67d81f36..f265e229 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -17,7 +17,7 @@ //! file (typically stdout). use std::borrow::Cow; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use conserve::backup::BackupMonitor; use conserve::ui::duration_to_hms; @@ -25,6 +25,8 @@ use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_hum use tracing::{warn, info}; use nutmeg::View; +use crate::log::{ViewLogGuard, self}; + /// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. const TIMESTAMP_FORMAT: &str = "%F %T"; @@ -183,19 +185,37 @@ impl nutmeg::Model for BackupProgressModel { } } -pub struct NutmegBackupMonitor { - view: Arc>>, +pub struct NutmegMonitor { + _log_guard: ViewLogGuard, + view: Arc>>, } -impl NutmegBackupMonitor { - pub fn new(view: Arc>>) -> Self { - Self { view } +impl NutmegMonitor { + pub fn new() -> Self { + let view = Arc::new( + Mutex::new( + nutmeg::View::new(T::default(), nutmeg::Options::default()) + ) + ); + + let log_guard = log::update_terminal_target(view.clone()); + Self { _log_guard: log_guard, view } + } +} + +impl NutmegMonitor { + pub fn view(&self) -> &Arc>> { + &self.view + } + + fn locked_view(&self) -> MutexGuard> { + self.view.lock().expect("lock() should not fail") } } -impl BackupMonitor for NutmegBackupMonitor { +impl BackupMonitor for NutmegMonitor { fn copy(&mut self, entry: &conserve::LiveEntry) { - let view = self.view.lock().expect("lock() should not fail"); + let view = self.locked_view(); view.update(|model| { model.filename = entry.apath().to_string(); match entry.kind() { @@ -207,7 +227,7 @@ impl BackupMonitor for NutmegBackupMonitor { } fn copy_result(&mut self, entry: &conserve::LiveEntry, result: &Option) { - let view = self.view.lock().expect("lock() should not fail"); + let view = self.locked_view(); if let Some(diff_kind) = result.as_ref() { view.update(|model| match diff_kind { &DiffKind::Changed => model.entries_changed += 1, @@ -223,7 +243,7 @@ impl BackupMonitor for NutmegBackupMonitor { } fn copy_error(&mut self, entry: &conserve::LiveEntry, _error: &conserve::Error) { - let view = self.view.lock().expect("lock() should not fail"); + let view = self.locked_view(); if let Some(size) = entry.size() { view.update(|model| model.scanned_file_bytes += size); } From fae7f356191f83f7989f573c6e04c16eceaa36d1 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 18:11:35 +0200 Subject: [PATCH 09/39] Removing terminal output from validate functions and using a monitor instead --- src/archive.rs | 34 ++++--- src/band.rs | 28 +++--- src/bin/conserve/log.rs | 4 +- src/bin/conserve/main.rs | 10 +- src/bin/conserve/show.rs | 187 +++++++++++++++++++++++++++++++++++++- src/blockdir.rs | 85 ++++++----------- src/lib.rs | 2 +- src/validate.rs | 126 ++++++++++++++++--------- tests/api/backup.rs | 2 +- tests/api/damaged.rs | 4 +- tests/api/old_archives.rs | 2 +- 11 files changed, 347 insertions(+), 137 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index f7e86913..8d6bf235 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -33,6 +33,7 @@ use crate::misc::remove_item; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; +use crate::validate::{ValidateMonitor, BlockMissingReason}; use crate::*; const HEADER_FILENAME: &str = "CONSERVE"; @@ -58,6 +59,9 @@ pub struct DeleteOptions { pub break_lock: bool, } +struct DefaultValidateMonitor {} +impl ValidateMonitor for DefaultValidateMonitor {} + impl Archive { /// Make a new archive in a local directory. pub fn create_path(path: &Path) -> Result { @@ -298,52 +302,58 @@ impl Archive { Ok(stats) } - pub fn validate(&self, options: &ValidateOptions) -> Result { + pub fn validate(&self, options: &ValidateOptions, monitor: Option<&mut dyn ValidateMonitor>) -> Result { + let mut default_monitor = DefaultValidateMonitor{}; + let monitor = monitor.unwrap_or(&mut default_monitor); + let start = Instant::now(); let mut stats = self.validate_archive_dir()?; - ui::println("Count indexes..."); + monitor.count_bands(); let band_ids = self.list_band_ids()?; - ui::println(&format!("Checking {} indexes...", band_ids.len())); + monitor.count_bands_result(&band_ids); + - // 1. Walk all indexes, collecting a list of (block_hash6, min_length) + // 1. Walk all indexes, collecting a list of (block_hash, min_length) // values referenced by all the indexes. - let (referenced_lens, ref_stats) = validate::validate_bands(self, &band_ids); + monitor.validate_bands(); + let (referenced_lens, ref_stats) = validate::validate_bands(self, &band_ids, monitor); stats += ref_stats; + monitor.validate_bands_finished(); + monitor.validate_blocks(); if options.skip_block_hashes { // 3a. Check that all referenced blocks are present, without spending time reading their // content. - ui::println("List present blocks..."); // TODO: Just validate blockdir structure. - let present_blocks: HashSet = self.block_dir.block_names_set()?; + let present_blocks: HashSet = self.block_dir.block_names_set(monitor)?; for block_hash in referenced_lens .0 .keys() .filter(|&bh| !present_blocks.contains(bh)) { - ui::problem(&format!("Block {:?} is missing", block_hash)); + monitor.validate_block_missing(&block_hash, &BlockMissingReason::NotExisting); stats.block_missing_count += 1; } } else { // 2. Check the hash of all blocks are correct, and remember how long // the uncompressed data is. - ui::println("Check blockdir..."); - let block_lengths: HashMap = self.block_dir.validate(&mut stats)?; + let block_lengths: HashMap = self.block_dir.validate(&mut stats, monitor)?; // 3b. Check that all referenced ranges are inside the present data. for (block_hash, referenced_len) in referenced_lens.0 { if let Some(actual_len) = block_lengths.get(&block_hash) { if referenced_len > (*actual_len as u64) { - ui::problem(&format!("Block {:?} is too short", block_hash,)); + monitor.validate_block_missing(&block_hash, &BlockMissingReason::InvalidRange); // TODO: A separate counter; this is worse than just being missing stats.block_missing_count += 1; } } else { - ui::problem(&format!("Block {:?} is missing", block_hash)); + monitor.validate_block_missing(&block_hash, &BlockMissingReason::NotExisting); stats.block_missing_count += 1; } } } + monitor.validate_blocks_finished(); stats.elapsed = start.elapsed(); Ok(stats) diff --git a/src/band.rs b/src/band.rs index 8654a651..9efcd27f 100644 --- a/src/band.rs +++ b/src/band.rs @@ -28,6 +28,7 @@ use crate::jsonio::{read_json, write_json}; use crate::misc::remove_item; use crate::transport::{ListDirNames, Transport}; use crate::*; +use crate::validate::{ValidateMonitor, BandProblem}; static INDEX_DIR: &str = "i"; @@ -215,34 +216,35 @@ impl Band { }) } - pub fn validate(&self, stats: &mut ValidateStats) -> Result<()> { - let ListDirNames { mut files, dirs } = + pub fn validate(&self, stats: &mut ValidateStats, monitor: &mut dyn ValidateMonitor) -> Result<()> { + let ListDirNames { mut files, mut dirs } = self.transport.list_dir_names("").map_err(Error::from)?; - if !files.contains(&BAND_HEAD_FILENAME.to_string()) { - ui::problem(&format!("No band head file in {:?}", self.transport)); + + let band_head_filename = BAND_HEAD_FILENAME.to_string(); + if !files.contains(&band_head_filename) { + monitor.validate_band_problem(self, &BandProblem::MissingHeadFile{ band_head_filename }); stats.missing_band_heads += 1; } remove_item(&mut files, &BAND_HEAD_FILENAME); remove_item(&mut files, &BAND_TAIL_FILENAME); + remove_item(&mut dirs, &INDEX_DIR); if !files.is_empty() { - ui::problem(&format!( - "Unexpected files in band directory {:?}: {:?}", - self.transport, files - )); + monitor.validate_band_problem(self, &BandProblem::UnexpectedFiles{ files }); stats.unexpected_files += 1; } - if dirs != [INDEX_DIR.to_string()] { - ui::problem(&format!( - "Incongruous directories in band directory {:?}: {:?}", - self.transport, dirs - )); + if !dirs.is_empty() { + monitor.validate_band_problem(self, &BandProblem::UnexpectedDirectories { directories: dirs }); stats.unexpected_files += 1; } Ok(()) } + + pub fn transport(&self) -> &Box { + &self.transport + } } #[cfg(test)] diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 3fe0f450..5f3d7612 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -75,7 +75,7 @@ pub struct ViewLogGuard { } impl ViewLogGuard { - fn restore_previous_(&mut self) { + fn restore_previous(&mut self) { if self.released { return; } @@ -89,7 +89,7 @@ impl ViewLogGuard { impl Drop for ViewLogGuard { fn drop(&mut self) { - self.restore_previous_(); + self.restore_previous(); } } diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 76643cdb..4da1f281 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -29,6 +29,8 @@ use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; +use crate::show::ValidateProgressModel; + mod log; mod show; @@ -465,9 +467,13 @@ impl Command { let options = ValidateOptions { skip_block_hashes: *quick, }; - let stats = Archive::open(open_transport(archive)?)?.validate(&options)?; + + let mut monitor = NutmegMonitor::::new(); + let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &mut dyn ValidateMonitor))?; + drop(monitor); + if !no_stats { - println!("{}", stats); + info!("{}", stats); } if stats.has_problems() { warn!("Archive has some problems."); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index f265e229..40c05821 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -18,10 +18,13 @@ use std::borrow::Cow; use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Instant, Duration}; use conserve::backup::BackupMonitor; +use conserve::stats::Sizes; use conserve::ui::duration_to_hms; -use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree, Kind, Entry, DiffKind}; +use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree, Kind, Entry, DiffKind, ValidateMonitor, BandProblem, BlockMissingReason}; +use thousands::Separable; use tracing::{warn, info}; use nutmeg::View; @@ -204,10 +207,6 @@ impl NutmegMonitor { } impl NutmegMonitor { - pub fn view(&self) -> &Arc>> { - &self.view - } - fn locked_view(&self) -> MutexGuard> { self.view.lock().expect("lock() should not fail") } @@ -248,4 +247,182 @@ impl BackupMonitor for NutmegMonitor { view.update(|model| model.scanned_file_bytes += size); } } +} + +enum ValidateProgressState { + CountBands, + ValidateBands { + bands_done: usize, + bands_total: usize, + start: Instant, + }, + ListBlockes { discovered: usize }, + ReadBlocks { + total_blocks: usize, + blocks_done: usize, + bytes_done: usize, + start: Instant, + }, +} + +pub struct ValidateProgressModel { + bands_total: Option, + state: ValidateProgressState +} + +impl Default for ValidateProgressModel { + fn default() -> Self { + Self { + bands_total: None, + state: ValidateProgressState::CountBands { } + } + } +} + +impl nutmeg::Model for ValidateProgressModel { + fn render(&mut self, _width: usize) -> String { + match &self.state { + ValidateProgressState::CountBands => "Counting bands".to_string(), + ValidateProgressState::ValidateBands { bands_done, bands_total, start } => { + format!( + "Check index {}/{}, {} done, {} remaining", + bands_done, + bands_total, + nutmeg::percent_done(*bands_done, *bands_total), + nutmeg::estimate_remaining(start, *bands_done, *bands_total) + ) + }, + ValidateProgressState::ListBlockes { discovered } => { + format!( + "Listing blocks ({} blocks discovered)", + discovered + ) + }, + ValidateProgressState::ReadBlocks { + total_blocks, + blocks_done, + bytes_done, + start + } => { + format!( + "Check block {}/{}: {} done, {} MB checked, {} remaining", + *blocks_done, + *total_blocks, + nutmeg::percent_done(*blocks_done, *total_blocks), + *bytes_done / 1_000_000, + nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) + ) + }, + } + } +} + +impl ValidateMonitor for NutmegMonitor { + fn count_bands(&mut self) { + info!("Count bands..."); + } + + fn count_bands_result(&mut self, bands: &[conserve::BandId]) { + info!("Checking {} bands...", bands.len()); + + let view = self.locked_view(); + view.update(|model| model.bands_total = Some(bands.len())); + } + + fn validate_bands(&mut self) { + let view = self.locked_view(); + view.update(|model| { + let bands_total = model.bands_total.expect("bands have been counted"); + model.state = ValidateProgressState::ValidateBands { + bands_done: 0, + bands_total, + start: Instant::now() + }; + }); + } + + fn validate_band_problem(&mut self, band: &Band, problem: &conserve::BandProblem) { + match problem { + BandProblem::MissingHeadFile { .. } => warn!("No band head file in {:?}", band.transport()), + BandProblem::UnexpectedFiles { files } => warn!("Unexpected files in band directory {:?}: {:?}", band.transport(), files), + BandProblem::UnexpectedDirectories { directories } => warn!("Incongruous directories in band directory {:?}: {:?}", band.transport(), directories), + } + } + + fn validate_band_result(&mut self, _band_id: &conserve::BandId, _result: &conserve::BandValidateResult) { + let view = self.locked_view(); + view.update(|model| { + if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { + *bands_done += 1; + } else { + panic!("Expected state ValidateProgressState::ValidateBands"); + } + }); + } + + fn validate_bands_finished(&mut self) { + let mut elapsed: Option = None; + + // We can't use logging while locked_view is held since we would deadlock. + { + let view = self.locked_view(); + view.update(|model| { + if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { + elapsed = Some(start.elapsed()); + } else { + panic!("Expected state ValidateProgressState::ValidateBands"); + } + }); + } + + info!("Finished validating bands in {:#?}.", elapsed.expect("elapsed to be set")); + } + + fn list_block_names(&mut self, current_count: usize) { + if current_count == 0 { + info!("Count blocks..."); + } + + let view = self.locked_view(); + view.update(|model| model.state = ValidateProgressState::ListBlockes { + discovered: current_count + }); + } + + fn read_blocks(&mut self, count: usize) { + info!("Check {} blocks...", count.separate_with_commas()); + + let view = self.locked_view(); + view.update(|model| model.state = ValidateProgressState::ReadBlocks { + total_blocks: count, + blocks_done: 0, + bytes_done: 0, + start: Instant::now(), + }); + } + + fn read_block_result(&mut self, _block_hash: &conserve::BlockHash, result: &Result<(Vec, Sizes)>) { + let view = self.locked_view(); + + view.update(|model| { + if let ValidateProgressState::ReadBlocks { blocks_done, bytes_done, .. } = &mut model.state { + if let Ok((bytes, _sizes)) = result { + *bytes_done += bytes.len(); + } else { + // TODO: Add a fail counter. + } + + *blocks_done += 1; + } else { + panic!("Expected state ValidateProgressState::ReadBlocks"); + } + }); + } + + fn validate_block_missing(&mut self, block_hash: &conserve::BlockHash, reason: &conserve::BlockMissingReason) { + match reason { + BlockMissingReason::NotExisting => warn!("Block {:?} is missing", block_hash), + BlockMissingReason::InvalidRange => warn!("Block {:?} is too short", block_hash), + } + } } \ No newline at end of file diff --git a/src/blockdir.rs b/src/blockdir.rs index 43423b4b..327ebaef 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -26,14 +26,11 @@ use std::convert::TryInto; use std::io; use std::path::Path; use std::sync::Arc; -use std::time::Instant; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; -use nutmeg::models::UnboundedModel; -use rayon::prelude::*; use serde::{Deserialize, Serialize}; -use thousands::Separable; +use tracing::warn; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; @@ -251,12 +248,17 @@ impl BlockDir { } /// Return all the blocknames in the blockdir. - pub fn block_names_set(&self) -> Result> { - let progress = nutmeg::View::new(UnboundedModel::new("List blocks"), ui::nutmeg_options()); + pub fn block_names_set(&self, monitor: &mut dyn ValidateMonitor) -> Result> { + let mut block_count = 0usize; + + monitor.list_block_names(block_count); Ok(self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok()) - .inspect(|_| progress.update(|model| model.increment(1))) + .inspect(|_| { + block_count += 1; + monitor.list_block_names(block_count); + }) .collect()) } @@ -264,67 +266,41 @@ impl BlockDir { /// /// Return a dict describing which blocks are present, and the length of their uncompressed /// data. - pub fn validate(&self, stats: &mut ValidateStats) -> Result> { + pub fn validate(&self, stats: &mut ValidateStats, monitor: &mut dyn ValidateMonitor) -> Result> { // TODO: In the top-level directory, no files or directories other than prefix // directories of the right length. // TODO: Test having a block with the right compression but the wrong contents. - ui::println("Count blocks..."); - let blocks = self.block_names_set()?; - crate::ui::println(&format!( - "Check {} blocks...", - blocks.len().separate_with_commas() - )); + let blocks = self.block_names_set(monitor)?; + monitor.read_blocks(blocks.len()); + stats.block_read_count = blocks.len().try_into().unwrap(); - struct ProgressModel { - total_blocks: usize, - blocks_done: usize, - bytes_done: usize, - start: Instant, - } - impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Check block {}/{}: {} done, {} MB checked, {} remaining", - self.blocks_done, - self.total_blocks, - nutmeg::percent_done(self.blocks_done, self.total_blocks), - self.bytes_done / 1_000_000, - nutmeg::estimate_remaining(&self.start, self.blocks_done, self.total_blocks) - ) - } - } - let progress_bar = nutmeg::View::new( - ProgressModel { - total_blocks: blocks.len(), - blocks_done: 0, - bytes_done: 0, - start: Instant::now(), - }, - ui::nutmeg_options(), - ); + // Make a vec of Some(usize) if the block could be read, or None if it // failed, where the usize gives the uncompressed data size. let results: Vec> = blocks - .into_par_iter() - // .into_iter() + //.into_par_iter() + .into_iter() .map(|hash| { - let r = self - .get_block_content(&hash) + let result = self.get_block_content(&hash); + // TODO(MH): Should we realy provide the block contents? May only return the size or the read error. + // This would also allow to read the blocks in parallel and use a simple iterator where + // we call the monitor. + monitor.read_block_result(&hash, &result); + + result .map(|(bytes, _sizes)| (hash, bytes.len())) - .ok(); - let bytes = r.as_ref().map(|x| x.1).unwrap_or_default(); - progress_bar.update(|model| { - model.blocks_done += 1; - model.bytes_done += bytes - }); - r + .ok() }) .collect(); + + // TODO: Only iter the results once. stats.block_error_count += results.iter().filter(|o| o.is_none()).count(); let len_map: HashMap = results .into_iter() .flatten() // keep only Some values .collect(); + + monitor.read_blocks_finished(); Ok(len_map) } @@ -350,10 +326,7 @@ impl BlockDir { decompressed_bytes, )); if actual_hash != *hash { - ui::problem(&format!( - "Block file {:?} has actual decompressed hash {}", - &block_relpath, actual_hash - )); + warn!("Block file {:?} has actual decompressed hash {}", &block_relpath, actual_hash); return Err(Error::BlockCorrupt { hash: hash.to_string(), actual_hash: actual_hash.to_string(), diff --git a/src/lib.rs b/src/lib.rs index 635c2801..12dceb5a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,7 +69,7 @@ pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; -pub use crate::validate::ValidateOptions; +pub use crate::validate::{ ValidateOptions, ValidateMonitor, BandProblem, BandValidateResult, BlockMissingReason }; pub type Result = std::result::Result; diff --git a/src/validate.rs b/src/validate.rs index 0d60b1f0..f0f5c177 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -12,12 +12,12 @@ use std::cmp::max; use std::collections::HashMap; -use std::time::Instant; use crate::blockdir::Address; use crate::*; +use crate::stats::Sizes; -pub(crate) struct BlockLengths(pub(crate) HashMap); +pub struct BlockLengths(pub HashMap); #[derive(Debug, Default)] pub struct ValidateOptions { @@ -25,6 +25,54 @@ pub struct ValidateOptions { pub skip_block_hashes: bool, } +pub trait ValidateMonitor { + fn count_bands(&mut self) {} + fn count_bands_result(&mut self, _bands: &[BandId]) {} + + fn validate_bands(&mut self) {} + fn validate_bands_finished(&mut self) {} + + fn validate_band(&mut self, _band_id: &BandId) {} + fn validate_band_problem(&mut self, _band: &Band, _problem: &BandProblem) {} + fn validate_band_result(&mut self, _band_id: &BandId, _result: &BandValidateResult) {} + + fn validate_block_missing(&mut self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} + fn validate_blocks(&mut self) {} + fn validate_blocks_finished(&mut self) {} + + fn list_block_names(&mut self, _current_count: usize) {} + fn list_block_names_finished(&mut self) {} + + fn read_blocks(&mut self, _count: usize) {} + fn read_block_result(&mut self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} + fn read_blocks_finished(&mut self) {} +} + +/// Band validation result. +pub enum BandValidateResult { + MetadataError(Error), + + OpenError(Error), + TreeOpenError(Error), + + TreeValidateError(Error), + + Valid(BlockLengths, ValidateStats), +} +pub enum BlockMissingReason { + /// The target bock can not be found. + NotExisting, + + /// The block reference points to an invalid data segment. + InvalidRange, +} + +pub enum BandProblem { + MissingHeadFile{ band_head_filename: String }, + UnexpectedFiles{ files: Vec }, + UnexpectedDirectories{ directories: Vec } +} + impl BlockLengths { fn new() -> BlockLengths { BlockLengths(HashMap::new()) @@ -52,58 +100,52 @@ impl BlockLengths { pub(crate) fn validate_bands( archive: &Archive, band_ids: &[BandId], + monitor: &mut dyn ValidateMonitor, ) -> (BlockLengths, ValidateStats) { let mut stats = ValidateStats::default(); let mut block_lens = BlockLengths::new(); - struct ProgressModel { - bands_done: usize, - bands_total: usize, - start: Instant, - } - impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Check index {}/{}, {} done, {} remaining", - self.bands_done, - self.bands_total, - nutmeg::percent_done(self.bands_done, self.bands_total), - nutmeg::estimate_remaining(&self.start, self.bands_done, self.bands_total) - ) - } - } - let view = nutmeg::View::new( - ProgressModel { - start: Instant::now(), - bands_done: 0, - bands_total: band_ids.len(), - }, - ui::nutmeg_options(), - ); + for band_id in band_ids { - if let Ok(b) = Band::open(archive, band_id) { - if b.validate(&mut stats).is_err() { - stats.band_metadata_problems += 1; - } - } else { - stats.band_open_errors += 1; - continue; - } - if let Ok(st) = archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { - if let Ok((st_block_lens, st_stats)) = validate_stored_tree(&st) { + monitor.validate_band(band_id); + let result = validate_band(archive, &mut stats, band_id, monitor); + monitor.validate_band_result(band_id, &result); + + match result { + BandValidateResult::MetadataError(_) => stats.band_metadata_problems += 1, + BandValidateResult::OpenError(_) => stats.band_open_errors += 1, + BandValidateResult::TreeOpenError(_) => stats.tree_open_errors += 1, + BandValidateResult::TreeValidateError(_) => stats.tree_validate_errors += 1, + BandValidateResult::Valid(st_block_lens, st_stats) => { stats += st_stats; block_lens.update(st_block_lens); - } else { - stats.tree_validate_errors += 1 } - } else { - stats.tree_open_errors += 1; - continue; } - view.update(|model| model.bands_done += 1); } + (block_lens, stats) } +pub(crate) fn validate_band(archive: &Archive, stats: &mut ValidateStats, band_id: &BandId, monitor: &mut dyn ValidateMonitor) -> BandValidateResult { + let band = match Band::open(archive, band_id) { + Ok(band) => band, + Err(error) => return BandValidateResult::OpenError(error) + }; + + if let Err(error) = band.validate(stats, monitor) { + return BandValidateResult::MetadataError(error); + } + + let stored_tree = match archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { + Ok(tree) => tree, + Err(error) => return BandValidateResult::TreeOpenError(error), + }; + + match validate_stored_tree(&stored_tree) { + Ok(result) => BandValidateResult::Valid(result.0, result.1), + Err(error) => BandValidateResult::TreeValidateError(error), + } +} + pub(crate) fn validate_stored_tree(st: &StoredTree) -> Result<(BlockLengths, ValidateStats)> { let mut block_lens = BlockLengths::new(); let stats = ValidateStats::default(); diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 8333c015..29b7a7c0 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -98,7 +98,7 @@ pub fn simple_backup_with_excludes() -> Result<()> { // TODO: Check index stats. // TODO: Check what was restored. - let validate_stats = af.validate(&ValidateOptions::default()).unwrap(); + let validate_stats = af.validate(&ValidateOptions::default(), None).unwrap(); assert!(!validate_stats.has_problems()); Ok(()) } diff --git a/tests/api/damaged.rs b/tests/api/damaged.rs index a42595a2..cd02098c 100644 --- a/tests/api/damaged.rs +++ b/tests/api/damaged.rs @@ -21,7 +21,7 @@ use conserve::*; fn missing_block() -> Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; - let validate_stats = archive.validate(&ValidateOptions::default())?; + let validate_stats = archive.validate(&ValidateOptions::default(), None)?; assert!(validate_stats.has_problems()); assert_eq!(validate_stats.block_missing_count, 1); Ok(()) @@ -33,7 +33,7 @@ fn missing_block_skip_block_hashes() -> Result<()> { let validate_stats = archive.validate(&ValidateOptions { skip_block_hashes: true, - })?; + }, None)?; assert!(validate_stats.has_problems()); assert_eq!(validate_stats.block_missing_count, 1); Ok(()) diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index ef4e6b64..b703c797 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -77,7 +77,7 @@ fn validate_archive() { let archive = open_old_archive(ver, "minimal"); let stats = archive - .validate(&ValidateOptions::default()) + .validate(&ValidateOptions::default(), None) .expect("validate archive"); assert_eq!(stats.structure_problems, 0); assert_eq!(stats.io_errors, 0); From 42fe67bcc7f50812dc206e28a905ea5c7bd19c75 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 18:39:14 +0200 Subject: [PATCH 10/39] Fixed missing set_symlink_file_times import for unix --- src/restore.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/restore.rs b/src/restore.rs index e10c5e62..4237b3cb 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -20,6 +20,9 @@ use std::{fs, time::Instant}; use filetime::{set_file_handle_times}; +#[cfg(unix)] +use filetime::{set_symlink_file_times}; + use crate::band::BandSelectionPolicy; use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; From 5cc17a7b3bd29cdc7967f2f5c2c6e6014fd5e872 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 18:40:15 +0200 Subject: [PATCH 11/39] Adding a file output writer to tracing subscribers allowing file logging --- src/bin/conserve/log.rs | 55 ++++++++++++++++++++++++++++++++-------- src/bin/conserve/main.rs | 17 +++++++++++-- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 5f3d7612..9f394a43 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -5,7 +5,9 @@ use std::sync::Mutex; use std::ops::Deref; use lazy_static::lazy_static; +use tracing::Subscriber; use tracing::metadata::LevelFilter; +use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::prelude::*; use tracing_subscriber::Registry; use tracing_subscriber::fmt; @@ -48,25 +50,58 @@ pub struct LoggingOptions { } pub fn init(options: LoggingOptions) -> std::result::Result { - let subscriber = Registry::default() - .with( - fmt::Layer::default() - .with_target(false) - .with_writer(|| TerminalWriter{}) - .with_filter(LevelFilter::from(options.level)) - ); + let mut worker_guard = None; + let registry = Registry::default(); - tracing::subscriber::set_global_default(subscriber) + // Terminal logger. + let registry = { + registry.with( + fmt::Layer::default() + .with_target(false) + .with_writer(|| TerminalWriter{}) + .with_filter(LevelFilter::from(options.level)) + ) + }; + + // File logger. + let registry: Box = if let Some(path) = options.file { + let directory = path.parent() + .ok_or("can't resolve log file directory")?; + + let file_name = path.file_name() + .ok_or("can't get log file name")? + .to_string_lossy() + .to_string(); + + let writer = tracing_appender::rolling::never(directory, file_name); + let (writer, guard) = tracing_appender::non_blocking(writer); + worker_guard = Some(guard); + + Box::new( + registry.with( + fmt::Layer::default() + .with_ansi(false) + .with_target(false) + .with_writer(writer) + .with_filter(LevelFilter::from(options.level)) + ) + ) + } else { + Box::new(registry) + }; + + + tracing::subscriber::set_global_default(registry) .map_err(|_| "Failed to update global default logger".to_string())?; - Ok(LogGuard{ }) + Ok(LogGuard{ _worker_guard: worker_guard }) } /// Guards all logging activity. /// When dropping the pending logs will be written synchronously /// and all open handles closed. pub struct LogGuard { - + _worker_guard: Option, } pub struct ViewLogGuard { diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 4da1f281..1a1d3a3b 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -22,7 +22,7 @@ use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; use show::{NutmegMonitor, BackupProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; -use tracing::{ trace, error, info, warn }; +use tracing::{ trace, error, info, warn, Level }; use conserve::backup::{BackupOptions, BackupMonitor}; use conserve::ReadTree; @@ -49,6 +49,10 @@ struct Args { #[clap(long, short = 'P', global = true)] no_progress: bool, + /// Show debug trace to stdout. + #[clap(long, short = 'D', global = true)] + debug: bool, + /// Set the log level to trace #[clap(long, short = 'L', global = true)] log_level: Option, @@ -468,6 +472,7 @@ impl Command { skip_block_hashes: *quick, }; + // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::::new(); let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &mut dyn ValidateMonitor))?; drop(monitor); @@ -526,9 +531,17 @@ fn initialize_log(args: &Args) -> std::result::Result { .transpose() .map_err(|_| "Unparseable log file path".to_string())?; + let level = args.log_level.unwrap_or({ + if args.debug { + Level::TRACE + } else { + Level::INFO + } + }); + let guard = log::init(LoggingOptions{ file, - level: args.log_level.unwrap_or(tracing::Level::INFO) + level })?; if args.log_level == Some(tracing::Level::TRACE) { From b12b6fef5ddc7a89a48edfa1ee92504c9bcecccc Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sat, 6 Aug 2022 18:46:28 +0200 Subject: [PATCH 12/39] Fixed a build error preventing the tests from building --- tests/api/backup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 29b7a7c0..447eae79 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -250,7 +250,7 @@ fn mtime_before_epoch() { assert_eq!(entries[1].apath(), "/old_file"); let af = ScratchArchive::new(); - backup(&af, &tf.live_tree(), &BackupOptions::default()) + backup(&af, &tf.live_tree(), &BackupOptions::default(), None) .expect("backup shouldn't crash on before-epoch mtimes"); } From dfbaf669fc4ac9b633902dd2726665bcb3664ec2 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Sun, 7 Aug 2022 01:51:39 +0200 Subject: [PATCH 13/39] Applying new monitor pattern for diretory size listing --- src/bin/conserve/main.rs | 11 ++- src/bin/conserve/show.rs | 183 ++++++++++++++++++++++++++------------- src/lib.rs | 2 +- src/tree.rs | 41 ++++----- 4 files changed, 148 insertions(+), 89 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 1a1d3a3b..fe143a54 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,7 +20,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; -use show::{NutmegMonitor, BackupProgressModel}; +use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn, Level }; @@ -448,15 +448,20 @@ impl Command { exclude_from, } => { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; + + // FIXME: Respect the "no progress" option and only log messages. + let mut monitor = NutmegMonitor::::new(); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? - .size(excludes)? + .size(excludes, Some(&mut monitor as &mut dyn TreeSizeMonitor<_>))? .file_bytes } else { LiveTree::open(stos.source.as_ref().unwrap())? - .size(excludes)? + .size(excludes, Some(&mut monitor as &mut dyn TreeSizeMonitor<_>))? .file_bytes }; + drop(monitor); + if *bytes { info!("{}", size); } else { diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 40c05821..163c0561 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -18,17 +18,20 @@ use std::borrow::Cow; use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::{Instant, Duration}; +use std::time::{Duration, Instant}; use conserve::backup::BackupMonitor; use conserve::stats::Sizes; use conserve::ui::duration_to_hms; -use conserve::{Archive, Result, Band, BandSelectionPolicy, Exclude, bytes_to_human_mb, IndexEntry, DiffEntry, ReadTree, Kind, Entry, DiffKind, ValidateMonitor, BandProblem, BlockMissingReason}; -use thousands::Separable; -use tracing::{warn, info}; +use conserve::{ + bytes_to_human_mb, Archive, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, + DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, ValidateMonitor, TreeSizeMonitor, +}; use nutmeg::View; +use thousands::Separable; +use tracing::{info, warn}; -use crate::log::{ViewLogGuard, self}; +use crate::log::{self, ViewLogGuard}; /// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. const TIMESTAMP_FORMAT: &str = "%F %T"; @@ -50,10 +53,7 @@ pub struct ShowVersionsOptions { } /// Prinat all available versions to the `tracing`. -pub fn show_versions( - archive: &Archive, - options: &ShowVersionsOptions, -) -> Result<()> { +pub fn show_versions(archive: &Archive, options: &ShowVersionsOptions) -> Result<()> { let mut band_ids = archive.list_band_ids()?; if options.newest_first { band_ids.reverse(); @@ -106,10 +106,11 @@ pub fn show_versions( } if options.tree_size { + // TODO(MH): Readd a monitor here to indicate progress let tree_mb_str = bytes_to_human_mb( archive .open_stored_tree(BandSelectionPolicy::Specified(band_id.clone()))? - .size(Exclude::nothing())? + .size(Exclude::nothing(), None)? .file_bytes, ); l.push(format!("{:>14}", tree_mb_str,)); @@ -145,7 +146,7 @@ pub fn show_diff>(diff: D) -> Result<()> { for de in diff { info!("{}", de); } - + Ok(()) } @@ -188,21 +189,23 @@ impl nutmeg::Model for BackupProgressModel { } } -pub struct NutmegMonitor { +pub struct NutmegMonitor { _log_guard: ViewLogGuard, view: Arc>>, } impl NutmegMonitor { pub fn new() -> Self { - let view = Arc::new( - Mutex::new( - nutmeg::View::new(T::default(), nutmeg::Options::default()) - ) - ); + let view = Arc::new(Mutex::new(nutmeg::View::new( + T::default(), + nutmeg::Options::default(), + ))); let log_guard = log::update_terminal_target(view.clone()); - Self { _log_guard: log_guard, view } + Self { + _log_guard: log_guard, + view, + } } } @@ -256,25 +259,27 @@ enum ValidateProgressState { bands_total: usize, start: Instant, }, - ListBlockes { discovered: usize }, + ListBlockes { + discovered: usize, + }, ReadBlocks { total_blocks: usize, - blocks_done: usize, - bytes_done: usize, - start: Instant, + blocks_done: usize, + bytes_done: usize, + start: Instant, }, } pub struct ValidateProgressModel { bands_total: Option, - state: ValidateProgressState + state: ValidateProgressState, } impl Default for ValidateProgressModel { fn default() -> Self { Self { bands_total: None, - state: ValidateProgressState::CountBands { } + state: ValidateProgressState::CountBands {}, } } } @@ -283,7 +288,11 @@ impl nutmeg::Model for ValidateProgressModel { fn render(&mut self, _width: usize) -> String { match &self.state { ValidateProgressState::CountBands => "Counting bands".to_string(), - ValidateProgressState::ValidateBands { bands_done, bands_total, start } => { + ValidateProgressState::ValidateBands { + bands_done, + bands_total, + start, + } => { format!( "Check index {}/{}, {} done, {} remaining", bands_done, @@ -291,18 +300,15 @@ impl nutmeg::Model for ValidateProgressModel { nutmeg::percent_done(*bands_done, *bands_total), nutmeg::estimate_remaining(start, *bands_done, *bands_total) ) - }, + } ValidateProgressState::ListBlockes { discovered } => { - format!( - "Listing blocks ({} blocks discovered)", - discovered - ) - }, - ValidateProgressState::ReadBlocks { - total_blocks, - blocks_done, - bytes_done, - start + format!("Listing blocks ({} blocks discovered)", discovered) + } + ValidateProgressState::ReadBlocks { + total_blocks, + blocks_done, + bytes_done, + start, } => { format!( "Check block {}/{}: {} done, {} MB checked, {} remaining", @@ -312,7 +318,7 @@ impl nutmeg::Model for ValidateProgressModel { *bytes_done / 1_000_000, nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) ) - }, + } } } } @@ -324,7 +330,7 @@ impl ValidateMonitor for NutmegMonitor { fn count_bands_result(&mut self, bands: &[conserve::BandId]) { info!("Checking {} bands...", bands.len()); - + let view = self.locked_view(); view.update(|model| model.bands_total = Some(bands.len())); } @@ -333,23 +339,37 @@ impl ValidateMonitor for NutmegMonitor { let view = self.locked_view(); view.update(|model| { let bands_total = model.bands_total.expect("bands have been counted"); - model.state = ValidateProgressState::ValidateBands { + model.state = ValidateProgressState::ValidateBands { bands_done: 0, - bands_total, - start: Instant::now() + bands_total, + start: Instant::now(), }; }); } fn validate_band_problem(&mut self, band: &Band, problem: &conserve::BandProblem) { match problem { - BandProblem::MissingHeadFile { .. } => warn!("No band head file in {:?}", band.transport()), - BandProblem::UnexpectedFiles { files } => warn!("Unexpected files in band directory {:?}: {:?}", band.transport(), files), - BandProblem::UnexpectedDirectories { directories } => warn!("Incongruous directories in band directory {:?}: {:?}", band.transport(), directories), + BandProblem::MissingHeadFile { .. } => { + warn!("No band head file in {:?}", band.transport()) + } + BandProblem::UnexpectedFiles { files } => warn!( + "Unexpected files in band directory {:?}: {:?}", + band.transport(), + files + ), + BandProblem::UnexpectedDirectories { directories } => warn!( + "Incongruous directories in band directory {:?}: {:?}", + band.transport(), + directories + ), } } - fn validate_band_result(&mut self, _band_id: &conserve::BandId, _result: &conserve::BandValidateResult) { + fn validate_band_result( + &mut self, + _band_id: &conserve::BandId, + _result: &conserve::BandValidateResult, + ) { let view = self.locked_view(); view.update(|model| { if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { @@ -375,7 +395,10 @@ impl ValidateMonitor for NutmegMonitor { }); } - info!("Finished validating bands in {:#?}.", elapsed.expect("elapsed to be set")); + info!( + "Finished validating bands in {:#?}.", + elapsed.expect("elapsed to be set") + ); } fn list_block_names(&mut self, current_count: usize) { @@ -384,28 +407,41 @@ impl ValidateMonitor for NutmegMonitor { } let view = self.locked_view(); - view.update(|model| model.state = ValidateProgressState::ListBlockes { - discovered: current_count + view.update(|model| { + model.state = ValidateProgressState::ListBlockes { + discovered: current_count, + } }); } fn read_blocks(&mut self, count: usize) { info!("Check {} blocks...", count.separate_with_commas()); - + let view = self.locked_view(); - view.update(|model| model.state = ValidateProgressState::ReadBlocks { - total_blocks: count, - blocks_done: 0, - bytes_done: 0, - start: Instant::now(), + view.update(|model| { + model.state = ValidateProgressState::ReadBlocks { + total_blocks: count, + blocks_done: 0, + bytes_done: 0, + start: Instant::now(), + } }); } - fn read_block_result(&mut self, _block_hash: &conserve::BlockHash, result: &Result<(Vec, Sizes)>) { + fn read_block_result( + &mut self, + _block_hash: &conserve::BlockHash, + result: &Result<(Vec, Sizes)>, + ) { let view = self.locked_view(); - + view.update(|model| { - if let ValidateProgressState::ReadBlocks { blocks_done, bytes_done, .. } = &mut model.state { + if let ValidateProgressState::ReadBlocks { + blocks_done, + bytes_done, + .. + } = &mut model.state + { if let Ok((bytes, _sizes)) = result { *bytes_done += bytes.len(); } else { @@ -419,10 +455,39 @@ impl ValidateMonitor for NutmegMonitor { }); } - fn validate_block_missing(&mut self, block_hash: &conserve::BlockHash, reason: &conserve::BlockMissingReason) { + fn validate_block_missing( + &mut self, + block_hash: &conserve::BlockHash, + reason: &conserve::BlockMissingReason, + ) { match reason { BlockMissingReason::NotExisting => warn!("Block {:?} is missing", block_hash), BlockMissingReason::InvalidRange => warn!("Block {:?} is too short", block_hash), } } -} \ No newline at end of file +} + +#[derive(Default)] +pub struct SizeProgressModel { + files: usize, + total_bytes: u64, +} +impl nutmeg::Model for SizeProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Measuring... {} files, {} MB", + self.files, + self.total_bytes / 1_000_000 + ) + } +} + +impl TreeSizeMonitor for NutmegMonitor { + fn entry_discovered(&mut self, _entry: &::Entry, size: &Option) { + let view = self.locked_view(); + view.update(|model| { + model.files += 1; + model.total_bytes += size.unwrap_or(0); + }); + } +} diff --git a/src/lib.rs b/src/lib.rs index 12dceb5a..5b8e1d12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,7 +68,7 @@ pub use crate::restore::{restore, RestoreOptions, RestoreTree}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; -pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; +pub use crate::tree::{ReadBlocks, ReadTree, TreeSize, TreeSizeMonitor}; pub use crate::validate::{ ValidateOptions, ValidateMonitor, BandProblem, BandValidateResult, BlockMissingReason }; pub type Result = std::result::Result; diff --git a/src/tree.rs b/src/tree.rs index e8953020..4f0a511a 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -18,6 +18,13 @@ use std::ops::Range; use crate::stats::Sizes; use crate::*; +pub trait TreeSizeMonitor { + fn entry_discovered(&mut self, _entry: &T::Entry, _size: &Option) {} +} + +struct DefaultTreeSizeMonitor {} +impl TreeSizeMonitor for DefaultTreeSizeMonitor {} + /// Abstract Tree that may be either on the real filesystem or stored in an archive. pub trait ReadTree { type Entry: Entry + 'static; @@ -42,36 +49,18 @@ pub trait ReadTree { /// Measure the tree size. /// /// This typically requires walking all entries, which may take a while. - fn size(&self, exclude: Exclude) -> Result { - struct Model { - files: usize, - total_bytes: u64, - } - impl nutmeg::Model for Model { - fn render(&mut self, _width: usize) -> String { - format!( - "Measuring... {} files, {} MB", - self.files, - self.total_bytes / 1_000_000 - ) - } - } - let progress = nutmeg::View::new( - Model { - files: 0, - total_bytes: 0, - }, - ui::nutmeg_options(), - ); + fn size(&self, exclude: Exclude, monitor: Option<&mut dyn TreeSizeMonitor>) -> Result where Self: Sized { + let mut default_monitor = DefaultTreeSizeMonitor{}; + let monitor = monitor.unwrap_or(&mut default_monitor as &mut dyn TreeSizeMonitor); + let mut tot = 0u64; for e in self.iter_entries(Apath::root(), exclude)? { + let size = e.size(); + monitor.entry_discovered(&e, &size); + // While just measuring size, ignore directories/files we can't stat. - if let Some(bytes) = e.size() { + if let Some(bytes) = size { tot += bytes; - progress.update(|model| { - model.files += 1; - model.total_bytes += bytes; - }); } } Ok(TreeSize { file_bytes: tot }) From c932a2052554e280ebb0203e16de870cc12a1d8f Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 12:15:13 +0200 Subject: [PATCH 14/39] Moved monitor definitions into a seperate file --- src/archive.rs | 6 ++---- src/backup.rs | 14 +----------- src/bin/conserve/log.rs | 2 ++ src/lib.rs | 6 ++++-- src/monitor.rs | 48 +++++++++++++++++++++++++++++++++++++++++ src/tree.rs | 10 ++------- src/validate.rs | 24 --------------------- 7 files changed, 59 insertions(+), 51 deletions(-) create mode 100644 src/monitor.rs diff --git a/src/archive.rs b/src/archive.rs index 8d6bf235..e14af289 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -30,6 +30,7 @@ use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; use crate::misc::remove_item; +use crate::monitor::DefaultMonitor; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; @@ -59,9 +60,6 @@ pub struct DeleteOptions { pub break_lock: bool, } -struct DefaultValidateMonitor {} -impl ValidateMonitor for DefaultValidateMonitor {} - impl Archive { /// Make a new archive in a local directory. pub fn create_path(path: &Path) -> Result { @@ -303,7 +301,7 @@ impl Archive { } pub fn validate(&self, options: &ValidateOptions, monitor: Option<&mut dyn ValidateMonitor>) -> Result { - let mut default_monitor = DefaultValidateMonitor{}; + let mut default_monitor = DefaultMonitor{}; let monitor = monitor.unwrap_or(&mut default_monitor); let start = Instant::now(); diff --git a/src/backup.rs b/src/backup.rs index 88d7d0eb..a64b4dc8 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -22,6 +22,7 @@ use tracing::{debug, Level}; use crate::blockdir::Address; use crate::io::read_with_retries; +use crate::monitor::DefaultMonitor; use crate::stats::BackupStats; use crate::stitch::IterStitchedIndexHunks; use crate::tree::ReadTree; @@ -49,19 +50,6 @@ impl Default for BackupOptions { } } -/// Monitor the backup progress. -pub trait BackupMonitor { - /// Will be called before the entry will be backupped - fn copy(&mut self, _entry: &LiveEntry) {} - fn copy_error(&mut self, _entry: &LiveEntry, _error: &Error) {} - fn copy_result(&mut self, _entry: &LiveEntry, _result: &Option) {} - - fn finished(&mut self, _stats: &BackupStats) {} -} - -struct DefaultMonitor {} -impl BackupMonitor for DefaultMonitor {} - /// Backup a source directory into a new band in the archive. /// /// Returns statistics about what was copied. diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 9f394a43..d0a6bf53 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -57,6 +57,8 @@ pub fn init(options: LoggingOptions) -> std::result::Result { let registry = { registry.with( fmt::Layer::default() + //.without_time() + //.with_level(false) .with_target(false) .with_writer(|| TerminalWriter{}) .with_filter(LevelFilter::from(options.level)) diff --git a/src/lib.rs b/src/lib.rs index 5b8e1d12..d7af3706 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,7 @@ mod tree; pub mod ui; pub mod unix_time; mod validate; +pub(crate) mod monitor; pub use crate::apath::Apath; pub use crate::archive::Archive; @@ -68,8 +69,9 @@ pub use crate::restore::{restore, RestoreOptions, RestoreTree}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; -pub use crate::tree::{ReadBlocks, ReadTree, TreeSize, TreeSizeMonitor}; -pub use crate::validate::{ ValidateOptions, ValidateMonitor, BandProblem, BandValidateResult, BlockMissingReason }; +pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; +pub use crate::validate::{ ValidateOptions, BandProblem, BandValidateResult, BlockMissingReason }; +pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor }; pub type Result = std::result::Result; diff --git a/src/monitor.rs b/src/monitor.rs new file mode 100644 index 00000000..7abf184e --- /dev/null +++ b/src/monitor.rs @@ -0,0 +1,48 @@ +use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result}; + +/// Monitor the backup progress. +pub trait BackupMonitor { + /// Will be called before the entry will be backupped + fn copy(&mut self, _entry: &LiveEntry) {} + fn copy_error(&mut self, _entry: &LiveEntry, _error: &Error) {} + fn copy_result(&mut self, _entry: &LiveEntry, _result: &Option) {} + + fn finished(&mut self, _stats: &BackupStats) {} +} + +/// Monitor the validation progress. +pub trait ValidateMonitor { + fn count_bands(&mut self) {} + fn count_bands_result(&mut self, _bands: &[BandId]) {} + + fn validate_bands(&mut self) {} + fn validate_bands_finished(&mut self) {} + + fn validate_band(&mut self, _band_id: &BandId) {} + fn validate_band_problem(&mut self, _band: &Band, _problem: &BandProblem) {} + fn validate_band_result(&mut self, _band_id: &BandId, _result: &BandValidateResult) {} + + fn validate_block_missing(&mut self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} + fn validate_blocks(&mut self) {} + fn validate_blocks_finished(&mut self) {} + + fn list_block_names(&mut self, _current_count: usize) {} + fn list_block_names_finished(&mut self) {} + + fn read_blocks(&mut self, _count: usize) {} + fn read_block_result(&mut self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} + fn read_blocks_finished(&mut self) {} +} + +/// Monitor for iterating trees. +pub trait TreeSizeMonitor { + fn entry_discovered(&mut self, _entry: &T::Entry, _size: &Option) {} +} + +/// Default monitor which does nothing. +/// Will be used when no monitor has been specified by the caller. +pub(crate) struct DefaultMonitor {} + +impl BackupMonitor for DefaultMonitor {} +impl ValidateMonitor for DefaultMonitor {} +impl TreeSizeMonitor for DefaultMonitor { } \ No newline at end of file diff --git a/src/tree.rs b/src/tree.rs index 4f0a511a..58fb3482 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -15,16 +15,10 @@ use std::ops::Range; +use crate::monitor::DefaultMonitor; use crate::stats::Sizes; use crate::*; -pub trait TreeSizeMonitor { - fn entry_discovered(&mut self, _entry: &T::Entry, _size: &Option) {} -} - -struct DefaultTreeSizeMonitor {} -impl TreeSizeMonitor for DefaultTreeSizeMonitor {} - /// Abstract Tree that may be either on the real filesystem or stored in an archive. pub trait ReadTree { type Entry: Entry + 'static; @@ -50,7 +44,7 @@ pub trait ReadTree { /// /// This typically requires walking all entries, which may take a while. fn size(&self, exclude: Exclude, monitor: Option<&mut dyn TreeSizeMonitor>) -> Result where Self: Sized { - let mut default_monitor = DefaultTreeSizeMonitor{}; + let mut default_monitor = DefaultMonitor{}; let monitor = monitor.unwrap_or(&mut default_monitor as &mut dyn TreeSizeMonitor); let mut tot = 0u64; diff --git a/src/validate.rs b/src/validate.rs index f0f5c177..81111ad5 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -15,7 +15,6 @@ use std::collections::HashMap; use crate::blockdir::Address; use crate::*; -use crate::stats::Sizes; pub struct BlockLengths(pub HashMap); @@ -25,29 +24,6 @@ pub struct ValidateOptions { pub skip_block_hashes: bool, } -pub trait ValidateMonitor { - fn count_bands(&mut self) {} - fn count_bands_result(&mut self, _bands: &[BandId]) {} - - fn validate_bands(&mut self) {} - fn validate_bands_finished(&mut self) {} - - fn validate_band(&mut self, _band_id: &BandId) {} - fn validate_band_problem(&mut self, _band: &Band, _problem: &BandProblem) {} - fn validate_band_result(&mut self, _band_id: &BandId, _result: &BandValidateResult) {} - - fn validate_block_missing(&mut self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} - fn validate_blocks(&mut self) {} - fn validate_blocks_finished(&mut self) {} - - fn list_block_names(&mut self, _current_count: usize) {} - fn list_block_names_finished(&mut self) {} - - fn read_blocks(&mut self, _count: usize) {} - fn read_block_result(&mut self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} - fn read_blocks_finished(&mut self) {} -} - /// Band validation result. pub enum BandValidateResult { MetadataError(Error), From 7c5f2fcfb82d68e15f6de4923271f11061e54e86 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 12:59:42 +0200 Subject: [PATCH 15/39] Using a monitor for validating the top level archive as well --- src/archive.rs | 46 ++++++++++++++++++++++------------------ src/band.rs | 2 +- src/bin/conserve/main.rs | 2 +- src/bin/conserve/show.rs | 37 +++++++++++++++++++++++++++++--- src/monitor.rs | 6 +++++- 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index e14af289..112929e9 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -34,7 +34,7 @@ use crate::monitor::DefaultMonitor; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; -use crate::validate::{ValidateMonitor, BlockMissingReason}; +use crate::validate::{BlockMissingReason}; use crate::*; const HEADER_FILENAME: &str = "CONSERVE"; @@ -60,6 +60,15 @@ pub struct DeleteOptions { pub break_lock: bool, } +#[derive(Debug)] +pub enum ValidateArchiveProblem { + UnexpectedFiles{ path: String, files: Vec }, + UnexpectedFileType{ name: String, kind: Kind }, + UnexpectedDirectory{ path: String, directory: String }, + DuplicateBand{ path: String, directory: String }, + DirectoryListError { error: std::io::Error }, +} + impl Archive { /// Make a new archive in a local directory. pub fn create_path(path: &Path) -> Result { @@ -305,7 +314,9 @@ impl Archive { let monitor = monitor.unwrap_or(&mut default_monitor); let start = Instant::now(); - let mut stats = self.validate_archive_dir()?; + monitor.validate_archive(); + let mut stats = self.validate_archive_dir(monitor)?; + monitor.validate_archive_finished(); monitor.count_bands(); let band_ids = self.list_band_ids()?; @@ -357,10 +368,9 @@ impl Archive { Ok(stats) } - fn validate_archive_dir(&self) -> Result { + fn validate_archive_dir(&self, monitor: &mut dyn ValidateMonitor) -> Result { // TODO: Tests for the problems detected here. let mut stats = ValidateStats::default(); - ui::println("Check archive top-level directory..."); let mut files: Vec = Vec::new(); let mut dirs: Vec = Vec::new(); @@ -374,15 +384,12 @@ impl Archive { Kind::Dir => dirs.push(name), Kind::File => files.push(name), other_kind => { - ui::problem(&format!( - "Unexpected file kind in archive directory: {:?} of kind {:?}", - name, other_kind - )); + monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFileType { name: name, kind: other_kind }); stats.unexpected_files += 1; } }, Err(source) => { - ui::problem(&format!("Error listing archive directory: {:?}", source)); + monitor.validate_archive_problem(&ValidateArchiveProblem::DirectoryListError { error: source }); stats.io_errors += 1; } } @@ -391,10 +398,7 @@ impl Archive { if !files.is_empty() { // TODO: Ignore .DS_Store stats.unexpected_files += 1; - ui::problem(&format!( - "Unexpected files in archive directory {:?}: {:?}", - self.transport, files - )); + monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFiles { path: format!("{:?}", self.transport), files }); } remove_item(&mut dirs, &BLOCK_DIR); dirs.sort(); @@ -403,19 +407,19 @@ impl Archive { if let Ok(b) = d.parse() { if bs.contains(&b) { stats.structure_problems += 1; - ui::problem(&format!( - "Duplicated band directory in {:?}: {:?}", - self.transport, d - )); + monitor.validate_archive_problem(&ValidateArchiveProblem::DuplicateBand { + path: format!("{:?}", self.transport), + directory: d.clone() + }); } else { bs.insert(b); } } else { stats.structure_problems += 1; - ui::problem(&format!( - "Unexpected directory in {:?}: {:?}", - self.transport, d - )); + monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedDirectory { + path: format!("{:?}", self.transport), + directory: d.clone() + }); } } Ok(stats) diff --git a/src/band.rs b/src/band.rs index 9efcd27f..3287b77b 100644 --- a/src/band.rs +++ b/src/band.rs @@ -28,7 +28,7 @@ use crate::jsonio::{read_json, write_json}; use crate::misc::remove_item; use crate::transport::{ListDirNames, Transport}; use crate::*; -use crate::validate::{ValidateMonitor, BandProblem}; +use crate::validate::{BandProblem}; static INDEX_DIR: &str = "i"; diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index fe143a54..4ecb3703 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -24,7 +24,7 @@ use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn, Level }; -use conserve::backup::{BackupOptions, BackupMonitor}; +use conserve::backup::{BackupOptions}; use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 163c0561..136c58ce 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -20,16 +20,17 @@ use std::borrow::Cow; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; -use conserve::backup::BackupMonitor; +use conserve::archive::ValidateArchiveProblem; use conserve::stats::Sizes; use conserve::ui::duration_to_hms; use conserve::{ bytes_to_human_mb, Archive, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, - DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, ValidateMonitor, TreeSizeMonitor, + DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, TreeSizeMonitor, + ValidateMonitor, BackupMonitor }; use nutmeg::View; use thousands::Separable; -use tracing::{info, warn}; +use tracing::{info, warn, error}; use crate::log::{self, ViewLogGuard}; @@ -324,6 +325,36 @@ impl nutmeg::Model for ValidateProgressModel { } impl ValidateMonitor for NutmegMonitor { + fn validate_archive(&mut self) { + info!("Check archive top-level directory..."); + } + + fn validate_archive_problem(&mut self, problem: &ValidateArchiveProblem) { + match problem { + ValidateArchiveProblem::UnexpectedFileType { name, kind } => { + error!( + "Unexpected file kind in archive directory: {:?} of kind {:?}", + name, kind + ); + }, + ValidateArchiveProblem::DirectoryListError { error } => { + error!("Error listing archive directory: {:?}", error); + } + ValidateArchiveProblem::UnexpectedFiles { path, files } => { + error!( + "Unexpected files in archive directory {:?}: {:?}", + path, files + ); + } + ValidateArchiveProblem::DuplicateBand { path, directory } => { + error!("Duplicated band directory in {:?}: {:?}", path, directory); + } + ValidateArchiveProblem::UnexpectedDirectory { path, directory } => { + error!("Unexpected directory in {:?}: {:?}", path, directory); + } + } + } + fn count_bands(&mut self) { info!("Count bands..."); } diff --git a/src/monitor.rs b/src/monitor.rs index 7abf184e..20ff78b3 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -1,4 +1,4 @@ -use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result}; +use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result, archive::ValidateArchiveProblem}; /// Monitor the backup progress. pub trait BackupMonitor { @@ -15,6 +15,10 @@ pub trait ValidateMonitor { fn count_bands(&mut self) {} fn count_bands_result(&mut self, _bands: &[BandId]) {} + fn validate_archive(&mut self) {} + fn validate_archive_problem(&mut self, _problem: &ValidateArchiveProblem) {} + fn validate_archive_finished(&mut self) {} + fn validate_bands(&mut self) {} fn validate_bands_finished(&mut self) {} From 7b3091c5d177b70b35ee7f9f7555aa463a607c6f Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 13:39:44 +0200 Subject: [PATCH 16/39] Using a monitor for deletions --- src/archive.rs | 152 ++++++++++++++++++++++++--------------- src/bin/conserve/main.rs | 16 ++++- src/bin/conserve/show.rs | 96 ++++++++++++++++++++++++- src/lib.rs | 2 +- src/monitor.rs | 29 +++++++- 5 files changed, 230 insertions(+), 65 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 112929e9..2b9a0f62 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -21,8 +21,6 @@ use std::sync::Arc; use std::time::Instant; use itertools::Itertools; -use nutmeg::models::{LinearModel, UnboundedModel}; -use rayon::prelude::*; use serde::{Deserialize, Serialize}; use crate::blockhash::BlockHash; @@ -30,7 +28,7 @@ use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; use crate::misc::remove_item; -use crate::monitor::DefaultMonitor; +use crate::monitor::{DefaultMonitor, DeleteMonitor, ReferencedBlocksMonitor}; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; @@ -205,25 +203,33 @@ impl Archive { /// Returns all blocks referenced by all bands. /// /// Shows a progress bar as they're collected. - pub fn referenced_blocks(&self, band_ids: &[BandId]) -> Result> { + pub fn referenced_blocks(&self, band_ids: &[BandId], monitor: Option<&mut dyn ReferencedBlocksMonitor>) -> Result> { + let mut default_monitor = DefaultMonitor{}; + let monitor = monitor.unwrap_or(&mut default_monitor); + let archive = self.clone(); - let progress = nutmeg::View::new( - LinearModel::new("Find referenced blocks in band", band_ids.len()), - ui::nutmeg_options(), - ); - Ok(band_ids - .par_iter() - .inspect(move |_| progress.update(|model| model.increment(1))) + let mut current_count = 0; + + let result = band_ids + // .par_iter() + .iter() + .inspect(|_| { + current_count += 1; + monitor.list_referenced_blocks(current_count); + }) .map(move |band_id| Band::open(&archive, band_id).expect("Failed to open band")) - .flat_map_iter(|band| band.index().iter_entries()) - .flat_map_iter(|entry| entry.addrs) + .flat_map(|band| band.index().iter_entries()) + .flat_map(|entry| entry.addrs) .map(|addr| addr.hash) - .collect()) + .collect(); + + monitor.list_referenced_blocks_finished(); + Ok(result) } /// Returns an iterator of blocks that are present and referenced by no index. - pub fn unreferenced_blocks(&self) -> Result> { - let referenced = self.referenced_blocks(&self.list_band_ids()?)?; + pub fn unreferenced_blocks(&self, monitor: Option<&mut dyn ReferencedBlocksMonitor>) -> Result> { + let referenced = self.referenced_blocks(&self.list_band_ids()?, monitor)?; Ok(self .block_dir() .block_names()? @@ -238,7 +244,11 @@ impl Archive { &self, delete_band_ids: &[BandId], options: &DeleteOptions, + monitor: Option<&mut dyn DeleteMonitor>, ) -> Result { + let mut default_monitor = DefaultMonitor{}; + let monitor_ = monitor.unwrap_or(&mut default_monitor); + let mut stats = DeleteStats::default(); let start = Instant::now(); @@ -253,56 +263,82 @@ impl Archive { let mut keep_band_ids = self.list_band_ids()?; keep_band_ids.retain(|b| !delete_band_ids.contains(b)); - let referenced = self.referenced_blocks(&keep_band_ids)?; - let progress = nutmeg::View::new( - UnboundedModel::new("Find present blocks"), - ui::nutmeg_options(), - ); - let unref = self - .block_dir() - .block_names()? - .inspect(|_| progress.update(|model| model.increment(1))) - .filter(|bh| !referenced.contains(bh)) - .collect_vec(); - drop(progress); + let referenced = self.referenced_blocks(&keep_band_ids, Some(monitor_.referenced_blocks_monitor()))?; + let unref = { + monitor_.find_present_blocks(0); + + let mut present_block_index = 0; + let unref = self + .block_dir() + .block_names()? + .inspect(|_| { + present_block_index += 1; + monitor_.find_present_blocks(present_block_index); + }) + .filter(|bh| !referenced.contains(bh)) + .collect_vec(); + + monitor_.find_present_blocks_finished(); + unref + }; + let unref_count = unref.len(); stats.unreferenced_block_count = unref_count; - let progress = nutmeg::View::new( - LinearModel::new("Measure unreferenced blocks", unref.len()), - ui::nutmeg_options(), - ); - let total_bytes = unref - .par_iter() - .inspect(|_| progress.update(|model| model.increment(1))) - .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) - .sum(); + let total_bytes = { + monitor_.measure_unreferenced_blocks(0, unref_count); + + let mut block_index = 0; + let total_bytes = unref + //.par_iter() + .iter() + .inspect(|_| { + block_index += 1; + monitor_.measure_unreferenced_blocks(block_index, unref_count); + }) + .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) + .sum(); + + monitor_.measure_unreferenced_blocks_finished(); + total_bytes + }; + stats.unreferenced_block_bytes = total_bytes; if !options.dry_run { delete_guard.check()?; - let progress = nutmeg::View::new( - LinearModel::new("Delete bands", delete_band_ids.len()), - ui::nutmeg_options(), - ); - for band_id in delete_band_ids { - Band::delete(self, band_id)?; - stats.deleted_band_count += 1; - progress.update(|model| model.increment(1)); + { + let mut delete_band_count = 0; + monitor_.delete_bands(0, delete_band_ids.len()); + for band_id in delete_band_ids { + delete_band_count += 1; + monitor_.delete_bands(delete_band_count, delete_band_ids.len()); + + Band::delete(self, band_id)?; + stats.deleted_band_count += 1; + } + monitor_.delete_bands_finished(); } - let progress = nutmeg::View::new( - LinearModel::new("Delete blocks", unref_count), - ui::nutmeg_options(), - ); - let error_count = unref - .par_iter() - .inspect(|_| progress.update(|model| model.increment(1))) - .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) - .count(); - stats.deletion_errors += error_count; - stats.deleted_block_count += unref_count - error_count; + { + monitor_.delete_blocks(0, unref.len()); + + let mut delete_block_count = 0; + let error_count = unref + //.par_iter() + .iter() + .inspect(|_| { + delete_block_count += 1; + monitor_.delete_blocks(delete_block_count, unref.len()); + }) + .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) + .count(); + + stats.deletion_errors += error_count; + stats.deleted_block_count += unref_count - error_count; + monitor_.delete_blocks_finished(); + } } stats.elapsed = start.elapsed(); @@ -493,7 +529,7 @@ mod tests { "Archive should have no bands yet" ); assert_eq!( - af.referenced_blocks(&af.list_band_ids().unwrap()) + af.referenced_blocks(&af.list_band_ids().unwrap(), None) .unwrap() .len(), 0 @@ -525,7 +561,7 @@ mod tests { assert_eq!(af.last_band_id().unwrap(), Some(BandId::new(&[1]))); assert_eq!( - af.referenced_blocks(&af.list_band_ids().unwrap()) + af.referenced_blocks(&af.list_band_ids().unwrap(), None) .unwrap() .len(), 0 diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 4ecb3703..32eac4fb 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,7 +20,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; -use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel}; +use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn, Level }; @@ -327,12 +327,14 @@ impl Command { } Command::Debug(Debug::Referenced { archive }) => { let archive = Archive::open(open_transport(archive)?)?; - for hash in archive.referenced_blocks(&archive.list_band_ids()?)? { + // FIXME: Monitor? + for hash in archive.referenced_blocks(&archive.list_band_ids()?, None)? { info!("{}", hash); } } Command::Debug(Debug::Unreferenced { archive }) => { - for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks()? { + // FIXME: Monitor? + for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks(None)? { info!("{}", hash); } } @@ -343,12 +345,16 @@ impl Command { break_lock, no_stats, } => { + // FIXME: Respect the "no progress" option and only log messages. + let mut monitor = NutmegMonitor::::new(); + let stats = Archive::open(open_transport(archive)?)?.delete_bands( backup, &DeleteOptions { dry_run: *dry_run, break_lock: *break_lock, }, + Some(&mut monitor), )?; if !no_stats { info!("{}", stats); @@ -378,6 +384,9 @@ impl Command { break_lock, no_stats, } => { + // FIXME: Respect the "no progress" option and only log messages. + let mut monitor = NutmegMonitor::::new(); + let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( &[], @@ -385,6 +394,7 @@ impl Command { dry_run: *dry_run, break_lock: *break_lock, }, + Some(&mut monitor), )?; if !no_stats { info!("{}", stats); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 136c58ce..4f294387 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -26,9 +26,9 @@ use conserve::ui::duration_to_hms; use conserve::{ bytes_to_human_mb, Archive, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, TreeSizeMonitor, - ValidateMonitor, BackupMonitor + ValidateMonitor, BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor }; -use nutmeg::View; +use nutmeg::{View, Model}; use thousands::Separable; use tracing::{info, warn, error}; @@ -522,3 +522,95 @@ impl TreeSizeMonitor for NutmegMonitor { }); } } + +pub enum DeleteProcessState { + ListReferencedBlocks { + count: usize, + }, + FindPresentBlocks { + count: usize, + }, + MeasureUnreferencedBlocks { + count: usize, + target: usize, + }, + DeleteBands { + count: usize, + target: usize, + }, + DeleteBlocks { + count: usize, + target: usize, + } +} + +impl Default for DeleteProcessState { + fn default() -> Self { + DeleteProcessState::ListReferencedBlocks { count: 0 } + } +} + +impl Model for DeleteProcessState { + fn render(&mut self, _width: usize) -> String { + match self { + DeleteProcessState::ListReferencedBlocks { count } => { + format!("Find referenced blocks in band ({} discovered)", count) + }, + DeleteProcessState::FindPresentBlocks { count } => { + format!("Find present blocks ({} discovered)", count) + }, + DeleteProcessState::MeasureUnreferencedBlocks { count, target } => { + format!("Measure unreferenced blocks ({}/{})", count, target) + }, + DeleteProcessState::DeleteBands { count, target } => { + format!("Delete bands ({}/{})", count, target) + }, + DeleteProcessState::DeleteBlocks { count, target } => { + format!("Delete blocks ({}/{})", count, target) + } + } + } +} + +impl DeleteMonitor for NutmegMonitor { + fn referenced_blocks_monitor(&mut self) -> &mut dyn conserve::ReferencedBlocksMonitor { + self + } + + fn find_present_blocks(&mut self, current_count: usize) { + let view = self.locked_view(); + view.update(|view| { + *view = DeleteProcessState::FindPresentBlocks { count: current_count }; + }); + } + + fn measure_unreferenced_blocks(&mut self, current_count: usize, target_count: usize) { + let view = self.locked_view(); + view.update(|view| { + *view = DeleteProcessState::MeasureUnreferencedBlocks { count: current_count, target: target_count }; + }); + } + + fn delete_bands(&mut self, current_count: usize, target_count: usize) { + let view = self.locked_view(); + view.update(|view| { + *view = DeleteProcessState::DeleteBands { count: current_count, target: target_count }; + }); + } + + fn delete_blocks(&mut self, current_count: usize, target_count: usize) { + let view = self.locked_view(); + view.update(|view| { + *view = DeleteProcessState::DeleteBlocks { count: current_count, target: target_count }; + }); + } +} + +impl ReferencedBlocksMonitor for NutmegMonitor { + fn list_referenced_blocks(&mut self, current_count: usize) { + let view = self.locked_view(); + view.update(|view| { + *view = DeleteProcessState::ListReferencedBlocks { count: current_count }; + }); + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index d7af3706..38c98624 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,7 @@ pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; pub use crate::validate::{ ValidateOptions, BandProblem, BandValidateResult, BlockMissingReason }; -pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor }; +pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor, ReferencedBlocksMonitor, DeleteMonitor }; pub type Result = std::result::Result; diff --git a/src/monitor.rs b/src/monitor.rs index 20ff78b3..06d3057f 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -43,10 +43,37 @@ pub trait TreeSizeMonitor { fn entry_discovered(&mut self, _entry: &T::Entry, _size: &Option) {} } +pub trait ReferencedBlocksMonitor { + fn list_referenced_blocks(&mut self, _current_count: usize) {} + fn list_referenced_blocks_finished(&mut self) {} +} + +pub trait DeleteMonitor { + fn referenced_blocks_monitor(&mut self) -> &mut dyn ReferencedBlocksMonitor; + + fn find_present_blocks(&mut self, _current_count: usize) {} + fn find_present_blocks_finished(&mut self) {} + + fn measure_unreferenced_blocks(&mut self, _current_count: usize, _target_count: usize) {} + fn measure_unreferenced_blocks_finished(&mut self) {} + + fn delete_bands(&mut self, _current_count: usize, _target_count: usize) {} + fn delete_bands_finished(&mut self) {} + + fn delete_blocks(&mut self, _current_count: usize, _target_count: usize) {} + fn delete_blocks_finished(&mut self) {} +} + /// Default monitor which does nothing. /// Will be used when no monitor has been specified by the caller. pub(crate) struct DefaultMonitor {} impl BackupMonitor for DefaultMonitor {} impl ValidateMonitor for DefaultMonitor {} -impl TreeSizeMonitor for DefaultMonitor { } \ No newline at end of file +impl TreeSizeMonitor for DefaultMonitor {} +impl ReferencedBlocksMonitor for DefaultMonitor {} +impl DeleteMonitor for DefaultMonitor { + fn referenced_blocks_monitor(&mut self) -> &mut dyn ReferencedBlocksMonitor { + self + } +} \ No newline at end of file From 1e25ddc1658625a901c9b34765d1c90ade04b0d3 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:04:13 +0200 Subject: [PATCH 17/39] Using a monitor for restore. This removes all nutmeg modals within the library. --- src/bin/conserve/main.rs | 17 ++++++------ src/bin/conserve/show.rs | 58 +++++++++++++++++++++++++++++++++++++--- src/blockdir.rs | 3 +-- src/lib.rs | 2 +- src/monitor.rs | 10 +++++-- src/restore.rs | 51 ++++++++++------------------------- src/ui.rs | 4 --- 7 files changed, 87 insertions(+), 58 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 32eac4fb..9f585957 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,7 +20,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; -use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState}; +use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState, RestoreProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn, Level }; @@ -295,7 +295,7 @@ impl Command { let mut monitor = if args.no_progress { None } else { - Some(NutmegMonitor::::new()) + Some(NutmegMonitor::new(BackupProgressModel::default())) }; let stats = backup( @@ -346,7 +346,7 @@ impl Command { no_stats, } => { // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::::new(); + let mut monitor = NutmegMonitor::new(DeleteProcessState::default()); let stats = Archive::open(open_transport(archive)?)?.delete_bands( backup, @@ -385,7 +385,7 @@ impl Command { no_stats, } => { // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::::new(); + let mut monitor = NutmegMonitor::new(DeleteProcessState::default()); let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( @@ -438,14 +438,15 @@ impl Command { let archive = Archive::open(open_transport(archive)?)?; let exclude = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; let options = RestoreOptions { - print_filenames: *verbose, exclude, only_subtree: only_subtree.clone(), band_selection, overwrite: *force_overwrite, }; - let stats = restore(&archive, destination, &options)?; + // FIXME: Respect the "no progress" option and only log messages. + let mut monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose)); + let stats = restore(&archive, destination, &options, Some(&mut monitor))?; if !no_stats { info!("Restore complete."); info!("{}", stats); @@ -460,7 +461,7 @@ impl Command { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::::new(); + let mut monitor = NutmegMonitor::new(SizeProgressModel::default()); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? .size(excludes, Some(&mut monitor as &mut dyn TreeSizeMonitor<_>))? @@ -488,7 +489,7 @@ impl Command { }; // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::::new(); + let mut monitor = NutmegMonitor::new(ValidateProgressModel::default()); let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &mut dyn ValidateMonitor))?; drop(monitor); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 4f294387..67a93250 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -26,7 +26,7 @@ use conserve::ui::duration_to_hms; use conserve::{ bytes_to_human_mb, Archive, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, TreeSizeMonitor, - ValidateMonitor, BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor + ValidateMonitor, BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor, RestoreMonitor }; use nutmeg::{View, Model}; use thousands::Separable; @@ -195,10 +195,10 @@ pub struct NutmegMonitor { view: Arc>>, } -impl NutmegMonitor { - pub fn new() -> Self { +impl NutmegMonitor { + pub fn new(initial_state: T) -> Self { let view = Arc::new(Mutex::new(nutmeg::View::new( - T::default(), + initial_state, nutmeg::Options::default(), ))); @@ -613,4 +613,54 @@ impl ReferencedBlocksMonitor for NutmegMonitor { *view = DeleteProcessState::ListReferencedBlocks { count: current_count }; }); } +} + +pub struct RestoreProgressModel { + print_filenames: bool, + filename: String, + bytes_done: u64, +} + +impl RestoreProgressModel { + pub fn new(print_filenames: bool) -> Self { + Self { + print_filenames, + filename: "".to_string(), + bytes_done: 0 + } + } +} + +impl nutmeg::Model for RestoreProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Restoring: {} MB\n{}", + self.bytes_done / 1_000_000, + self.filename + ) + } +} + +impl RestoreMonitor for NutmegMonitor { + fn restore_entry(&mut self, entry: &IndexEntry) { + let mut print_filename = false; + { + let view = self.locked_view(); + view.update(|view| { + print_filename = view.print_filenames; + view.filename = entry.apath().to_string(); + }); + } + + if print_filename { + info!("{}", entry.apath()); + } + } + + fn restore_entry_result(&mut self, entry: &IndexEntry, _result: &Result<()>) { + if let Some(bytes) = entry.size() { + let view = self.locked_view(); + view.update(|view| view.bytes_done += bytes); + } + } } \ No newline at end of file diff --git a/src/blockdir.rs b/src/blockdir.rs index 327ebaef..339baffe 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -239,9 +239,8 @@ impl BlockDir { } /// Return all the blocknames in the blockdir, in arbitrary order. + // TODO: Add a monitor since this operation might takes longer. pub fn block_names(&self) -> Result> { - let progress = nutmeg::View::new("List blocks", ui::nutmeg_options()); - progress.update(|_| ()); Ok(self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok())) diff --git a/src/lib.rs b/src/lib.rs index 38c98624..92375e1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,7 @@ pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; pub use crate::validate::{ ValidateOptions, BandProblem, BandValidateResult, BlockMissingReason }; -pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor, ReferencedBlocksMonitor, DeleteMonitor }; +pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor, ReferencedBlocksMonitor, DeleteMonitor, RestoreMonitor }; pub type Result = std::result::Result; diff --git a/src/monitor.rs b/src/monitor.rs index 06d3057f..9077849c 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -1,4 +1,4 @@ -use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result, archive::ValidateArchiveProblem}; +use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result, archive::ValidateArchiveProblem, IndexEntry}; /// Monitor the backup progress. pub trait BackupMonitor { @@ -64,6 +64,11 @@ pub trait DeleteMonitor { fn delete_blocks_finished(&mut self) {} } +pub trait RestoreMonitor { + fn restore_entry(&mut self, _entry: &IndexEntry) {} + fn restore_entry_result(&mut self, _entry: &IndexEntry, _result: &Result<()>) {} +} + /// Default monitor which does nothing. /// Will be used when no monitor has been specified by the caller. pub(crate) struct DefaultMonitor {} @@ -76,4 +81,5 @@ impl DeleteMonitor for DefaultMonitor { fn referenced_blocks_monitor(&mut self) -> &mut dyn ReferencedBlocksMonitor { self } -} \ No newline at end of file +} +impl RestoreMonitor for DefaultMonitor {} \ No newline at end of file diff --git a/src/restore.rs b/src/restore.rs index 4237b3cb..0426c113 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -22,10 +22,12 @@ use filetime::{set_file_handle_times}; #[cfg(unix)] use filetime::{set_symlink_file_times}; +use tracing::debug; use crate::band::BandSelectionPolicy; use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; +use crate::monitor::{RestoreMonitor, DefaultMonitor}; use crate::stats::RestoreStats; use crate::unix_time::UnixTime; use crate::*; @@ -33,7 +35,6 @@ use crate::*; /// Description of how to restore a tree. #[derive(Debug)] pub struct RestoreOptions { - pub print_filenames: bool, pub exclude: Exclude, /// Restore only this subdirectory. pub only_subtree: Option, @@ -45,7 +46,6 @@ pub struct RestoreOptions { impl Default for RestoreOptions { fn default() -> Self { RestoreOptions { - print_filenames: false, overwrite: false, band_selection: BandSelectionPolicy::LatestClosed, exclude: Exclude::nothing(), @@ -54,27 +54,16 @@ impl Default for RestoreOptions { } } -struct ProgressModel { - filename: String, - bytes_done: u64, -} - -impl nutmeg::Model for ProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Restoring: {} MB\n{}", - self.bytes_done / 1_000_000, - self.filename - ) - } -} - /// Restore a selected version, or by default the latest, to a destination directory. pub fn restore( archive: &Archive, destination_path: &Path, options: &RestoreOptions, + monitor: Option<&mut dyn RestoreMonitor>, ) -> Result { + let mut default_monitor = DefaultMonitor{}; + let monitor = monitor.unwrap_or(&mut default_monitor); + let st = archive.open_stored_tree(options.band_selection.clone())?; let mut rt = if options.overwrite { RestoreTree::create_overwrite(destination_path) @@ -82,13 +71,7 @@ pub fn restore( RestoreTree::create(destination_path) }?; let mut stats = RestoreStats::default(); - let progress_bar = nutmeg::View::new( - ProgressModel { - filename: String::new(), - bytes_done: 0, - }, - ui::nutmeg_options(), - ); + let start = Instant::now(); // // This causes us to walk the source tree twice, which is probably an acceptable option // // since it's nice to see realistic overall progress. We could keep all the entries @@ -106,22 +89,15 @@ pub fn restore( options.exclude.clone(), )?; for entry in entry_iter { - if options.print_filenames { - progress_bar.message(&format!("{}\n", entry.apath())); - } - progress_bar.update(|model| model.filename = entry.apath().to_string()); - if let Err(e) = match entry.kind() { + monitor.restore_entry(&entry); + let result = match entry.kind() { Kind::Dir => { stats.directories += 1; rt.copy_dir(&entry) } Kind::File => { stats.files += 1; - let result = rt.copy_file(&entry, &st).map(|s| stats += s); - if let Some(bytes) = entry.size() { - progress_bar.update(|model| model.bytes_done += bytes); - } - result + rt.copy_file(&entry, &st).map(|s| stats += s) } Kind::Symlink => { stats.symlinks += 1; @@ -134,10 +110,11 @@ pub fn restore( // https://github.com/sourcefrog/conserve/issues/82 continue; } - } { - ui::show_error(&e); + }; + monitor.restore_entry_result(&entry, &result); + if let Err(error) = result { + debug!("{}", ui::format_error_causes(&error)); stats.errors += 1; - continue; } } stats += rt.finish()?; diff --git a/src/ui.rs b/src/ui.rs index 7f15b05a..c333781b 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -158,10 +158,6 @@ impl UIState { } } -pub(crate) fn nutmeg_options() -> nutmeg::Options { - nutmeg::Options::default().progress_enabled(UI_STATE.lock().unwrap().progress_enabled) -} - #[cfg(test)] mod tests { use super::*; From 66fcf188eeeeccc72418de4f8752a19639a4aab3 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:22:32 +0200 Subject: [PATCH 18/39] Removing mutability state and adding Sync constraint to monitors to allow parallel processing. Therefore each monitor needs to sync state on its own. --- src/archive.rs | 46 ++++++++++++----------- src/backup.rs | 2 +- src/band.rs | 2 +- src/bin/conserve/main.rs | 8 ++-- src/bin/conserve/show.rs | 48 ++++++++++++------------ src/blockdir.rs | 14 ++++--- src/monitor.rs | 80 ++++++++++++++++++++-------------------- src/restore.rs | 2 +- src/tree.rs | 4 +- src/validate.rs | 4 +- 10 files changed, 107 insertions(+), 103 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 2b9a0f62..bfbd8ecd 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -18,10 +18,12 @@ use std::io::ErrorKind; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; use itertools::Itertools; use serde::{Deserialize, Serialize}; +use rayon::prelude::*; use crate::blockhash::BlockHash; use crate::errors::Error; @@ -203,23 +205,21 @@ impl Archive { /// Returns all blocks referenced by all bands. /// /// Shows a progress bar as they're collected. - pub fn referenced_blocks(&self, band_ids: &[BandId], monitor: Option<&mut dyn ReferencedBlocksMonitor>) -> Result> { + pub fn referenced_blocks(&self, band_ids: &[BandId], monitor: Option<&dyn ReferencedBlocksMonitor>) -> Result> { let mut default_monitor = DefaultMonitor{}; let monitor = monitor.unwrap_or(&mut default_monitor); let archive = self.clone(); - let mut current_count = 0; + let current_count = AtomicUsize::new(0); let result = band_ids - // .par_iter() - .iter() + .par_iter() .inspect(|_| { - current_count += 1; - monitor.list_referenced_blocks(current_count); + monitor.list_referenced_blocks(current_count.fetch_add(1, Ordering::Relaxed) + 1); }) .map(move |band_id| Band::open(&archive, band_id).expect("Failed to open band")) - .flat_map(|band| band.index().iter_entries()) - .flat_map(|entry| entry.addrs) + .flat_map_iter(|band| band.index().iter_entries()) + .flat_map_iter(|entry| entry.addrs) .map(|addr| addr.hash) .collect(); @@ -228,7 +228,7 @@ impl Archive { } /// Returns an iterator of blocks that are present and referenced by no index. - pub fn unreferenced_blocks(&self, monitor: Option<&mut dyn ReferencedBlocksMonitor>) -> Result> { + pub fn unreferenced_blocks(&self, monitor: Option<&dyn ReferencedBlocksMonitor>) -> Result> { let referenced = self.referenced_blocks(&self.list_band_ids()?, monitor)?; Ok(self .block_dir() @@ -244,7 +244,7 @@ impl Archive { &self, delete_band_ids: &[BandId], options: &DeleteOptions, - monitor: Option<&mut dyn DeleteMonitor>, + monitor: Option<&dyn DeleteMonitor>, ) -> Result { let mut default_monitor = DefaultMonitor{}; let monitor_ = monitor.unwrap_or(&mut default_monitor); @@ -288,13 +288,14 @@ impl Archive { let total_bytes = { monitor_.measure_unreferenced_blocks(0, unref_count); - let mut block_index = 0; + let block_index = AtomicUsize::new(0); let total_bytes = unref - //.par_iter() - .iter() + .par_iter() .inspect(|_| { - block_index += 1; - monitor_.measure_unreferenced_blocks(block_index, unref_count); + monitor_.measure_unreferenced_blocks( + block_index.fetch_add(1, Ordering::Relaxed) + 1, + unref_count + ); }) .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) .sum(); @@ -324,13 +325,14 @@ impl Archive { { monitor_.delete_blocks(0, unref.len()); - let mut delete_block_count = 0; + let delete_block_count = AtomicUsize::new(0); let error_count = unref - //.par_iter() - .iter() + .par_iter() .inspect(|_| { - delete_block_count += 1; - monitor_.delete_blocks(delete_block_count, unref.len()); + monitor_.delete_blocks( + delete_block_count.fetch_add(1, Ordering::Relaxed) + 1, + unref.len() + ); }) .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) .count(); @@ -345,7 +347,7 @@ impl Archive { Ok(stats) } - pub fn validate(&self, options: &ValidateOptions, monitor: Option<&mut dyn ValidateMonitor>) -> Result { + pub fn validate(&self, options: &ValidateOptions, monitor: Option<&dyn ValidateMonitor>) -> Result { let mut default_monitor = DefaultMonitor{}; let monitor = monitor.unwrap_or(&mut default_monitor); @@ -404,7 +406,7 @@ impl Archive { Ok(stats) } - fn validate_archive_dir(&self, monitor: &mut dyn ValidateMonitor) -> Result { + fn validate_archive_dir(&self, monitor: &dyn ValidateMonitor) -> Result { // TODO: Tests for the problems detected here. let mut stats = ValidateStats::default(); diff --git a/src/backup.rs b/src/backup.rs index a64b4dc8..18cfb2f9 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -57,7 +57,7 @@ pub fn backup( archive: &Archive, source: &LiveTree, options: &BackupOptions, - monitor: Option<&mut dyn BackupMonitor>, + monitor: Option<&dyn BackupMonitor>, ) -> Result { let _span = tracing::span!(Level::DEBUG, "backup"); diff --git a/src/band.rs b/src/band.rs index 3287b77b..60faa9c8 100644 --- a/src/band.rs +++ b/src/band.rs @@ -216,7 +216,7 @@ impl Band { }) } - pub fn validate(&self, stats: &mut ValidateStats, monitor: &mut dyn ValidateMonitor) -> Result<()> { + pub fn validate(&self, stats: &mut ValidateStats, monitor: &dyn ValidateMonitor) -> Result<()> { let ListDirNames { mut files, mut dirs } = self.transport.list_dir_names("").map_err(Error::from)?; diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 9f585957..b8361806 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -302,7 +302,7 @@ impl Command { &Archive::open(open_transport(archive)?)?, source, &options, - monitor.as_mut().map(|v| v as &mut dyn BackupMonitor) + monitor.as_mut().map(|v| v as &dyn BackupMonitor) )?; drop(monitor); @@ -464,11 +464,11 @@ impl Command { let mut monitor = NutmegMonitor::new(SizeProgressModel::default()); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? - .size(excludes, Some(&mut monitor as &mut dyn TreeSizeMonitor<_>))? + .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? .file_bytes } else { LiveTree::open(stos.source.as_ref().unwrap())? - .size(excludes, Some(&mut monitor as &mut dyn TreeSizeMonitor<_>))? + .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? .file_bytes }; drop(monitor); @@ -490,7 +490,7 @@ impl Command { // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::new(ValidateProgressModel::default()); - let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &mut dyn ValidateMonitor))?; + let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; drop(monitor); if !no_stats { diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 67a93250..3411264f 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -217,7 +217,7 @@ impl NutmegMonitor { } impl BackupMonitor for NutmegMonitor { - fn copy(&mut self, entry: &conserve::LiveEntry) { + fn copy(&self, entry: &conserve::LiveEntry) { let view = self.locked_view(); view.update(|model| { model.filename = entry.apath().to_string(); @@ -229,7 +229,7 @@ impl BackupMonitor for NutmegMonitor { }); } - fn copy_result(&mut self, entry: &conserve::LiveEntry, result: &Option) { + fn copy_result(&self, entry: &conserve::LiveEntry, result: &Option) { let view = self.locked_view(); if let Some(diff_kind) = result.as_ref() { view.update(|model| match diff_kind { @@ -245,7 +245,7 @@ impl BackupMonitor for NutmegMonitor { } } - fn copy_error(&mut self, entry: &conserve::LiveEntry, _error: &conserve::Error) { + fn copy_error(&self, entry: &conserve::LiveEntry, _error: &conserve::Error) { let view = self.locked_view(); if let Some(size) = entry.size() { view.update(|model| model.scanned_file_bytes += size); @@ -325,11 +325,11 @@ impl nutmeg::Model for ValidateProgressModel { } impl ValidateMonitor for NutmegMonitor { - fn validate_archive(&mut self) { + fn validate_archive(&self) { info!("Check archive top-level directory..."); } - fn validate_archive_problem(&mut self, problem: &ValidateArchiveProblem) { + fn validate_archive_problem(&self, problem: &ValidateArchiveProblem) { match problem { ValidateArchiveProblem::UnexpectedFileType { name, kind } => { error!( @@ -355,18 +355,18 @@ impl ValidateMonitor for NutmegMonitor { } } - fn count_bands(&mut self) { + fn count_bands(&self) { info!("Count bands..."); } - fn count_bands_result(&mut self, bands: &[conserve::BandId]) { + fn count_bands_result(&self, bands: &[conserve::BandId]) { info!("Checking {} bands...", bands.len()); let view = self.locked_view(); view.update(|model| model.bands_total = Some(bands.len())); } - fn validate_bands(&mut self) { + fn validate_bands(&self) { let view = self.locked_view(); view.update(|model| { let bands_total = model.bands_total.expect("bands have been counted"); @@ -378,7 +378,7 @@ impl ValidateMonitor for NutmegMonitor { }); } - fn validate_band_problem(&mut self, band: &Band, problem: &conserve::BandProblem) { + fn validate_band_problem(&self, band: &Band, problem: &conserve::BandProblem) { match problem { BandProblem::MissingHeadFile { .. } => { warn!("No band head file in {:?}", band.transport()) @@ -397,7 +397,7 @@ impl ValidateMonitor for NutmegMonitor { } fn validate_band_result( - &mut self, + &self, _band_id: &conserve::BandId, _result: &conserve::BandValidateResult, ) { @@ -411,7 +411,7 @@ impl ValidateMonitor for NutmegMonitor { }); } - fn validate_bands_finished(&mut self) { + fn validate_bands_finished(&self) { let mut elapsed: Option = None; // We can't use logging while locked_view is held since we would deadlock. @@ -432,7 +432,7 @@ impl ValidateMonitor for NutmegMonitor { ); } - fn list_block_names(&mut self, current_count: usize) { + fn list_block_names(&self, current_count: usize) { if current_count == 0 { info!("Count blocks..."); } @@ -445,7 +445,7 @@ impl ValidateMonitor for NutmegMonitor { }); } - fn read_blocks(&mut self, count: usize) { + fn read_blocks(&self, count: usize) { info!("Check {} blocks...", count.separate_with_commas()); let view = self.locked_view(); @@ -460,7 +460,7 @@ impl ValidateMonitor for NutmegMonitor { } fn read_block_result( - &mut self, + &self, _block_hash: &conserve::BlockHash, result: &Result<(Vec, Sizes)>, ) { @@ -487,7 +487,7 @@ impl ValidateMonitor for NutmegMonitor { } fn validate_block_missing( - &mut self, + &self, block_hash: &conserve::BlockHash, reason: &conserve::BlockMissingReason, ) { @@ -514,7 +514,7 @@ impl nutmeg::Model for SizeProgressModel { } impl TreeSizeMonitor for NutmegMonitor { - fn entry_discovered(&mut self, _entry: &::Entry, size: &Option) { + fn entry_discovered(&self, _entry: &::Entry, size: &Option) { let view = self.locked_view(); view.update(|model| { model.files += 1; @@ -573,32 +573,32 @@ impl Model for DeleteProcessState { } impl DeleteMonitor for NutmegMonitor { - fn referenced_blocks_monitor(&mut self) -> &mut dyn conserve::ReferencedBlocksMonitor { + fn referenced_blocks_monitor(&self) -> &dyn conserve::ReferencedBlocksMonitor { self } - fn find_present_blocks(&mut self, current_count: usize) { + fn find_present_blocks(&self, current_count: usize) { let view = self.locked_view(); view.update(|view| { *view = DeleteProcessState::FindPresentBlocks { count: current_count }; }); } - fn measure_unreferenced_blocks(&mut self, current_count: usize, target_count: usize) { + fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { let view = self.locked_view(); view.update(|view| { *view = DeleteProcessState::MeasureUnreferencedBlocks { count: current_count, target: target_count }; }); } - fn delete_bands(&mut self, current_count: usize, target_count: usize) { + fn delete_bands(&self, current_count: usize, target_count: usize) { let view = self.locked_view(); view.update(|view| { *view = DeleteProcessState::DeleteBands { count: current_count, target: target_count }; }); } - fn delete_blocks(&mut self, current_count: usize, target_count: usize) { + fn delete_blocks(&self, current_count: usize, target_count: usize) { let view = self.locked_view(); view.update(|view| { *view = DeleteProcessState::DeleteBlocks { count: current_count, target: target_count }; @@ -607,7 +607,7 @@ impl DeleteMonitor for NutmegMonitor { } impl ReferencedBlocksMonitor for NutmegMonitor { - fn list_referenced_blocks(&mut self, current_count: usize) { + fn list_referenced_blocks(&self, current_count: usize) { let view = self.locked_view(); view.update(|view| { *view = DeleteProcessState::ListReferencedBlocks { count: current_count }; @@ -642,7 +642,7 @@ impl nutmeg::Model for RestoreProgressModel { } impl RestoreMonitor for NutmegMonitor { - fn restore_entry(&mut self, entry: &IndexEntry) { + fn restore_entry(&self, entry: &IndexEntry) { let mut print_filename = false; { let view = self.locked_view(); @@ -657,7 +657,7 @@ impl RestoreMonitor for NutmegMonitor { } } - fn restore_entry_result(&mut self, entry: &IndexEntry, _result: &Result<()>) { + fn restore_entry_result(&self, entry: &IndexEntry, _result: &Result<()>) { if let Some(bytes) = entry.size() { let view = self.locked_view(); view.update(|view| view.bytes_done += bytes); diff --git a/src/blockdir.rs b/src/blockdir.rs index 339baffe..3e65d7fd 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -26,6 +26,7 @@ use std::convert::TryInto; use std::io; use std::path::Path; use std::sync::Arc; +use rayon::prelude::*; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; @@ -247,25 +248,27 @@ impl BlockDir { } /// Return all the blocknames in the blockdir. - pub fn block_names_set(&self, monitor: &mut dyn ValidateMonitor) -> Result> { + pub fn block_names_set(&self, monitor: &dyn ValidateMonitor) -> Result> { let mut block_count = 0usize; monitor.list_block_names(block_count); - Ok(self + let result = self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok()) .inspect(|_| { block_count += 1; monitor.list_block_names(block_count); }) - .collect()) + .collect(); + monitor.list_block_names_finished(); + Ok(result) } /// Check format invariants of the BlockDir. /// /// Return a dict describing which blocks are present, and the length of their uncompressed /// data. - pub fn validate(&self, stats: &mut ValidateStats, monitor: &mut dyn ValidateMonitor) -> Result> { + pub fn validate(&self, stats: &mut ValidateStats, monitor: &dyn ValidateMonitor) -> Result> { // TODO: In the top-level directory, no files or directories other than prefix // directories of the right length. // TODO: Test having a block with the right compression but the wrong contents. @@ -277,8 +280,7 @@ impl BlockDir { // Make a vec of Some(usize) if the block could be read, or None if it // failed, where the usize gives the uncompressed data size. let results: Vec> = blocks - //.into_par_iter() - .into_iter() + .into_par_iter() .map(|hash| { let result = self.get_block_content(&hash); // TODO(MH): Should we realy provide the block contents? May only return the size or the read error. diff --git a/src/monitor.rs b/src/monitor.rs index 9077849c..5070167a 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -3,70 +3,70 @@ use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProbl /// Monitor the backup progress. pub trait BackupMonitor { /// Will be called before the entry will be backupped - fn copy(&mut self, _entry: &LiveEntry) {} - fn copy_error(&mut self, _entry: &LiveEntry, _error: &Error) {} - fn copy_result(&mut self, _entry: &LiveEntry, _result: &Option) {} + fn copy(&self, _entry: &LiveEntry) {} + fn copy_error(&self, _entry: &LiveEntry, _error: &Error) {} + fn copy_result(&self, _entry: &LiveEntry, _result: &Option) {} - fn finished(&mut self, _stats: &BackupStats) {} + fn finished(&self, _stats: &BackupStats) {} } /// Monitor the validation progress. -pub trait ValidateMonitor { - fn count_bands(&mut self) {} - fn count_bands_result(&mut self, _bands: &[BandId]) {} +pub trait ValidateMonitor : Sync { + fn count_bands(&self) {} + fn count_bands_result(&self, _bands: &[BandId]) {} - fn validate_archive(&mut self) {} - fn validate_archive_problem(&mut self, _problem: &ValidateArchiveProblem) {} - fn validate_archive_finished(&mut self) {} + fn validate_archive(&self) {} + fn validate_archive_problem(&self, _problem: &ValidateArchiveProblem) {} + fn validate_archive_finished(&self) {} - fn validate_bands(&mut self) {} - fn validate_bands_finished(&mut self) {} + fn validate_bands(&self) {} + fn validate_bands_finished(&self) {} - fn validate_band(&mut self, _band_id: &BandId) {} - fn validate_band_problem(&mut self, _band: &Band, _problem: &BandProblem) {} - fn validate_band_result(&mut self, _band_id: &BandId, _result: &BandValidateResult) {} + fn validate_band(&self, _band_id: &BandId) {} + fn validate_band_problem(&self, _band: &Band, _problem: &BandProblem) {} + fn validate_band_result(&self, _band_id: &BandId, _result: &BandValidateResult) {} - fn validate_block_missing(&mut self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} - fn validate_blocks(&mut self) {} - fn validate_blocks_finished(&mut self) {} + fn validate_block_missing(&self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} + fn validate_blocks(&self) {} + fn validate_blocks_finished(&self) {} - fn list_block_names(&mut self, _current_count: usize) {} - fn list_block_names_finished(&mut self) {} + fn list_block_names(&self, _current_count: usize) {} + fn list_block_names_finished(&self) {} - fn read_blocks(&mut self, _count: usize) {} - fn read_block_result(&mut self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} - fn read_blocks_finished(&mut self) {} + fn read_blocks(&self, _count: usize) {} + fn read_block_result(&self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} + fn read_blocks_finished(&self) {} } /// Monitor for iterating trees. pub trait TreeSizeMonitor { - fn entry_discovered(&mut self, _entry: &T::Entry, _size: &Option) {} + fn entry_discovered(&self, _entry: &T::Entry, _size: &Option) {} } -pub trait ReferencedBlocksMonitor { - fn list_referenced_blocks(&mut self, _current_count: usize) {} - fn list_referenced_blocks_finished(&mut self) {} +pub trait ReferencedBlocksMonitor : Sync { + fn list_referenced_blocks(&self, _current_count: usize) {} + fn list_referenced_blocks_finished(&self) {} } -pub trait DeleteMonitor { - fn referenced_blocks_monitor(&mut self) -> &mut dyn ReferencedBlocksMonitor; +pub trait DeleteMonitor : Sync { + fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor; - fn find_present_blocks(&mut self, _current_count: usize) {} - fn find_present_blocks_finished(&mut self) {} + fn find_present_blocks(&self, _current_count: usize) {} + fn find_present_blocks_finished(&self) {} - fn measure_unreferenced_blocks(&mut self, _current_count: usize, _target_count: usize) {} - fn measure_unreferenced_blocks_finished(&mut self) {} + fn measure_unreferenced_blocks(&self, _current_count: usize, _target_count: usize) {} + fn measure_unreferenced_blocks_finished(&self) {} - fn delete_bands(&mut self, _current_count: usize, _target_count: usize) {} - fn delete_bands_finished(&mut self) {} + fn delete_bands(&self, _current_count: usize, _target_count: usize) {} + fn delete_bands_finished(&self) {} - fn delete_blocks(&mut self, _current_count: usize, _target_count: usize) {} - fn delete_blocks_finished(&mut self) {} + fn delete_blocks(&self, _current_count: usize, _target_count: usize) {} + fn delete_blocks_finished(&self) {} } pub trait RestoreMonitor { - fn restore_entry(&mut self, _entry: &IndexEntry) {} - fn restore_entry_result(&mut self, _entry: &IndexEntry, _result: &Result<()>) {} + fn restore_entry(&self, _entry: &IndexEntry) {} + fn restore_entry_result(&self, _entry: &IndexEntry, _result: &Result<()>) {} } /// Default monitor which does nothing. @@ -78,7 +78,7 @@ impl ValidateMonitor for DefaultMonitor {} impl TreeSizeMonitor for DefaultMonitor {} impl ReferencedBlocksMonitor for DefaultMonitor {} impl DeleteMonitor for DefaultMonitor { - fn referenced_blocks_monitor(&mut self) -> &mut dyn ReferencedBlocksMonitor { + fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor { self } } diff --git a/src/restore.rs b/src/restore.rs index 0426c113..2492e801 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -59,7 +59,7 @@ pub fn restore( archive: &Archive, destination_path: &Path, options: &RestoreOptions, - monitor: Option<&mut dyn RestoreMonitor>, + monitor: Option<&dyn RestoreMonitor>, ) -> Result { let mut default_monitor = DefaultMonitor{}; let monitor = monitor.unwrap_or(&mut default_monitor); diff --git a/src/tree.rs b/src/tree.rs index 58fb3482..89ccd50b 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -43,9 +43,9 @@ pub trait ReadTree { /// Measure the tree size. /// /// This typically requires walking all entries, which may take a while. - fn size(&self, exclude: Exclude, monitor: Option<&mut dyn TreeSizeMonitor>) -> Result where Self: Sized { + fn size(&self, exclude: Exclude, monitor: Option<&dyn TreeSizeMonitor>) -> Result where Self: Sized { let mut default_monitor = DefaultMonitor{}; - let monitor = monitor.unwrap_or(&mut default_monitor as &mut dyn TreeSizeMonitor); + let monitor = monitor.unwrap_or(&mut default_monitor as &dyn TreeSizeMonitor); let mut tot = 0u64; for e in self.iter_entries(Apath::root(), exclude)? { diff --git a/src/validate.rs b/src/validate.rs index 81111ad5..d4af65f0 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -76,7 +76,7 @@ impl BlockLengths { pub(crate) fn validate_bands( archive: &Archive, band_ids: &[BandId], - monitor: &mut dyn ValidateMonitor, + monitor: &dyn ValidateMonitor, ) -> (BlockLengths, ValidateStats) { let mut stats = ValidateStats::default(); let mut block_lens = BlockLengths::new(); @@ -101,7 +101,7 @@ pub(crate) fn validate_bands( (block_lens, stats) } -pub(crate) fn validate_band(archive: &Archive, stats: &mut ValidateStats, band_id: &BandId, monitor: &mut dyn ValidateMonitor) -> BandValidateResult { +pub(crate) fn validate_band(archive: &Archive, stats: &mut ValidateStats, band_id: &BandId, monitor: &dyn ValidateMonitor) -> BandValidateResult { let band = match Band::open(archive, band_id) { Ok(band) => band, Err(error) => return BandValidateResult::OpenError(error) From cf487578e77bf23ec44b42ccca1c666fefcee801 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:28:06 +0200 Subject: [PATCH 19/39] Implementing support for no progress CLI flag --- src/bin/conserve/main.rs | 19 +++++++------------ src/bin/conserve/show.rs | 7 +++++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index b8361806..240839f4 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -292,17 +292,12 @@ impl Command { ..Default::default() }; - let mut monitor = if args.no_progress { - None - } else { - Some(NutmegMonitor::new(BackupProgressModel::default())) - }; - + let monitor = NutmegMonitor::new(BackupProgressModel::default(), !args.no_progress); let stats = backup( &Archive::open(open_transport(archive)?)?, source, &options, - monitor.as_mut().map(|v| v as &dyn BackupMonitor) + Some(&monitor) )?; drop(monitor); @@ -346,7 +341,7 @@ impl Command { no_stats, } => { // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(DeleteProcessState::default()); + let mut monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)?.delete_bands( backup, @@ -385,7 +380,7 @@ impl Command { no_stats, } => { // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(DeleteProcessState::default()); + let mut monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( @@ -445,7 +440,7 @@ impl Command { }; // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose)); + let mut monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose), !args.no_progress); let stats = restore(&archive, destination, &options, Some(&mut monitor))?; if !no_stats { info!("Restore complete."); @@ -461,7 +456,7 @@ impl Command { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(SizeProgressModel::default()); + let mut monitor = NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? @@ -489,7 +484,7 @@ impl Command { }; // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(ValidateProgressModel::default()); + let mut monitor = NutmegMonitor::new(ValidateProgressModel::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; drop(monitor); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 3411264f..3dce6954 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -196,10 +196,13 @@ pub struct NutmegMonitor { } impl NutmegMonitor { - pub fn new(initial_state: T) -> Self { + pub fn new(initial_state: T, progress_enabled: bool) -> Self { + // FIXME: Speed up if `progress_enabled` is false. + // There is no need to proxy the log output. + // Also updating the state can be refactored. let view = Arc::new(Mutex::new(nutmeg::View::new( initial_state, - nutmeg::Options::default(), + nutmeg::Options::default().progress_enabled(progress_enabled), ))); let log_guard = log::update_terminal_target(view.clone()); From 9cb791fbdbc08040dc4e8a713cd325d688a6bff7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:36:48 +0200 Subject: [PATCH 20/39] Optimizing NutmegMonitor when not showing a progress bar --- src/bin/conserve/show.rs | 142 ++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 76 deletions(-) diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 3dce6954..b06c20a0 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -17,8 +17,8 @@ //! file (typically stdout). use std::borrow::Cow; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::{Duration, Instant}; +use std::sync::{Arc, Mutex}; +use std::time::{Instant}; use conserve::archive::ValidateArchiveProblem; use conserve::stats::Sizes; @@ -190,39 +190,54 @@ impl nutmeg::Model for BackupProgressModel { } } +enum NutmegMonitorState { + View { + view: Arc>>, + _log_guard: ViewLogGuard, + }, + Simple { + state: Mutex + } +} + pub struct NutmegMonitor { - _log_guard: ViewLogGuard, - view: Arc>>, + state: NutmegMonitorState, } impl NutmegMonitor { pub fn new(initial_state: T, progress_enabled: bool) -> Self { - // FIXME: Speed up if `progress_enabled` is false. - // There is no need to proxy the log output. - // Also updating the state can be refactored. - let view = Arc::new(Mutex::new(nutmeg::View::new( - initial_state, - nutmeg::Options::default().progress_enabled(progress_enabled), - ))); - - let log_guard = log::update_terminal_target(view.clone()); - Self { - _log_guard: log_guard, - view, - } + let state = if progress_enabled { + let view = Arc::new(Mutex::new(nutmeg::View::new( + initial_state, + nutmeg::Options::default().progress_enabled(progress_enabled), + ))); + + let log_guard = log::update_terminal_target(view.clone()); + NutmegMonitorState::View { view, _log_guard: log_guard } + } else { + NutmegMonitorState::Simple { state: Mutex::new(initial_state) } + }; + + Self { state } } -} -impl NutmegMonitor { - fn locked_view(&self) -> MutexGuard> { - self.view.lock().expect("lock() should not fail") + fn update_model R, R>(&self, update_fn: F) -> R { + match &self.state { + NutmegMonitorState::View { view, .. } => { + let view = view.lock().expect("lock() should not fail"); + view.update(update_fn) + }, + NutmegMonitorState::Simple { state } => { + let mut state = state.lock().expect("lock() should not fail"); + update_fn(&mut *state) + } + } } } impl BackupMonitor for NutmegMonitor { fn copy(&self, entry: &conserve::LiveEntry) { - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { model.filename = entry.apath().to_string(); match entry.kind() { Kind::Dir => model.scanned_dirs += 1, @@ -233,9 +248,8 @@ impl BackupMonitor for NutmegMonitor { } fn copy_result(&self, entry: &conserve::LiveEntry, result: &Option) { - let view = self.locked_view(); if let Some(diff_kind) = result.as_ref() { - view.update(|model| match diff_kind { + self.update_model(|model| match diff_kind { &DiffKind::Changed => model.entries_changed += 1, &DiffKind::New => model.entries_new += 1, &DiffKind::Unchanged => model.entries_unchanged += 1, @@ -244,14 +258,13 @@ impl BackupMonitor for NutmegMonitor { } if let Some(size) = entry.size() { - view.update(|model| model.scanned_file_bytes += size); + self.update_model(|model| model.scanned_file_bytes += size); } } fn copy_error(&self, entry: &conserve::LiveEntry, _error: &conserve::Error) { - let view = self.locked_view(); if let Some(size) = entry.size() { - view.update(|model| model.scanned_file_bytes += size); + self.update_model(|model| model.scanned_file_bytes += size); } } } @@ -365,13 +378,11 @@ impl ValidateMonitor for NutmegMonitor { fn count_bands_result(&self, bands: &[conserve::BandId]) { info!("Checking {} bands...", bands.len()); - let view = self.locked_view(); - view.update(|model| model.bands_total = Some(bands.len())); + self.update_model(|model| model.bands_total = Some(bands.len())); } fn validate_bands(&self) { - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { let bands_total = model.bands_total.expect("bands have been counted"); model.state = ValidateProgressState::ValidateBands { bands_done: 0, @@ -404,8 +415,7 @@ impl ValidateMonitor for NutmegMonitor { _band_id: &conserve::BandId, _result: &conserve::BandValidateResult, ) { - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { *bands_done += 1; } else { @@ -415,23 +425,18 @@ impl ValidateMonitor for NutmegMonitor { } fn validate_bands_finished(&self) { - let mut elapsed: Option = None; - // We can't use logging while locked_view is held since we would deadlock. - { - let view = self.locked_view(); - view.update(|model| { - if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { - elapsed = Some(start.elapsed()); - } else { - panic!("Expected state ValidateProgressState::ValidateBands"); - } - }); - } + let elapsed = self.update_model(|model| { + if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { + start.elapsed() + } else { + panic!("Expected state ValidateProgressState::ValidateBands"); + } + }); info!( "Finished validating bands in {:#?}.", - elapsed.expect("elapsed to be set") + elapsed ); } @@ -440,8 +445,7 @@ impl ValidateMonitor for NutmegMonitor { info!("Count blocks..."); } - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { model.state = ValidateProgressState::ListBlockes { discovered: current_count, } @@ -451,8 +455,7 @@ impl ValidateMonitor for NutmegMonitor { fn read_blocks(&self, count: usize) { info!("Check {} blocks...", count.separate_with_commas()); - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { model.state = ValidateProgressState::ReadBlocks { total_blocks: count, blocks_done: 0, @@ -467,9 +470,7 @@ impl ValidateMonitor for NutmegMonitor { _block_hash: &conserve::BlockHash, result: &Result<(Vec, Sizes)>, ) { - let view = self.locked_view(); - - view.update(|model| { + self.update_model(|model| { if let ValidateProgressState::ReadBlocks { blocks_done, bytes_done, @@ -518,8 +519,7 @@ impl nutmeg::Model for SizeProgressModel { impl TreeSizeMonitor for NutmegMonitor { fn entry_discovered(&self, _entry: &::Entry, size: &Option) { - let view = self.locked_view(); - view.update(|model| { + self.update_model(|model| { model.files += 1; model.total_bytes += size.unwrap_or(0); }); @@ -581,29 +581,25 @@ impl DeleteMonitor for NutmegMonitor { } fn find_present_blocks(&self, current_count: usize) { - let view = self.locked_view(); - view.update(|view| { + self.update_model(|view| { *view = DeleteProcessState::FindPresentBlocks { count: current_count }; }); } fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { - let view = self.locked_view(); - view.update(|view| { + self.update_model(|view| { *view = DeleteProcessState::MeasureUnreferencedBlocks { count: current_count, target: target_count }; }); } fn delete_bands(&self, current_count: usize, target_count: usize) { - let view = self.locked_view(); - view.update(|view| { + self.update_model(|view| { *view = DeleteProcessState::DeleteBands { count: current_count, target: target_count }; }); } fn delete_blocks(&self, current_count: usize, target_count: usize) { - let view = self.locked_view(); - view.update(|view| { + self.update_model(|view| { *view = DeleteProcessState::DeleteBlocks { count: current_count, target: target_count }; }); } @@ -611,8 +607,7 @@ impl DeleteMonitor for NutmegMonitor { impl ReferencedBlocksMonitor for NutmegMonitor { fn list_referenced_blocks(&self, current_count: usize) { - let view = self.locked_view(); - view.update(|view| { + self.update_model(|view| { *view = DeleteProcessState::ListReferencedBlocks { count: current_count }; }); } @@ -646,14 +641,10 @@ impl nutmeg::Model for RestoreProgressModel { impl RestoreMonitor for NutmegMonitor { fn restore_entry(&self, entry: &IndexEntry) { - let mut print_filename = false; - { - let view = self.locked_view(); - view.update(|view| { - print_filename = view.print_filenames; - view.filename = entry.apath().to_string(); - }); - } + let print_filename = self.update_model(|view| { + view.filename = entry.apath().to_string(); + view.print_filenames + }); if print_filename { info!("{}", entry.apath()); @@ -662,8 +653,7 @@ impl RestoreMonitor for NutmegMonitor { fn restore_entry_result(&self, entry: &IndexEntry, _result: &Result<()>) { if let Some(bytes) = entry.size() { - let view = self.locked_view(); - view.update(|view| view.bytes_done += bytes); + self.update_model(|view| view.bytes_done += bytes); } } } \ No newline at end of file From 0693108acc848f06c964e65b36372cbe3b85aeea Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:42:56 +0200 Subject: [PATCH 21/39] Removing crate::ui print functions and replace whem with tracing::error / tracing::info --- src/bin/conserve/main.rs | 2 - src/blockdir.rs | 14 +++---- src/index.rs | 5 ++- src/live_tree.rs | 35 ++++++++-------- src/restore.rs | 10 ++--- src/ui.rs | 86 ---------------------------------------- 6 files changed, 33 insertions(+), 119 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 240839f4..b104851a 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -505,7 +505,6 @@ impl Command { sizes, utc, } => { - ui::enable_progress(false); let archive = Archive::open(open_transport(archive)?)?; let options = ShowVersionsOptions { newest_first: *newest, @@ -573,7 +572,6 @@ fn main() -> ExitCode { } }; - ui::enable_progress(!args.no_progress); let result = args.command.run(&args); let exit_code = match result { Err(ref e) => { diff --git a/src/blockdir.rs b/src/blockdir.rs index 3e65d7fd..48529b16 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -31,7 +31,7 @@ use rayon::prelude::*; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; use serde::{Deserialize, Serialize}; -use tracing::warn; +use tracing::{ warn, error }; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; @@ -120,10 +120,10 @@ impl BlockDir { .or_else(|io_err| { if io_err.kind() == io::ErrorKind::AlreadyExists { // Perhaps it was simultaneously created by another thread or process. - ui::problem(&format!( + error!( "Unexpected late detection of existing block {:?}", hex_hash - )); + ); Ok(()) } else { Err(Error::WriteBlock { @@ -203,10 +203,10 @@ impl BlockDir { if dirname.len() == SUBDIR_NAME_CHARS { true } else { - ui::problem(&format!( + error!( "Unexpected subdirectory in blockdir: {:?}", dirname - )); + ); false } }); @@ -221,14 +221,14 @@ impl BlockDir { .map(move |subdir_name| transport.iter_dir_entries(&subdir_name)) .filter_map(|iter_or| { if let Err(ref err) = iter_or { - ui::problem(&format!("Error listing block directory: {:?}", &err)); + error!("Error listing block directory: {:?}", &err); } iter_or.ok() }) .flatten() .filter_map(|iter_or| { if let Err(ref err) = iter_or { - ui::problem(&format!("Error listing block subdirectory: {:?}", &err)); + error!("Error listing block subdirectory: {:?}", &err); } iter_or.ok() }) diff --git a/src/index.rs b/src/index.rs index f9cbb94c..1620db50 100644 --- a/src/index.rs +++ b/src/index.rs @@ -19,6 +19,7 @@ use std::iter::Peekable; use std::path::Path; use std::sync::Arc; use std::vec; +use tracing::error; use crate::compress::snappy::{Compressor, Decompressor}; use crate::kind::Kind; @@ -320,10 +321,10 @@ impl Iterator for IndexHunkIter { Ok(Some(entries)) => entries, Err(err) => { self.stats.errors += 1; - ui::problem(&format!( + error!( "Error reading index hunk {:?}: {:?} ", hunk_number, err - )); + ); continue; } }; diff --git a/src/live_tree.rs b/src/live_tree.rs index dd4d3224..b3e2eeb3 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -17,6 +17,7 @@ use std::collections::vec_deque::VecDeque; use std::fs; use std::io::ErrorKind; use std::path::{Path, PathBuf}; +use tracing::error; use crate::stats::LiveTreeIterStats; use crate::unix_time::UnixTime; @@ -196,7 +197,7 @@ impl Iter { let dir_iter = match fs::read_dir(&dir_path) { Ok(i) => i, Err(e) => { - ui::problem(&format!("Error reading directory {:?}: {}", &dir_path, e)); + error!("Error reading directory {:?}: {}", &dir_path, e); return; } }; @@ -205,10 +206,10 @@ impl Iter { let dir_entry = match dir_entry { Ok(dir_entry) => dir_entry, Err(e) => { - ui::problem(&format!( + error!( "Error reading next entry from directory {:?}: {}", &dir_path, e - )); + ); continue; } }; @@ -216,10 +217,10 @@ impl Iter { let child_name = match child_osstr.to_str() { Some(c) => c, None => { - ui::problem(&format!( + error!( "Couldn't decode filename {:?} in {:?}", child_osstr, dir_path, - )); + ); continue; } }; @@ -233,10 +234,10 @@ impl Iter { let ft = match dir_entry.file_type() { Ok(ft) => ft, Err(e) => { - ui::problem(&format!( + error!( "Error getting type of {:?} during iteration: {}", child_apath, e - )); + ); continue; } }; @@ -247,10 +248,10 @@ impl Iter { Ok(true) => continue, Ok(false) => (), Err(e) => { - ui::problem(&format!( + error!( "Error checking CACHEDIR.TAG in {:?}: {}", dir_entry, e - )); + ); } } } @@ -262,16 +263,16 @@ impl Iter { ErrorKind::NotFound => { // Fairly harmless, and maybe not even worth logging. Just a race // between listing the directory and looking at the contents. - ui::problem(&format!( + error!( "File disappeared during iteration: {:?}: {}", child_apath, e - )); + ); } _ => { - ui::problem(&format!( + error!( "Failed to read source metadata from {:?}: {}", child_apath, e - )); + ); self.stats.metadata_error += 1; } }; @@ -285,20 +286,20 @@ impl Iter { let t = match dir_path.join(dir_entry.file_name()).read_link() { Ok(t) => t, Err(e) => { - ui::problem(&format!( + error!( "Failed to read target of symlink {:?}: {}", child_apath, e - )); + ); continue; } }; match t.into_os_string().into_string() { Ok(t) => Some(t), Err(e) => { - ui::problem(&format!( + error!( "Failed to decode target of symlink {:?}: {:?}", child_apath, e - )); + ); continue; } } diff --git a/src/restore.rs b/src/restore.rs index 2492e801..c7c279e8 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -22,7 +22,7 @@ use filetime::{set_file_handle_times}; #[cfg(unix)] use filetime::{set_symlink_file_times}; -use tracing::debug; +use tracing::{debug, warn}; use crate::band::BandSelectionPolicy; use crate::entry::Entry; @@ -164,7 +164,7 @@ impl RestoreTree { fn finish(self) -> Result { for (path, time) in self.dir_mtimes { if let Err(err) = filetime::set_file_mtime(path, time.into()) { - ui::problem(&format!("Failed to set directory mtime: {:?}", err)); + warn!("Failed to set directory mtime: {:?}", err); } } Ok(RestoreStats::default()) @@ -228,7 +228,7 @@ impl RestoreTree { } } else { // TODO: Treat as an error. - ui::problem(&format!("No target in symlink entry {}", entry.apath())); + warn!("No target in symlink entry {}", entry.apath()); } Ok(()) } @@ -237,10 +237,10 @@ impl RestoreTree { fn copy_symlink(&mut self, entry: &E) -> Result<()> { // TODO: Add a test with a canned index containing a symlink, and expect // it cannot be restored on Windows and can be on Unix. - ui::problem(&format!( + warn!( "Can't restore symlinks on non-Unix: {}", entry.apath() - )); + ); Ok(()) } } diff --git a/src/ui.rs b/src/ui.rs index c333781b..9275cc90 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -14,51 +14,10 @@ //! Console UI. use std::fmt::Write; -use std::sync::Mutex; use std::time::Duration; -use lazy_static::lazy_static; -use tracing::{debug, error}; - use crate::stats::Sizes; -/// A terminal/text UI. -/// -/// This manages interleaving log-type messages (info and error), interleaved -/// with progress bars. -/// -/// Progress bars are only drawn when the application requests them with -/// `enable_progress` and the output destination is a tty that's capable -/// of redrawing. -/// -/// So this class also works when stdout is redirected to a file, in -/// which case it will get only messages and no progress bar junk. -#[derive(Default)] -pub(crate) struct UIState { - /// Should a progress bar be drawn? - progress_enabled: bool, -} - -lazy_static! { - static ref UI_STATE: Mutex = Mutex::new(UIState::default()); -} - -pub fn println(s: &str) { - with_locked_ui(|ui| ui.println(s)) -} - -pub fn problem(s: &str) { - with_locked_ui(|ui| ui.problem(s)); -} - -pub(crate) fn with_locked_ui(mut cb: F) -where - F: FnMut(&mut UIState), -{ - use std::ops::DerefMut; - cb(UI_STATE.lock().unwrap().deref_mut()) -} - pub(crate) fn format_error_causes(error: &dyn std::error::Error) -> String { let mut buf = error.to_string(); let mut cause = error; @@ -69,22 +28,6 @@ pub(crate) fn format_error_causes(error: &dyn std::error::Error) -> String { buf } -/// Report that a non-fatal error occurred. -/// -/// The program will continue. -pub fn show_error(e: &dyn std::error::Error) { - // TODO: Log it. - problem(&format_error_causes(e)); -} - -/// Enable drawing progress bars, only if stdout is a tty. -/// -/// Progress bars are off by default. -pub fn enable_progress(enabled: bool) { - let mut ui = UI_STATE.lock().unwrap(); - ui.progress_enabled = enabled; -} - #[allow(unused)] pub(crate) fn compression_percent(s: &Sizes) -> i64 { if s.uncompressed > 0 { @@ -129,35 +72,6 @@ pub(crate) fn compression_ratio(s: &Sizes) -> f64 { } } -// FIXME: Don't use these functions any more. -// Directly log to tracing or to the monitor. -impl UIState { - pub(crate) fn println(&mut self, s: &str) { - // TODO: Go through Nutmeg instead... - // self.clear_progress(); - debug!("{}", s); - } - - fn problem(&mut self, s: &str) { - // TODO: Go through Nutmeg instead... - // self.clear_progress(); - error!("{}", s); - // Drawing this way makes messages leak from tests, for unclear reasons. - - // queue!( - // stdout, - // style::SetForegroundColor(style::Color::Red), - // style::SetAttribute(style::Attribute::Bold), - // style::Print("conserve error: "), - // style::SetAttribute(style::Attribute::Reset), - // style::Print(s), - // style::Print("\n"), - // style::ResetColor, - // ) - // .unwrap(); - } -} - #[cfg(test)] mod tests { use super::*; From 8978bcc4617702cdd1e4f6b5bcf00cf89f7ddf5d Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 14:44:31 +0200 Subject: [PATCH 22/39] Printing stats line by line trough tracing --- src/bin/conserve/main.rs | 16 ++++++++++++---- src/ui.rs | 1 - 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index b104851a..56c3345c 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -352,7 +352,9 @@ impl Command { Some(&mut monitor), )?; if !no_stats { - info!("{}", stats); + for line in format!("{}", stats).lines() { + info!("{}", line); + } } } Command::Diff { @@ -392,7 +394,9 @@ impl Command { Some(&mut monitor), )?; if !no_stats { - info!("{}", stats); + for line in format!("{}", stats).lines() { + info!("{}", line); + } } } Command::Init { archive } => { @@ -444,7 +448,9 @@ impl Command { let stats = restore(&archive, destination, &options, Some(&mut monitor))?; if !no_stats { info!("Restore complete."); - info!("{}", stats); + for line in format!("{}", stats).lines() { + info!("{}", line); + } } } Command::Size { @@ -489,7 +495,9 @@ impl Command { drop(monitor); if !no_stats { - info!("{}", stats); + for line in format!("{}", stats).lines() { + info!("{}", line); + } } if stats.has_problems() { warn!("Archive has some problems."); diff --git a/src/ui.rs b/src/ui.rs index 9275cc90..5601d796 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -37,7 +37,6 @@ pub(crate) fn compression_percent(s: &Sizes) -> i64 { } } -// FIXME: Move into conserve binary pub fn duration_to_hms(d: Duration) -> String { let elapsed_secs = d.as_secs(); if elapsed_secs >= 3600 { From 8912fb1cc8842ee0c9e8d583908cebfa172fe206 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 15:10:09 +0200 Subject: [PATCH 23/39] Adding a raw terminal mode hiding level and timestamp information --- src/bin/conserve/log.rs | 24 ++++++++++++------------ src/bin/conserve/main.rs | 7 ++++++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index d0a6bf53..074ad5a3 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -47,23 +47,24 @@ impl Write for TerminalWriter { pub struct LoggingOptions { pub file: Option, pub level: tracing::Level, + pub terminal_raw: bool, } pub fn init(options: LoggingOptions) -> std::result::Result { let mut worker_guard = None; let registry = Registry::default(); - + // Terminal logger. - let registry = { - registry.with( - fmt::Layer::default() - //.without_time() - //.with_level(false) - .with_target(false) - .with_writer(|| TerminalWriter{}) - .with_filter(LevelFilter::from(options.level)) - ) - }; + // TODO: Enable timestamps except when in raw mode. + // Right now this can't be achived since without_time updates the struct signature... + let registry = registry.with( + fmt::Layer::default() + .without_time() + .with_level(!options.terminal_raw) + .with_target(false) + .with_writer(|| TerminalWriter{}) + .with_filter(LevelFilter::from(options.level)) + ); // File logger. let registry: Box = if let Some(path) = options.file { @@ -92,7 +93,6 @@ pub fn init(options: LoggingOptions) -> std::result::Result { Box::new(registry) }; - tracing::subscriber::set_global_default(registry) .map_err(|_| "Failed to update global default logger".to_string())?; diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 56c3345c..d09e234b 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -53,6 +53,10 @@ struct Args { #[clap(long, short = 'D', global = true)] debug: bool, + /// Don't show log timestamps and levels for the terminal output. + #[clap(long, short = 'R', global = true)] + log_raw: bool, + /// Set the log level to trace #[clap(long, short = 'L', global = true)] log_level: Option, @@ -559,7 +563,8 @@ fn initialize_log(args: &Args) -> std::result::Result { let guard = log::init(LoggingOptions{ file, - level + level, + terminal_raw: args.log_raw })?; if args.log_level == Some(tracing::Level::TRACE) { From 7b89f0e3e98a539a21e46ad895187a611f0b29c3 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 15:35:32 +0200 Subject: [PATCH 24/39] Fixing verbose parameter for backups --- src/backup.rs | 4 ---- src/bin/conserve/main.rs | 6 ++++-- src/bin/conserve/show.rs | 23 +++++++++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 18cfb2f9..c433b492 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -31,9 +31,6 @@ use crate::*; /// Configuration of how to make a backup. #[derive(Debug, Clone)] pub struct BackupOptions { - /// Print filenames to the UI as they're copied. - pub print_filenames: bool, - /// Exclude these globs from the backup. pub exclude: Exclude, @@ -43,7 +40,6 @@ pub struct BackupOptions { impl Default for BackupOptions { fn default() -> BackupOptions { BackupOptions { - print_filenames: false, exclude: Exclude::nothing(), max_entries_per_hunk: crate::index::MAX_ENTRIES_PER_HUNK, } diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index d09e234b..4b4e7829 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -291,12 +291,14 @@ impl Command { let exclude = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; let source = &LiveTree::open(source)?; let options = BackupOptions { - print_filenames: *verbose, exclude, ..Default::default() }; - let monitor = NutmegMonitor::new(BackupProgressModel::default(), !args.no_progress); + let mut model = BackupProgressModel::default(); + model.verbose = *verbose; + + let monitor = NutmegMonitor::new(model, !args.no_progress); let stats = backup( &Archive::open(open_transport(archive)?)?, source, diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index b06c20a0..3e558e0f 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -167,10 +167,13 @@ pub fn show_diff>(diff: D) -> Result<()> { #[derive(Default)] pub struct BackupProgressModel { + pub verbose: bool, filename: String, + scanned_file_bytes: u64, scanned_dirs: usize, scanned_files: usize, + entries_new: usize, entries_changed: usize, entries_unchanged: usize, @@ -249,12 +252,20 @@ impl BackupMonitor for NutmegMonitor { fn copy_result(&self, entry: &conserve::LiveEntry, result: &Option) { if let Some(diff_kind) = result.as_ref() { - self.update_model(|model| match diff_kind { - &DiffKind::Changed => model.entries_changed += 1, - &DiffKind::New => model.entries_new += 1, - &DiffKind::Unchanged => model.entries_unchanged += 1, - &DiffKind::Deleted => model.entries_deleted += 1, - }) + let verbose = self.update_model(|model| { + match diff_kind { + &DiffKind::Changed => model.entries_changed += 1, + &DiffKind::New => model.entries_new += 1, + &DiffKind::Unchanged => model.entries_unchanged += 1, + &DiffKind::Deleted => model.entries_deleted += 1, + }; + + model.verbose + }); + + if verbose { + info!("{} {}", diff_kind.as_sigil(), entry.apath()); + } } if let Some(size) = entry.size() { From 73fcf8efd873354cc8e47499056766d6507f7c2f Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 15:35:56 +0200 Subject: [PATCH 25/39] Fixing CLI tests now passing the raw terminal argument --- tests/api/backup.rs | 13 ++++++------- tests/api/delete.rs | 2 +- tests/api/gc.rs | 8 +++++--- tests/api/old_archives.rs | 9 +++------ tests/api/restore.rs | 8 ++++---- tests/cli/backup.rs | 2 +- tests/cli/delete.rs | 3 ++- tests/cli/diff.rs | 13 +++++++++++++ tests/cli/exclude.rs | 10 +++++----- tests/cli/main.rs | 18 +++++++++++------- tests/cli/versions.rs | 10 ++++++---- tests/expensive/changes.rs | 2 +- 12 files changed, 58 insertions(+), 40 deletions(-) diff --git a/tests/api/backup.rs b/tests/api/backup.rs index 447eae79..c7e872b5 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -47,7 +47,7 @@ pub fn simple_backup() { assert!(archive.band_is_closed(&BandId::zero()).unwrap()); assert!(!archive.band_exists(&BandId::new(&[1])).unwrap()); let copy_stats = - restore(&archive, restore_dir.path(), &RestoreOptions::default()).expect("restore"); + restore(&archive, restore_dir.path(), &RestoreOptions::default(), None).expect("restore"); assert_eq!(copy_stats.uncompressed_file_bytes, 8); } @@ -89,7 +89,7 @@ pub fn simple_backup_with_excludes() -> Result<()> { assert!(band_info.end_time.is_some()); let copy_stats = - restore(&archive, restore_dir.path(), &RestoreOptions::default()).expect("restore"); + restore(&archive, restore_dir.path(), &RestoreOptions::default(), None).expect("restore"); assert_eq!(copy_stats.uncompressed_file_bytes, 8); // TODO: Read back contents of that file. @@ -121,7 +121,6 @@ pub fn backup_more_excludes() { let source = srcdir.live_tree(); let options = BackupOptions { exclude, - print_filenames: false, ..Default::default() }; let stats = backup(&af, &source, &options, None).expect("backup"); @@ -160,7 +159,7 @@ fn check_backup(af: &ScratchArchive) { assert!(file_entry.mtime > 0); assert_eq!( - af.referenced_blocks(&af.list_band_ids().unwrap()) + af.referenced_blocks(&af.list_band_ids().unwrap(), None) .unwrap() .into_iter() .map(|h| h.to_string()) @@ -175,7 +174,7 @@ fn check_backup(af: &ScratchArchive) { .collect::>(), vec![HELLO_HASH] ); - assert_eq!(af.unreferenced_blocks().unwrap().count(), 0); + assert_eq!(af.unreferenced_blocks(None).unwrap().count(), 0); } #[test] @@ -200,7 +199,7 @@ fn large_file() { let rd = TempDir::new().unwrap(); let restore_archive = Archive::open_path(af.path()).unwrap(); let restore_stats = - restore(&restore_archive, rd.path(), &RestoreOptions::default()).expect("restore"); + restore(&restore_archive, rd.path(), &RestoreOptions::default(), None).expect("restore"); assert_eq!(restore_stats.files, 1); let content = std::fs::read(rd.path().join("large")).unwrap(); @@ -308,7 +307,7 @@ pub fn empty_file_uses_zero_blocks() { // Restore it let dest = TempDir::new().unwrap(); - restore(&af, dest.path(), &RestoreOptions::default()).expect("restore"); + restore(&af, dest.path(), &RestoreOptions::default(), None).expect("restore"); // TODO: Check restore stats. dest.child("empty").assert(""); } diff --git a/tests/api/delete.rs b/tests/api/delete.rs index 8b2cc3d0..f4a47134 100644 --- a/tests/api/delete.rs +++ b/tests/api/delete.rs @@ -21,7 +21,7 @@ fn delete_all_bands() { af.store_two_versions(); let stats = af - .delete_bands(&[BandId::new(&[0]), BandId::new(&[1])], &Default::default()) + .delete_bands(&[BandId::new(&[0]), BandId::new(&[1])], &Default::default(), None) .expect("delete_bands"); assert_eq!(stats.deleted_block_count, 2); diff --git a/tests/api/gc.rs b/tests/api/gc.rs index cc0d18b3..73fa8e31 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -31,7 +31,7 @@ fn unreferenced_blocks() { // Delete the band and index std::fs::remove_dir_all(archive.path().join("b0000")).unwrap(); - let unreferenced: Vec = archive.unreferenced_blocks().unwrap().collect(); + let unreferenced: Vec = archive.unreferenced_blocks(None).unwrap().collect(); assert_eq!(unreferenced, [content_hash]); // Delete dry run. @@ -42,6 +42,7 @@ fn unreferenced_blocks() { dry_run: true, break_lock: false, }, + None, ) .unwrap(); assert_eq!( @@ -61,7 +62,7 @@ fn unreferenced_blocks() { dry_run: false, break_lock: false, }; - let delete_stats = archive.delete_bands(&[], &options).unwrap(); + let delete_stats = archive.delete_bands(&[], &options, None).unwrap(); assert_eq!( delete_stats, DeleteStats { @@ -75,7 +76,7 @@ fn unreferenced_blocks() { ); // Try again to delete: should find no garbage. - let delete_stats = archive.delete_bands(&[], &options).unwrap(); + let delete_stats = archive.delete_bands(&[], &options, None).unwrap(); assert_eq!( delete_stats, DeleteStats { @@ -112,6 +113,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { break_lock: true, ..Default::default() }, + None )?; // Backup should now succeed. diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index b703c797..9e385f7f 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -94,7 +94,7 @@ fn restore_old_archive() { let archive = open_old_archive(ver, "minimal"); let restore_stats = - restore(&archive, dest.path(), &RestoreOptions::default()).expect("restore"); + restore(&archive, dest.path(), &RestoreOptions::default(), None).expect("restore"); assert_eq!(restore_stats.files, 2); assert_eq!(restore_stats.symlinks, 0); @@ -136,7 +136,7 @@ fn restore_modify_backup() { let archive = open_old_archive(ver, "minimal"); - restore(&archive, working_tree.path(), &RestoreOptions::default()).expect("restore"); + restore(&archive, working_tree.path(), &RestoreOptions::default(), None).expect("restore"); // Write back into a new copy of the archive, without modifying the // testdata in the source tree. @@ -161,10 +161,7 @@ fn restore_modify_backup() { let backup_stats = backup( &new_archive, &LiveTree::open(working_tree.path()).unwrap(), - &BackupOptions { - print_filenames: true, - ..Default::default() - }, + &Default::default(), None ) .expect("Backup modified tree"); diff --git a/tests/api/restore.rs b/tests/api/restore.rs index ca7504ea..737530b1 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -32,7 +32,7 @@ fn simple_restore() { let options = RestoreOptions::default(); let restore_archive = Archive::open_path(af.path()).unwrap(); - let stats = restore(&restore_archive, destdir.path(), &options).expect("restore"); + let stats = restore(&restore_archive, destdir.path(), &options, None).expect("restore"); assert_eq!(stats.files, 3); @@ -60,7 +60,7 @@ fn restore_specified_band() { band_selection: BandSelectionPolicy::Specified(band_id), ..RestoreOptions::default() }; - let stats = restore(&archive, destdir.path(), &options).expect("restore"); + let stats = restore(&archive, destdir.path(), &options, None).expect("restore"); // Does not have the 'hello2' file added in the second version. assert_eq!(stats.files, 2); } @@ -89,7 +89,7 @@ pub fn forced_overwrite() { overwrite: true, ..RestoreOptions::default() }; - let stats = restore(&restore_archive, destdir.path(), &options).expect("restore"); + let stats = restore(&restore_archive, destdir.path(), &options, None).expect("restore"); assert_eq!(stats.files, 3); let dest = &destdir.path(); assert!(dest.join("hello").is_file()); @@ -107,7 +107,7 @@ fn exclude_files() { exclude: Exclude::from_strings(&["/**/subfile"]).unwrap(), ..RestoreOptions::default() }; - let stats = restore(&restore_archive, destdir.path(), &options).expect("restore"); + let stats = restore(&restore_archive, destdir.path(), &options, None).expect("restore"); let dest = &destdir.path(); assert!(dest.join("hello").is_file()); diff --git a/tests/cli/backup.rs b/tests/cli/backup.rs index 0b70cd43..897ef467 100644 --- a/tests/cli/backup.rs +++ b/tests/cli/backup.rs @@ -26,7 +26,7 @@ fn backup_verbose() { src.create_file("subdir/b"); run_conserve() - .args(&["backup", "--no-stats", "-v"]) + .args(&["backup", "--no-stats", "-v", "-R"]) .arg(af.path()) .arg(src.path()) .assert() diff --git a/tests/cli/delete.rs b/tests/cli/delete.rs index 7e91806d..56caf5d0 100644 --- a/tests/cli/delete.rs +++ b/tests/cli/delete.rs @@ -119,13 +119,14 @@ fn delete_nonexistent_band() { let af = ScratchArchive::new(); let pred_fn = predicate::str::is_match( - r"conserve error: Failed to delete band b0000 + r"Failed to delete band b0000 caused by: (No such file or directory|The system cannot find the file specified\.) \(os error \d+\) ", ) .unwrap(); run_conserve() + .arg("-R") .args(&["delete"]) .args(&["-b", "b0000"]) .arg(af.path()) diff --git a/tests/cli/diff.rs b/tests/cli/diff.rs index 0bcc8d7b..69a3dcf1 100644 --- a/tests/cli/diff.rs +++ b/tests/cli/diff.rs @@ -56,6 +56,7 @@ fn no_changes() { let (af, tf) = setup(); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -65,6 +66,7 @@ fn no_changes() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) @@ -82,6 +84,7 @@ fn add_entries() { tf.create_file_with_contents("src/new.rs", b"pub fn main() {}"); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -97,6 +100,7 @@ fn remove_file() { std::fs::remove_file(tf.path().join("hello.c")).unwrap(); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -106,6 +110,7 @@ fn remove_file() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) @@ -123,6 +128,7 @@ fn change_kind() { tf.create_file_with_contents("subdir", b"used to be a directory, no longer"); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -132,6 +138,7 @@ fn change_kind() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) @@ -149,6 +156,7 @@ fn change_file_content() { tf.create_file_with_contents("hello.c", b"int main() { abort(); }"); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -158,6 +166,7 @@ fn change_file_content() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) @@ -174,6 +183,7 @@ pub fn symlink_unchanged() { let (af, tf) = setup_symlink(); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -183,6 +193,7 @@ pub fn symlink_unchanged() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) @@ -201,6 +212,7 @@ pub fn symlink_changed() { tf.create_symlink("subdir/link", "newtarget"); run_conserve() + .arg("-R") .arg("diff") .arg(af.path()) .arg(tf.path()) @@ -210,6 +222,7 @@ pub fn symlink_changed() { .stderr(predicate::str::is_empty()); run_conserve() + .arg("-R") .arg("diff") .arg("--include-unchanged") .arg(af.path()) diff --git a/tests/cli/exclude.rs b/tests/cli/exclude.rs index 1cd14370..183a1201 100644 --- a/tests/cli/exclude.rs +++ b/tests/cli/exclude.rs @@ -50,7 +50,7 @@ fn exclude_simple_glob() { src.create_file("src/hello.o"); run_conserve() - .args(&["backup", "-v", "--exclude", "*.o", "--no-stats"]) + .args(&["-R", "backup", "-v", "--exclude", "*.o", "--no-stats"]) .arg(&af.path()) .arg(&src.path()) .assert() @@ -58,7 +58,7 @@ fn exclude_simple_glob() { .success(); run_conserve() - .args(&["ls"]) + .args(&["-R", "ls"]) .arg(&af.path()) .assert() .stdout("/\n/src\n/src/hello.c\n") @@ -76,7 +76,7 @@ fn exclude_glob_only_in_root() { src.create_file("src/hello.o"); run_conserve() - .args(&["backup", "-v", "--exclude", "/*.o", "--no-stats"]) + .args(&["-R", "backup", "-v", "--exclude", "/*.o", "--no-stats"]) .arg(&af.path()) .arg(&src.path()) .assert() @@ -84,7 +84,7 @@ fn exclude_glob_only_in_root() { .success(); run_conserve() - .args(&["ls"]) + .args(&["-R", "ls"]) .arg(&af.path()) .assert() .stdout("/\n/src\n/src/hello.c\n/src/hello.o\n") @@ -114,7 +114,7 @@ fn exclude_suffix_pattern() { .success(); run_conserve() - .args(&["ls"]) + .args(&["-R", "ls"]) .arg(&af.path()) .assert() .stdout("/\n/release\n/src\n/subproj\n/target\n/src/hello.rs\n/subproj/target\n") diff --git a/tests/cli/main.rs b/tests/cli/main.rs index f3095c1f..7035940b 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -78,6 +78,7 @@ fn basic_backup() { // conserve init run_conserve() + .arg("-R") .arg("init") .arg(&arch_dir) .assert() @@ -98,7 +99,7 @@ fn basic_backup() { assert!(src.is_dir()); run_conserve() - .args(&["ls", "--source"]) + .args(&["-R", "ls", "--source"]) .arg(&src) .assert() .success() @@ -111,7 +112,7 @@ fn basic_backup() { ); run_conserve() - .args(&["size", "-s"]) + .args(&["-R", "size", "-s"]) .arg(&src) .assert() .success() @@ -120,6 +121,7 @@ fn basic_backup() { // backup run_conserve() + .arg("-R") .arg("backup") .arg(&arch_dir) .arg(&src) @@ -130,7 +132,7 @@ fn basic_backup() { // TODO: Now inspect the archive. run_conserve() - .args(&["size"]) + .args(&["-R", "size"]) .arg(&arch_dir) .assert() .success() @@ -138,7 +140,7 @@ fn basic_backup() { .stdout("0 MB\n"); // "contents" run_conserve() - .args(&["versions", "--short"]) + .args(&["-R", "versions", "--short"]) .arg(&arch_dir) .assert() .success() @@ -156,7 +158,7 @@ fn basic_backup() { }; run_conserve() - .args(&["debug", "blocks"]) + .args(&["-R", "debug", "blocks"]) .arg(&arch_dir) .assert() .success() @@ -164,7 +166,7 @@ fn basic_backup() { .stdout(predicate::function(is_expected_blocks)); run_conserve() - .args(&["debug", "referenced"]) + .args(&["-R", "debug", "referenced"]) .arg(&arch_dir) .assert() .success() @@ -193,6 +195,7 @@ fn basic_backup() { // You can open it with a file URL. let file_url = Url::from_directory_path(&arch_dir).unwrap(); run_conserve() + .arg("-R") .arg("ls") .arg(file_url.as_str()) .assert() @@ -211,6 +214,7 @@ fn basic_backup() { // Also try --no-progress here; should make no difference because these tests run // without a pty. run_conserve() + .arg("-R") .arg("restore") .arg("-v") .arg("--no-progress") @@ -375,7 +379,7 @@ fn size_exclude() { source.create_file_with_contents("junk", b"01234567890123456789"); run_conserve() - .args(&["size", "--bytes", "--source"]) + .args(&["-R", "size", "--bytes", "--source"]) .arg(&source.path()) .args(&["--exclude=/junk"]) .assert() diff --git a/tests/cli/versions.rs b/tests/cli/versions.rs index e86c37c8..5feee355 100644 --- a/tests/cli/versions.rs +++ b/tests/cli/versions.rs @@ -23,7 +23,7 @@ use crate::run_conserve; #[test] fn utc() { run_conserve() - .args(&["versions", "--utc", "testdata/archive/simple/v0.6.10"]) + .args(&["versions", "--utc", "testdata/archive/simple/v0.6.10", "-R"]) .assert() .success() .stdout( @@ -43,6 +43,7 @@ fn newest_first() { "--newest", "--utc", "testdata/archive/simple/v0.6.10", + "-R" ]) .assert() .success() @@ -61,7 +62,7 @@ fn local_time() { // Without --utc we don't know exactly what times will be produced, // and it's hard to control the timezone for tests on Windows. run_conserve() - .args(&["versions", "testdata/archive/simple/v0.6.10"]) + .args(&["versions", "testdata/archive/simple/v0.6.10", "-R"]) .assert() .success() .stdout(function(|s: &str| s.lines().count() == 3)); @@ -70,7 +71,7 @@ fn local_time() { #[test] fn short() { run_conserve() - .args(&["versions", "--short", "testdata/archive/simple/v0.6.10"]) + .args(&["versions", "--short", "testdata/archive/simple/v0.6.10", "-R"]) .assert() .success() .stdout( @@ -90,6 +91,7 @@ fn tree_sizes() { "--sizes", "--utc", "testdata/archive/simple/v0.6.10", + "-R" ]) .assert() .success() @@ -108,7 +110,7 @@ fn short_newest_first() { af.store_two_versions(); run_conserve() - .args(&["versions", "--short", "--newest"]) + .args(&["versions", "--short", "--newest", "-R"]) .arg(af.path()) .assert() .success() diff --git a/tests/expensive/changes.rs b/tests/expensive/changes.rs index 7dd14ac2..dd17a766 100644 --- a/tests/expensive/changes.rs +++ b/tests/expensive/changes.rs @@ -125,7 +125,7 @@ fn check_restore_against_snapshot(archive: &Archive, band_id: BandId, snapshot: band_selection: BandSelectionPolicy::Specified(band_id), ..RestoreOptions::default() }; - restore(archive, restore_dir.path(), &options).unwrap(); + restore(archive, restore_dir.path(), &options, None).unwrap(); dir_assert::assert_paths(restore_dir.path(), snapshot).unwrap(); } From 64a748690645661e0945cf24206d7093198becb8 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 15:45:35 +0200 Subject: [PATCH 26/39] Some minor code cleanup --- src/bin/conserve/log.rs | 14 +++++-- src/bin/conserve/show.rs | 84 ++++++++++++++++++++-------------------- src/monitor.rs | 4 +- tests/cli/diff.rs | 4 +- 4 files changed, 56 insertions(+), 50 deletions(-) diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 074ad5a3..70f4d2c0 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -12,16 +12,17 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::Registry; use tracing_subscriber::fmt; -struct TerminalWriter { } - -impl TerminalWriter { } - lazy_static!{ pub static ref TERMINAL_OUTPUT: Mutex>>> = Mutex::new( Some(Arc::new(Mutex::new(std::io::stdout()))) ); } +/// Wrapper around TERMINAL_OUTPUT which dynamically writes +/// the tracing output to the output writer set at TERMINAL_OUTPUT. +struct TerminalWriter { } +impl TerminalWriter { } + impl Write for TerminalWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { let current_target = TERMINAL_OUTPUT.lock().expect("lock() should not fail"); @@ -50,6 +51,7 @@ pub struct LoggingOptions { pub terminal_raw: bool, } +/// Initialize tracing logging for the binary and the library. pub fn init(options: LoggingOptions) -> std::result::Result { let mut worker_guard = None; let registry = Registry::default(); @@ -106,6 +108,8 @@ pub struct LogGuard { _worker_guard: Option, } +/// Guard for the replaced log target. +/// When it drops it restores the last logging target. pub struct ViewLogGuard { released: bool, previous_logger: Option>>, @@ -130,6 +134,8 @@ impl Drop for ViewLogGuard { } } +/// Replace the current log target with a new one. +/// Returns a `ViewLogGuard` which will restore the old target when dropped. pub fn update_terminal_target(target: Arc>) -> ViewLogGuard { let mut output = TERMINAL_OUTPUT.lock().unwrap(); let previous_logger = output.replace(target); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 3e558e0f..b29df2c5 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -151,48 +151,6 @@ pub fn show_diff>(diff: D) -> Result<()> { Ok(()) } -// Considerations if we're trying properly extimate the remaining progress. -// -// This causes us to walk the source tree twice, which is probably an acceptable option -// since it's nice to see realistic overall progress. We could keep all the entries -// in memory, and maybe we should, but it might get unreasonably big. -// if options.measure_first { -// progress_bar.set_phase("Measure source tree".to_owned()); -// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it -// // again a second time? But, that'll potentially use memory proportional to tree size, which -// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were -// // deleted or changed while this is running. -// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); -// } - -#[derive(Default)] -pub struct BackupProgressModel { - pub verbose: bool, - filename: String, - - scanned_file_bytes: u64, - scanned_dirs: usize, - scanned_files: usize, - - entries_new: usize, - entries_changed: usize, - entries_unchanged: usize, - entries_deleted: usize, -} - -impl nutmeg::Model for BackupProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", - self.scanned_dirs, - self.scanned_files, - self.scanned_file_bytes / 1_000_000, - self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, - self.filename - ) - } -} - enum NutmegMonitorState { View { view: Arc>>, @@ -238,6 +196,48 @@ impl NutmegMonitor { } } +// Considerations if we're trying properly extimate the remaining progress. +// +// This causes us to walk the source tree twice, which is probably an acceptable option +// since it's nice to see realistic overall progress. We could keep all the entries +// in memory, and maybe we should, but it might get unreasonably big. +// if options.measure_first { +// progress_bar.set_phase("Measure source tree".to_owned()); +// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it +// // again a second time? But, that'll potentially use memory proportional to tree size, which +// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were +// // deleted or changed while this is running. +// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); +// } + +#[derive(Default)] +pub struct BackupProgressModel { + pub verbose: bool, + filename: String, + + scanned_file_bytes: u64, + scanned_dirs: usize, + scanned_files: usize, + + entries_new: usize, + entries_changed: usize, + entries_unchanged: usize, + entries_deleted: usize, +} + +impl nutmeg::Model for BackupProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", + self.scanned_dirs, + self.scanned_files, + self.scanned_file_bytes / 1_000_000, + self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, + self.filename + ) + } +} + impl BackupMonitor for NutmegMonitor { fn copy(&self, entry: &conserve::LiveEntry) { self.update_model(|model| { diff --git a/src/monitor.rs b/src/monitor.rs index 5070167a..9763b7f9 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -2,7 +2,6 @@ use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProbl /// Monitor the backup progress. pub trait BackupMonitor { - /// Will be called before the entry will be backupped fn copy(&self, _entry: &LiveEntry) {} fn copy_error(&self, _entry: &LiveEntry, _error: &Error) {} fn copy_result(&self, _entry: &LiveEntry, _result: &Option) {} @@ -43,11 +42,13 @@ pub trait TreeSizeMonitor { fn entry_discovered(&self, _entry: &T::Entry, _size: &Option) {} } +/// Monitor for iterating referenced blocks. pub trait ReferencedBlocksMonitor : Sync { fn list_referenced_blocks(&self, _current_count: usize) {} fn list_referenced_blocks_finished(&self) {} } +/// Monitor for deleting backups/blocks. pub trait DeleteMonitor : Sync { fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor; @@ -64,6 +65,7 @@ pub trait DeleteMonitor : Sync { fn delete_blocks_finished(&self) {} } +/// Monitor the progress of restoring files. pub trait RestoreMonitor { fn restore_entry(&self, _entry: &IndexEntry) {} fn restore_entry_result(&self, _entry: &IndexEntry, _result: &Result<()>) {} diff --git a/tests/cli/diff.rs b/tests/cli/diff.rs index 69a3dcf1..2677a666 100644 --- a/tests/cli/diff.rs +++ b/tests/cli/diff.rs @@ -13,8 +13,6 @@ //! Test `conserve diff`. -use std::fs; - use assert_cmd::prelude::*; use predicates::prelude::*; @@ -208,7 +206,7 @@ pub fn symlink_unchanged() { #[test] pub fn symlink_changed() { let (af, tf) = setup_symlink(); - fs::remove_file(tf.path().join("subdir/link")).unwrap(); + std::fs::remove_file(tf.path().join("subdir/link")).unwrap(); tf.create_symlink("subdir/link", "newtarget"); run_conserve() From 97541ec2a0df073520c55bb6ac688884c8bef5e8 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Tue, 9 Aug 2022 15:53:50 +0200 Subject: [PATCH 27/39] Added a monitor for the debug commands find (un-)referenced blocks --- src/bin/conserve/main.rs | 20 +++++++------------- src/bin/conserve/show.rs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 4b4e7829..a11e51b4 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,7 +20,7 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LoggingOptions, LogGuard}; -use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState, RestoreProgressModel}; +use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState, RestoreProgressModel, ReferencedBlocksProgressModel}; use show::{show_diff, ShowVersionsOptions, show_versions}; use tracing::{ trace, error, info, warn, Level }; @@ -328,14 +328,14 @@ impl Command { } Command::Debug(Debug::Referenced { archive }) => { let archive = Archive::open(open_transport(archive)?)?; - // FIXME: Monitor? - for hash in archive.referenced_blocks(&archive.list_band_ids()?, None)? { + let monitor = NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); + for hash in archive.referenced_blocks(&archive.list_band_ids()?, Some(&monitor))? { info!("{}", hash); } } Command::Debug(Debug::Unreferenced { archive }) => { - // FIXME: Monitor? - for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks(None)? { + let monitor = NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); + for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks(Some(&monitor))? { info!("{}", hash); } } @@ -346,16 +346,14 @@ impl Command { break_lock, no_stats, } => { - // FIXME: Respect the "no progress" option and only log messages. - let mut monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); - + let monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)?.delete_bands( backup, &DeleteOptions { dry_run: *dry_run, break_lock: *break_lock, }, - Some(&mut monitor), + Some(&monitor), )?; if !no_stats { for line in format!("{}", stats).lines() { @@ -387,7 +385,6 @@ impl Command { break_lock, no_stats, } => { - // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let archive = Archive::open(open_transport(archive)?)?; @@ -449,7 +446,6 @@ impl Command { overwrite: *force_overwrite, }; - // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose), !args.no_progress); let stats = restore(&archive, destination, &options, Some(&mut monitor))?; if !no_stats { @@ -467,7 +463,6 @@ impl Command { } => { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; - // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? @@ -495,7 +490,6 @@ impl Command { skip_block_hashes: *quick, }; - // FIXME: Respect the "no progress" option and only log messages. let mut monitor = NutmegMonitor::new(ValidateProgressModel::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; drop(monitor); diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index b29df2c5..57950045 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -667,4 +667,21 @@ impl RestoreMonitor for NutmegMonitor { self.update_model(|view| view.bytes_done += bytes); } } +} + +#[derive(Default)] +pub struct ReferencedBlocksProgressModel { + count: usize, +} + +impl Model for ReferencedBlocksProgressModel { + fn render(&mut self, _width: usize) -> String { + format!("Find referenced blocks in band ({} discovered)", self.count) + } +} + +impl ReferencedBlocksMonitor for NutmegMonitor { + fn list_referenced_blocks(&self, current_count: usize) { + self.update_model(|model| model.count = current_count); + } } \ No newline at end of file From 00ece3bc59577bb582655a5189d99efe8147837d Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 12 Aug 2022 17:15:09 +0200 Subject: [PATCH 28/39] Applied rust fmt --- src/archive.rs | 93 +++++++++++++++++++++++-------------- src/backup.rs | 6 +-- src/band.rs | 18 ++++--- src/bin/conserve/log.rs | 58 ++++++++++++----------- src/bin/conserve/main.rs | 67 +++++++++++++++----------- src/bin/conserve/show.rs | 98 ++++++++++++++++++++------------------- src/blockdir.rs | 33 +++++++------ src/gc_lock.rs | 15 +++++- src/index.rs | 5 +- src/lib.rs | 9 ++-- src/live_tree.rs | 10 +--- src/monitor.rs | 16 ++++--- src/restore.rs | 13 ++---- src/tree.rs | 11 ++++- src/validate.rs | 34 ++++++++------ tests/api/backup.rs | 42 ++++++++++++----- tests/api/damaged.rs | 9 ++-- tests/api/delete.rs | 6 ++- tests/api/gc.rs | 5 +- tests/api/old_archives.rs | 10 +++- tests/cli/versions.rs | 11 +++-- 21 files changed, 336 insertions(+), 233 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index bfbd8ecd..12899039 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -22,8 +22,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; use itertools::Itertools; -use serde::{Deserialize, Serialize}; use rayon::prelude::*; +use serde::{Deserialize, Serialize}; use crate::blockhash::BlockHash; use crate::errors::Error; @@ -34,7 +34,7 @@ use crate::monitor::{DefaultMonitor, DeleteMonitor, ReferencedBlocksMonitor}; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; -use crate::validate::{BlockMissingReason}; +use crate::validate::BlockMissingReason; use crate::*; const HEADER_FILENAME: &str = "CONSERVE"; @@ -62,10 +62,10 @@ pub struct DeleteOptions { #[derive(Debug)] pub enum ValidateArchiveProblem { - UnexpectedFiles{ path: String, files: Vec }, - UnexpectedFileType{ name: String, kind: Kind }, - UnexpectedDirectory{ path: String, directory: String }, - DuplicateBand{ path: String, directory: String }, + UnexpectedFiles { path: String, files: Vec }, + UnexpectedFileType { name: String, kind: Kind }, + UnexpectedDirectory { path: String, directory: String }, + DuplicateBand { path: String, directory: String }, DirectoryListError { error: std::io::Error }, } @@ -205,13 +205,17 @@ impl Archive { /// Returns all blocks referenced by all bands. /// /// Shows a progress bar as they're collected. - pub fn referenced_blocks(&self, band_ids: &[BandId], monitor: Option<&dyn ReferencedBlocksMonitor>) -> Result> { - let mut default_monitor = DefaultMonitor{}; + pub fn referenced_blocks( + &self, + band_ids: &[BandId], + monitor: Option<&dyn ReferencedBlocksMonitor>, + ) -> Result> { + let mut default_monitor = DefaultMonitor {}; let monitor = monitor.unwrap_or(&mut default_monitor); let archive = self.clone(); let current_count = AtomicUsize::new(0); - + let result = band_ids .par_iter() .inspect(|_| { @@ -228,7 +232,10 @@ impl Archive { } /// Returns an iterator of blocks that are present and referenced by no index. - pub fn unreferenced_blocks(&self, monitor: Option<&dyn ReferencedBlocksMonitor>) -> Result> { + pub fn unreferenced_blocks( + &self, + monitor: Option<&dyn ReferencedBlocksMonitor>, + ) -> Result> { let referenced = self.referenced_blocks(&self.list_band_ids()?, monitor)?; Ok(self .block_dir() @@ -246,7 +253,7 @@ impl Archive { options: &DeleteOptions, monitor: Option<&dyn DeleteMonitor>, ) -> Result { - let mut default_monitor = DefaultMonitor{}; + let mut default_monitor = DefaultMonitor {}; let monitor_ = monitor.unwrap_or(&mut default_monitor); let mut stats = DeleteStats::default(); @@ -263,10 +270,11 @@ impl Archive { let mut keep_band_ids = self.list_band_ids()?; keep_band_ids.retain(|b| !delete_band_ids.contains(b)); - let referenced = self.referenced_blocks(&keep_band_ids, Some(monitor_.referenced_blocks_monitor()))?; + let referenced = + self.referenced_blocks(&keep_band_ids, Some(monitor_.referenced_blocks_monitor()))?; let unref = { monitor_.find_present_blocks(0); - + let mut present_block_index = 0; let unref = self .block_dir() @@ -277,25 +285,25 @@ impl Archive { }) .filter(|bh| !referenced.contains(bh)) .collect_vec(); - + monitor_.find_present_blocks_finished(); unref }; - + let unref_count = unref.len(); stats.unreferenced_block_count = unref_count; let total_bytes = { - monitor_.measure_unreferenced_blocks(0, unref_count); - + monitor_.measure_unreferenced_blocks(0, unref_count); + let block_index = AtomicUsize::new(0); let total_bytes = unref .par_iter() .inspect(|_| { monitor_.measure_unreferenced_blocks( block_index.fetch_add(1, Ordering::Relaxed) + 1, - unref_count - ); + unref_count, + ); }) .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) .sum(); @@ -303,7 +311,7 @@ impl Archive { monitor_.measure_unreferenced_blocks_finished(); total_bytes }; - + stats.unreferenced_block_bytes = total_bytes; if !options.dry_run { @@ -331,12 +339,12 @@ impl Archive { .inspect(|_| { monitor_.delete_blocks( delete_block_count.fetch_add(1, Ordering::Relaxed) + 1, - unref.len() + unref.len(), ); }) .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) .count(); - + stats.deletion_errors += error_count; stats.deleted_block_count += unref_count - error_count; monitor_.delete_blocks_finished(); @@ -347,10 +355,14 @@ impl Archive { Ok(stats) } - pub fn validate(&self, options: &ValidateOptions, monitor: Option<&dyn ValidateMonitor>) -> Result { - let mut default_monitor = DefaultMonitor{}; + pub fn validate( + &self, + options: &ValidateOptions, + monitor: Option<&dyn ValidateMonitor>, + ) -> Result { + let mut default_monitor = DefaultMonitor {}; let monitor = monitor.unwrap_or(&mut default_monitor); - + let start = Instant::now(); monitor.validate_archive(); let mut stats = self.validate_archive_dir(monitor)?; @@ -360,7 +372,6 @@ impl Archive { let band_ids = self.list_band_ids()?; monitor.count_bands_result(&band_ids); - // 1. Walk all indexes, collecting a list of (block_hash, min_length) // values referenced by all the indexes. monitor.validate_bands(); @@ -385,12 +396,14 @@ impl Archive { } else { // 2. Check the hash of all blocks are correct, and remember how long // the uncompressed data is. - let block_lengths: HashMap = self.block_dir.validate(&mut stats, monitor)?; + let block_lengths: HashMap = + self.block_dir.validate(&mut stats, monitor)?; // 3b. Check that all referenced ranges are inside the present data. for (block_hash, referenced_len) in referenced_lens.0 { if let Some(actual_len) = block_lengths.get(&block_hash) { if referenced_len > (*actual_len as u64) { - monitor.validate_block_missing(&block_hash, &BlockMissingReason::InvalidRange); + monitor + .validate_block_missing(&block_hash, &BlockMissingReason::InvalidRange); // TODO: A separate counter; this is worse than just being missing stats.block_missing_count += 1; } @@ -422,12 +435,19 @@ impl Archive { Kind::Dir => dirs.push(name), Kind::File => files.push(name), other_kind => { - monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFileType { name: name, kind: other_kind }); + monitor.validate_archive_problem( + &ValidateArchiveProblem::UnexpectedFileType { + name: name, + kind: other_kind, + }, + ); stats.unexpected_files += 1; } }, Err(source) => { - monitor.validate_archive_problem(&ValidateArchiveProblem::DirectoryListError { error: source }); + monitor.validate_archive_problem(&ValidateArchiveProblem::DirectoryListError { + error: source, + }); stats.io_errors += 1; } } @@ -436,7 +456,10 @@ impl Archive { if !files.is_empty() { // TODO: Ignore .DS_Store stats.unexpected_files += 1; - monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFiles { path: format!("{:?}", self.transport), files }); + monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFiles { + path: format!("{:?}", self.transport), + files, + }); } remove_item(&mut dirs, &BLOCK_DIR); dirs.sort(); @@ -446,8 +469,8 @@ impl Archive { if bs.contains(&b) { stats.structure_problems += 1; monitor.validate_archive_problem(&ValidateArchiveProblem::DuplicateBand { - path: format!("{:?}", self.transport), - directory: d.clone() + path: format!("{:?}", self.transport), + directory: d.clone(), }); } else { bs.insert(b); @@ -455,8 +478,8 @@ impl Archive { } else { stats.structure_problems += 1; monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedDirectory { - path: format!("{:?}", self.transport), - directory: d.clone() + path: format!("{:?}", self.transport), + directory: d.clone(), }); } } diff --git a/src/backup.rs b/src/backup.rs index 4324898d..72e6a82e 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -57,7 +57,7 @@ pub fn backup( ) -> Result { let _span = tracing::span!(Level::DEBUG, "backup"); - let mut default_monitor = DefaultMonitor{}; + let mut default_monitor = DefaultMonitor {}; let monitor = monitor.unwrap_or(&mut default_monitor); let start = Instant::now(); @@ -68,13 +68,13 @@ pub fn backup( for entry_group in entry_iter.chunks(options.max_entries_per_hunk).into_iter() { for entry in entry_group { monitor.copy(&entry); - + match writer.copy_entry(&entry, source) { Err(e) => { for line in ui::format_error_causes(&e).lines() { debug!("{}", line); } - + monitor.copy_error(&entry, &e); stats.errors += 1; continue; diff --git a/src/band.rs b/src/band.rs index 60faa9c8..69f5019f 100644 --- a/src/band.rs +++ b/src/band.rs @@ -27,8 +27,8 @@ use serde::{Deserialize, Serialize}; use crate::jsonio::{read_json, write_json}; use crate::misc::remove_item; use crate::transport::{ListDirNames, Transport}; +use crate::validate::BandProblem; use crate::*; -use crate::validate::{BandProblem}; static INDEX_DIR: &str = "i"; @@ -217,12 +217,15 @@ impl Band { } pub fn validate(&self, stats: &mut ValidateStats, monitor: &dyn ValidateMonitor) -> Result<()> { - let ListDirNames { mut files, mut dirs } = - self.transport.list_dir_names("").map_err(Error::from)?; + let ListDirNames { + mut files, + mut dirs, + } = self.transport.list_dir_names("").map_err(Error::from)?; let band_head_filename = BAND_HEAD_FILENAME.to_string(); if !files.contains(&band_head_filename) { - monitor.validate_band_problem(self, &BandProblem::MissingHeadFile{ band_head_filename }); + monitor + .validate_band_problem(self, &BandProblem::MissingHeadFile { band_head_filename }); stats.missing_band_heads += 1; } remove_item(&mut files, &BAND_HEAD_FILENAME); @@ -230,12 +233,15 @@ impl Band { remove_item(&mut dirs, &INDEX_DIR); if !files.is_empty() { - monitor.validate_band_problem(self, &BandProblem::UnexpectedFiles{ files }); + monitor.validate_band_problem(self, &BandProblem::UnexpectedFiles { files }); stats.unexpected_files += 1; } if !dirs.is_empty() { - monitor.validate_band_problem(self, &BandProblem::UnexpectedDirectories { directories: dirs }); + monitor.validate_band_problem( + self, + &BandProblem::UnexpectedDirectories { directories: dirs }, + ); stats.unexpected_files += 1; } diff --git a/src/bin/conserve/log.rs b/src/bin/conserve/log.rs index 70f4d2c0..2f0b4395 100644 --- a/src/bin/conserve/log.rs +++ b/src/bin/conserve/log.rs @@ -1,27 +1,26 @@ use std::io::Write; +use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; -use std::ops::Deref; use lazy_static::lazy_static; -use tracing::Subscriber; use tracing::metadata::LevelFilter; +use tracing::Subscriber; use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::fmt; use tracing_subscriber::prelude::*; use tracing_subscriber::Registry; -use tracing_subscriber::fmt; -lazy_static!{ - pub static ref TERMINAL_OUTPUT: Mutex>>> = Mutex::new( - Some(Arc::new(Mutex::new(std::io::stdout()))) - ); +lazy_static! { + pub static ref TERMINAL_OUTPUT: Mutex>>> = + Mutex::new(Some(Arc::new(Mutex::new(std::io::stdout())))); } /// Wrapper around TERMINAL_OUTPUT which dynamically writes /// the tracing output to the output writer set at TERMINAL_OUTPUT. -struct TerminalWriter { } -impl TerminalWriter { } +struct TerminalWriter {} +impl TerminalWriter {} impl Write for TerminalWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { @@ -55,25 +54,25 @@ pub struct LoggingOptions { pub fn init(options: LoggingOptions) -> std::result::Result { let mut worker_guard = None; let registry = Registry::default(); - + // Terminal logger. // TODO: Enable timestamps except when in raw mode. // Right now this can't be achived since without_time updates the struct signature... let registry = registry.with( fmt::Layer::default() - .without_time() - .with_level(!options.terminal_raw) - .with_target(false) - .with_writer(|| TerminalWriter{}) - .with_filter(LevelFilter::from(options.level)) + .without_time() + .with_level(!options.terminal_raw) + .with_target(false) + .with_writer(|| TerminalWriter {}) + .with_filter(LevelFilter::from(options.level)), ); // File logger. let registry: Box = if let Some(path) = options.file { - let directory = path.parent() - .ok_or("can't resolve log file directory")?; + let directory = path.parent().ok_or("can't resolve log file directory")?; - let file_name = path.file_name() + let file_name = path + .file_name() .ok_or("can't get log file name")? .to_string_lossy() .to_string(); @@ -85,11 +84,11 @@ pub fn init(options: LoggingOptions) -> std::result::Result { Box::new( registry.with( fmt::Layer::default() - .with_ansi(false) - .with_target(false) - .with_writer(writer) - .with_filter(LevelFilter::from(options.level)) - ) + .with_ansi(false) + .with_target(false) + .with_writer(writer) + .with_filter(LevelFilter::from(options.level)), + ), ) } else { Box::new(registry) @@ -98,7 +97,9 @@ pub fn init(options: LoggingOptions) -> std::result::Result { tracing::subscriber::set_global_default(registry) .map_err(|_| "Failed to update global default logger".to_string())?; - Ok(LogGuard{ _worker_guard: worker_guard }) + Ok(LogGuard { + _worker_guard: worker_guard, + }) } /// Guards all logging activity. @@ -122,7 +123,7 @@ impl ViewLogGuard { } self.released = true; - + let mut output = TERMINAL_OUTPUT.lock().unwrap(); *output = self.previous_logger.take(); } @@ -140,5 +141,8 @@ pub fn update_terminal_target(target: Arc>) -> Vi let mut output = TERMINAL_OUTPUT.lock().unwrap(); let previous_logger = output.replace(target); - ViewLogGuard { previous_logger, released: false } -} \ No newline at end of file + ViewLogGuard { + previous_logger, + released: false, + } +} diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index a11e51b4..374666e5 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -19,12 +19,15 @@ use std::process::Termination; use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; -use log::{LoggingOptions, LogGuard}; -use show::{NutmegMonitor, BackupProgressModel, SizeProgressModel, DeleteProcessState, RestoreProgressModel, ReferencedBlocksProgressModel}; -use show::{show_diff, ShowVersionsOptions, show_versions}; -use tracing::{ trace, error, info, warn, Level }; - -use conserve::backup::{BackupOptions}; +use log::{LogGuard, LoggingOptions}; +use show::{show_diff, show_versions, ShowVersionsOptions}; +use show::{ + BackupProgressModel, DeleteProcessState, NutmegMonitor, ReferencedBlocksProgressModel, + RestoreProgressModel, SizeProgressModel, +}; +use tracing::{error, info, trace, warn, Level}; + +use conserve::backup::BackupOptions; use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; @@ -60,7 +63,7 @@ struct Args { /// Set the log level to trace #[clap(long, short = 'L', global = true)] log_level: Option, - + /// Path to the output log file #[clap(long, short = 'F', global = true)] log_file: Option, @@ -300,10 +303,10 @@ impl Command { let monitor = NutmegMonitor::new(model, !args.no_progress); let stats = backup( - &Archive::open(open_transport(archive)?)?, - source, - &options, - Some(&monitor) + &Archive::open(open_transport(archive)?)?, + source, + &options, + Some(&monitor), )?; drop(monitor); @@ -328,14 +331,18 @@ impl Command { } Command::Debug(Debug::Referenced { archive }) => { let archive = Archive::open(open_transport(archive)?)?; - let monitor = NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); + let monitor = + NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); for hash in archive.referenced_blocks(&archive.list_band_ids()?, Some(&monitor))? { info!("{}", hash); } } Command::Debug(Debug::Unreferenced { archive }) => { - let monitor = NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); - for hash in Archive::open(open_transport(archive)?)?.unreferenced_blocks(Some(&monitor))? { + let monitor = + NutmegMonitor::new(ReferencedBlocksProgressModel::default(), !args.no_progress); + for hash in + Archive::open(open_transport(archive)?)?.unreferenced_blocks(Some(&monitor))? + { info!("{}", hash); } } @@ -385,7 +392,8 @@ impl Command { break_lock, no_stats, } => { - let mut monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); + let mut monitor = + NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( @@ -446,7 +454,8 @@ impl Command { overwrite: *force_overwrite, }; - let mut monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose), !args.no_progress); + let mut monitor = + NutmegMonitor::new(RestoreProgressModel::new(*verbose), !args.no_progress); let stats = restore(&archive, destination, &options, Some(&mut monitor))?; if !no_stats { info!("Restore complete."); @@ -462,8 +471,9 @@ impl Command { exclude_from, } => { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; - - let mut monitor = NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); + + let mut monitor = + NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? @@ -474,7 +484,7 @@ impl Command { .file_bytes }; drop(monitor); - + if *bytes { info!("{}", size); } else { @@ -490,10 +500,12 @@ impl Command { skip_block_hashes: *quick, }; - let mut monitor = NutmegMonitor::new(ValidateProgressModel::default(), !args.no_progress); - let stats = Archive::open(open_transport(archive)?)?.validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; + let mut monitor = + NutmegMonitor::new(ValidateProgressModel::default(), !args.no_progress); + let stats = Archive::open(open_transport(archive)?)? + .validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; drop(monitor); - + if !no_stats { for line in format!("{}", stats).lines() { info!("{}", line); @@ -543,7 +555,8 @@ fn band_selection_policy_from_opt(backup: &Option) -> BandSelectionPolic } fn initialize_log(args: &Args) -> std::result::Result { - let file = args.log_file + let file = args + .log_file .as_ref() .map(|file| PathBuf::from_str(&file)) .transpose() @@ -557,10 +570,10 @@ fn initialize_log(args: &Args) -> std::result::Result { } }); - let guard = log::init(LoggingOptions{ + let guard = log::init(LoggingOptions { file, level, - terminal_raw: args.log_raw + terminal_raw: args.log_raw, })?; if args.log_level == Some(tracing::Level::TRACE) { @@ -585,13 +598,13 @@ fn main() -> ExitCode { let exit_code = match result { Err(ref e) => { error!("{}", e.to_string()); - + let mut cause: &dyn Error = e; while let Some(c) = cause.source() { error!(" caused by: {}", c); cause = c; } - + // // TODO: Perhaps always log the traceback to a log file. // // NOTE(WolverinDEV): May always log this as trace level? // if let Some(bt) = e.backtrace() { diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index 57950045..af256ade 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -18,19 +18,19 @@ use std::borrow::Cow; use std::sync::{Arc, Mutex}; -use std::time::{Instant}; +use std::time::Instant; use conserve::archive::ValidateArchiveProblem; use conserve::stats::Sizes; use conserve::ui::duration_to_hms; use conserve::{ - bytes_to_human_mb, Archive, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, - DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, ReadTree, Result, TreeSizeMonitor, - ValidateMonitor, BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor, RestoreMonitor + bytes_to_human_mb, Archive, BackupMonitor, Band, BandProblem, BandSelectionPolicy, + BlockMissingReason, DeleteMonitor, DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, + ReadTree, ReferencedBlocksMonitor, RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, }; -use nutmeg::{View, Model}; +use nutmeg::{Model, View}; use thousands::Separable; -use tracing::{info, warn, error}; +use tracing::{error, info, warn}; use crate::log::{self, ViewLogGuard}; @@ -157,8 +157,8 @@ enum NutmegMonitorState { _log_guard: ViewLogGuard, }, Simple { - state: Mutex - } + state: Mutex, + }, } pub struct NutmegMonitor { @@ -172,13 +172,18 @@ impl NutmegMonitor { initial_state, nutmeg::Options::default().progress_enabled(progress_enabled), ))); - + let log_guard = log::update_terminal_target(view.clone()); - NutmegMonitorState::View { view, _log_guard: log_guard } + NutmegMonitorState::View { + view, + _log_guard: log_guard, + } } else { - NutmegMonitorState::Simple { state: Mutex::new(initial_state) } + NutmegMonitorState::Simple { + state: Mutex::new(initial_state), + } }; - + Self { state } } @@ -187,7 +192,7 @@ impl NutmegMonitor { NutmegMonitorState::View { view, .. } => { let view = view.lock().expect("lock() should not fail"); view.update(update_fn) - }, + } NutmegMonitorState::Simple { state } => { let mut state = state.lock().expect("lock() should not fail"); update_fn(&mut *state) @@ -218,7 +223,7 @@ pub struct BackupProgressModel { scanned_file_bytes: u64, scanned_dirs: usize, scanned_files: usize, - + entries_new: usize, entries_changed: usize, entries_unchanged: usize, @@ -363,7 +368,7 @@ impl ValidateMonitor for NutmegMonitor { "Unexpected file kind in archive directory: {:?} of kind {:?}", name, kind ); - }, + } ValidateArchiveProblem::DirectoryListError { error } => { error!("Error listing archive directory: {:?}", error); } @@ -445,10 +450,7 @@ impl ValidateMonitor for NutmegMonitor { } }); - info!( - "Finished validating bands in {:#?}.", - elapsed - ); + info!("Finished validating bands in {:#?}.", elapsed); } fn list_block_names(&self, current_count: usize) { @@ -538,24 +540,11 @@ impl TreeSizeMonitor for NutmegMonitor { } pub enum DeleteProcessState { - ListReferencedBlocks { - count: usize, - }, - FindPresentBlocks { - count: usize, - }, - MeasureUnreferencedBlocks { - count: usize, - target: usize, - }, - DeleteBands { - count: usize, - target: usize, - }, - DeleteBlocks { - count: usize, - target: usize, - } + ListReferencedBlocks { count: usize }, + FindPresentBlocks { count: usize }, + MeasureUnreferencedBlocks { count: usize, target: usize }, + DeleteBands { count: usize, target: usize }, + DeleteBlocks { count: usize, target: usize }, } impl Default for DeleteProcessState { @@ -569,16 +558,16 @@ impl Model for DeleteProcessState { match self { DeleteProcessState::ListReferencedBlocks { count } => { format!("Find referenced blocks in band ({} discovered)", count) - }, + } DeleteProcessState::FindPresentBlocks { count } => { format!("Find present blocks ({} discovered)", count) - }, + } DeleteProcessState::MeasureUnreferencedBlocks { count, target } => { format!("Measure unreferenced blocks ({}/{})", count, target) - }, + } DeleteProcessState::DeleteBands { count, target } => { format!("Delete bands ({}/{})", count, target) - }, + } DeleteProcessState::DeleteBlocks { count, target } => { format!("Delete blocks ({}/{})", count, target) } @@ -593,25 +582,36 @@ impl DeleteMonitor for NutmegMonitor { fn find_present_blocks(&self, current_count: usize) { self.update_model(|view| { - *view = DeleteProcessState::FindPresentBlocks { count: current_count }; + *view = DeleteProcessState::FindPresentBlocks { + count: current_count, + }; }); } fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { self.update_model(|view| { - *view = DeleteProcessState::MeasureUnreferencedBlocks { count: current_count, target: target_count }; + *view = DeleteProcessState::MeasureUnreferencedBlocks { + count: current_count, + target: target_count, + }; }); } fn delete_bands(&self, current_count: usize, target_count: usize) { self.update_model(|view| { - *view = DeleteProcessState::DeleteBands { count: current_count, target: target_count }; + *view = DeleteProcessState::DeleteBands { + count: current_count, + target: target_count, + }; }); } fn delete_blocks(&self, current_count: usize, target_count: usize) { self.update_model(|view| { - *view = DeleteProcessState::DeleteBlocks { count: current_count, target: target_count }; + *view = DeleteProcessState::DeleteBlocks { + count: current_count, + target: target_count, + }; }); } } @@ -619,7 +619,9 @@ impl DeleteMonitor for NutmegMonitor { impl ReferencedBlocksMonitor for NutmegMonitor { fn list_referenced_blocks(&self, current_count: usize) { self.update_model(|view| { - *view = DeleteProcessState::ListReferencedBlocks { count: current_count }; + *view = DeleteProcessState::ListReferencedBlocks { + count: current_count, + }; }); } } @@ -635,7 +637,7 @@ impl RestoreProgressModel { Self { print_filenames, filename: "".to_string(), - bytes_done: 0 + bytes_done: 0, } } } @@ -684,4 +686,4 @@ impl ReferencedBlocksMonitor for NutmegMonitor { fn list_referenced_blocks(&self, current_count: usize) { self.update_model(|model| model.count = current_count); } -} \ No newline at end of file +} diff --git a/src/blockdir.rs b/src/blockdir.rs index 48529b16..5172089d 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -21,17 +21,17 @@ //! //! The structure is: archive > blockdir > subdir > file. +use rayon::prelude::*; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::io; use std::path::Path; use std::sync::Arc; -use rayon::prelude::*; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; use serde::{Deserialize, Serialize}; -use tracing::{ warn, error }; +use tracing::{error, warn}; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; @@ -120,10 +120,7 @@ impl BlockDir { .or_else(|io_err| { if io_err.kind() == io::ErrorKind::AlreadyExists { // Perhaps it was simultaneously created by another thread or process. - error!( - "Unexpected late detection of existing block {:?}", - hex_hash - ); + error!("Unexpected late detection of existing block {:?}", hex_hash); Ok(()) } else { Err(Error::WriteBlock { @@ -203,10 +200,7 @@ impl BlockDir { if dirname.len() == SUBDIR_NAME_CHARS { true } else { - error!( - "Unexpected subdirectory in blockdir: {:?}", - dirname - ); + error!("Unexpected subdirectory in blockdir: {:?}", dirname); false } }); @@ -268,15 +262,19 @@ impl BlockDir { /// /// Return a dict describing which blocks are present, and the length of their uncompressed /// data. - pub fn validate(&self, stats: &mut ValidateStats, monitor: &dyn ValidateMonitor) -> Result> { + pub fn validate( + &self, + stats: &mut ValidateStats, + monitor: &dyn ValidateMonitor, + ) -> Result> { // TODO: In the top-level directory, no files or directories other than prefix // directories of the right length. // TODO: Test having a block with the right compression but the wrong contents. let blocks = self.block_names_set(monitor)?; monitor.read_blocks(blocks.len()); - + stats.block_read_count = blocks.len().try_into().unwrap(); - + // Make a vec of Some(usize) if the block could be read, or None if it // failed, where the usize gives the uncompressed data size. let results: Vec> = blocks @@ -288,9 +286,7 @@ impl BlockDir { // we call the monitor. monitor.read_block_result(&hash, &result); - result - .map(|(bytes, _sizes)| (hash, bytes.len())) - .ok() + result.map(|(bytes, _sizes)| (hash, bytes.len())).ok() }) .collect(); @@ -327,7 +323,10 @@ impl BlockDir { decompressed_bytes, )); if actual_hash != *hash { - warn!("Block file {:?} has actual decompressed hash {}", &block_relpath, actual_hash); + warn!( + "Block file {:?} has actual decompressed hash {}", + &block_relpath, actual_hash + ); return Err(Error::BlockCorrupt { hash: hash.to_string(), actual_hash: actual_hash.to_string(), diff --git a/src/gc_lock.rs b/src/gc_lock.rs index 800244a2..f161b742 100644 --- a/src/gc_lock.rs +++ b/src/gc_lock.rs @@ -126,7 +126,13 @@ mod test { fn completed_backup_ok() { let archive = ScratchArchive::new(); let source = TreeFixture::new(); - backup(&archive, &source.live_tree(), &BackupOptions::default(), None).unwrap(); + backup( + &archive, + &source.live_tree(), + &BackupOptions::default(), + None, + ) + .unwrap(); let delete_guard = GarbageCollectionLock::new(&archive).unwrap(); delete_guard.check().unwrap(); } @@ -136,7 +142,12 @@ mod test { let archive = ScratchArchive::new(); let source = TreeFixture::new(); let _delete_guard = GarbageCollectionLock::new(&archive).unwrap(); - let backup_result = backup(&archive, &source.live_tree(), &BackupOptions::default(), None); + let backup_result = backup( + &archive, + &source.live_tree(), + &BackupOptions::default(), + None, + ); assert_eq!( backup_result.expect_err("backup fails").to_string(), "Archive is locked for garbage collection" diff --git a/src/index.rs b/src/index.rs index 1620db50..247463dd 100644 --- a/src/index.rs +++ b/src/index.rs @@ -321,10 +321,7 @@ impl Iterator for IndexHunkIter { Ok(Some(entries)) => entries, Err(err) => { self.stats.errors += 1; - error!( - "Error reading index hunk {:?}: {:?} ", - hunk_number, err - ); + error!("Error reading index hunk {:?}: {:?} ", hunk_number, err); continue; } }; diff --git a/src/lib.rs b/src/lib.rs index 92375e1e..e34789fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub mod kind; pub mod live_tree; mod merge; pub(crate) mod misc; +pub(crate) mod monitor; pub mod restore; pub mod stats; mod stitch; @@ -44,7 +45,6 @@ mod tree; pub mod ui; pub mod unix_time; mod validate; -pub(crate) mod monitor; pub use crate::apath::Apath; pub use crate::archive::Archive; @@ -65,13 +65,16 @@ pub use crate::kind::Kind; pub use crate::live_tree::{LiveEntry, LiveTree}; pub use crate::merge::{MergeTrees, MergedEntryKind}; pub use crate::misc::bytes_to_human_mb; +pub use crate::monitor::{ + BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor, RestoreMonitor, TreeSizeMonitor, + ValidateMonitor, +}; pub use crate::restore::{restore, RestoreOptions, RestoreTree}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; -pub use crate::validate::{ ValidateOptions, BandProblem, BandValidateResult, BlockMissingReason }; -pub use crate::monitor::{ BackupMonitor, ValidateMonitor, TreeSizeMonitor, ReferencedBlocksMonitor, DeleteMonitor, RestoreMonitor }; +pub use crate::validate::{BandProblem, BandValidateResult, BlockMissingReason, ValidateOptions}; pub type Result = std::result::Result; diff --git a/src/live_tree.rs b/src/live_tree.rs index b3e2eeb3..3cfa0632 100644 --- a/src/live_tree.rs +++ b/src/live_tree.rs @@ -248,10 +248,7 @@ impl Iter { Ok(true) => continue, Ok(false) => (), Err(e) => { - error!( - "Error checking CACHEDIR.TAG in {:?}: {}", - dir_entry, e - ); + error!("Error checking CACHEDIR.TAG in {:?}: {}", dir_entry, e); } } } @@ -286,10 +283,7 @@ impl Iter { let t = match dir_path.join(dir_entry.file_name()).read_link() { Ok(t) => t, Err(e) => { - error!( - "Failed to read target of symlink {:?}: {}", - child_apath, e - ); + error!("Failed to read target of symlink {:?}: {}", child_apath, e); continue; } }; diff --git a/src/monitor.rs b/src/monitor.rs index 9763b7f9..8284086e 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -1,4 +1,8 @@ -use crate::{ReadTree, LiveEntry, Error, DiffKind, BackupStats, BandId, BandProblem, BandValidateResult, BlockMissingReason, BlockHash, Band, stats::Sizes, Result, archive::ValidateArchiveProblem, IndexEntry}; +use crate::{ + archive::ValidateArchiveProblem, stats::Sizes, BackupStats, Band, BandId, BandProblem, + BandValidateResult, BlockHash, BlockMissingReason, DiffKind, Error, IndexEntry, LiveEntry, + ReadTree, Result, +}; /// Monitor the backup progress. pub trait BackupMonitor { @@ -10,7 +14,7 @@ pub trait BackupMonitor { } /// Monitor the validation progress. -pub trait ValidateMonitor : Sync { +pub trait ValidateMonitor: Sync { fn count_bands(&self) {} fn count_bands_result(&self, _bands: &[BandId]) {} @@ -31,7 +35,7 @@ pub trait ValidateMonitor : Sync { fn list_block_names(&self, _current_count: usize) {} fn list_block_names_finished(&self) {} - + fn read_blocks(&self, _count: usize) {} fn read_block_result(&self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} fn read_blocks_finished(&self) {} @@ -43,13 +47,13 @@ pub trait TreeSizeMonitor { } /// Monitor for iterating referenced blocks. -pub trait ReferencedBlocksMonitor : Sync { +pub trait ReferencedBlocksMonitor: Sync { fn list_referenced_blocks(&self, _current_count: usize) {} fn list_referenced_blocks_finished(&self) {} } /// Monitor for deleting backups/blocks. -pub trait DeleteMonitor : Sync { +pub trait DeleteMonitor: Sync { fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor; fn find_present_blocks(&self, _current_count: usize) {} @@ -84,4 +88,4 @@ impl DeleteMonitor for DefaultMonitor { self } } -impl RestoreMonitor for DefaultMonitor {} \ No newline at end of file +impl RestoreMonitor for DefaultMonitor {} diff --git a/src/restore.rs b/src/restore.rs index c7c279e8..b8e79e32 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -18,16 +18,16 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; -use filetime::{set_file_handle_times}; +use filetime::set_file_handle_times; #[cfg(unix)] -use filetime::{set_symlink_file_times}; +use filetime::set_symlink_file_times; use tracing::{debug, warn}; use crate::band::BandSelectionPolicy; use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; -use crate::monitor::{RestoreMonitor, DefaultMonitor}; +use crate::monitor::{DefaultMonitor, RestoreMonitor}; use crate::stats::RestoreStats; use crate::unix_time::UnixTime; use crate::*; @@ -61,7 +61,7 @@ pub fn restore( options: &RestoreOptions, monitor: Option<&dyn RestoreMonitor>, ) -> Result { - let mut default_monitor = DefaultMonitor{}; + let mut default_monitor = DefaultMonitor {}; let monitor = monitor.unwrap_or(&mut default_monitor); let st = archive.open_stored_tree(options.band_selection.clone())?; @@ -237,10 +237,7 @@ impl RestoreTree { fn copy_symlink(&mut self, entry: &E) -> Result<()> { // TODO: Add a test with a canned index containing a symlink, and expect // it cannot be restored on Windows and can be on Unix. - warn!( - "Can't restore symlinks on non-Unix: {}", - entry.apath() - ); + warn!("Can't restore symlinks on non-Unix: {}", entry.apath()); Ok(()) } } diff --git a/src/tree.rs b/src/tree.rs index 89ccd50b..6fa77c15 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -43,8 +43,15 @@ pub trait ReadTree { /// Measure the tree size. /// /// This typically requires walking all entries, which may take a while. - fn size(&self, exclude: Exclude, monitor: Option<&dyn TreeSizeMonitor>) -> Result where Self: Sized { - let mut default_monitor = DefaultMonitor{}; + fn size( + &self, + exclude: Exclude, + monitor: Option<&dyn TreeSizeMonitor>, + ) -> Result + where + Self: Sized, + { + let mut default_monitor = DefaultMonitor {}; let monitor = monitor.unwrap_or(&mut default_monitor as &dyn TreeSizeMonitor); let mut tot = 0u64; diff --git a/src/validate.rs b/src/validate.rs index d4af65f0..c2c3a1b2 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -27,10 +27,10 @@ pub struct ValidateOptions { /// Band validation result. pub enum BandValidateResult { MetadataError(Error), - + OpenError(Error), TreeOpenError(Error), - + TreeValidateError(Error), Valid(BlockLengths, ValidateStats), @@ -44,9 +44,9 @@ pub enum BlockMissingReason { } pub enum BandProblem { - MissingHeadFile{ band_head_filename: String }, - UnexpectedFiles{ files: Vec }, - UnexpectedDirectories{ directories: Vec } + MissingHeadFile { band_head_filename: String }, + UnexpectedFiles { files: Vec }, + UnexpectedDirectories { directories: Vec }, } impl BlockLengths { @@ -80,12 +80,12 @@ pub(crate) fn validate_bands( ) -> (BlockLengths, ValidateStats) { let mut stats = ValidateStats::default(); let mut block_lens = BlockLengths::new(); - + for band_id in band_ids { monitor.validate_band(band_id); let result = validate_band(archive, &mut stats, band_id, monitor); monitor.validate_band_result(band_id, &result); - + match result { BandValidateResult::MetadataError(_) => stats.band_metadata_problems += 1, BandValidateResult::OpenError(_) => stats.band_open_errors += 1, @@ -101,22 +101,28 @@ pub(crate) fn validate_bands( (block_lens, stats) } -pub(crate) fn validate_band(archive: &Archive, stats: &mut ValidateStats, band_id: &BandId, monitor: &dyn ValidateMonitor) -> BandValidateResult { +pub(crate) fn validate_band( + archive: &Archive, + stats: &mut ValidateStats, + band_id: &BandId, + monitor: &dyn ValidateMonitor, +) -> BandValidateResult { let band = match Band::open(archive, band_id) { Ok(band) => band, - Err(error) => return BandValidateResult::OpenError(error) + Err(error) => return BandValidateResult::OpenError(error), }; if let Err(error) = band.validate(stats, monitor) { return BandValidateResult::MetadataError(error); } - let stored_tree = match archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { - Ok(tree) => tree, - Err(error) => return BandValidateResult::TreeOpenError(error), - }; + let stored_tree = + match archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { + Ok(tree) => tree, + Err(error) => return BandValidateResult::TreeOpenError(error), + }; - match validate_stored_tree(&stored_tree) { + match validate_stored_tree(&stored_tree) { Ok(result) => BandValidateResult::Valid(result.0, result.1), Err(error) => BandValidateResult::TreeValidateError(error), } diff --git a/tests/api/backup.rs b/tests/api/backup.rs index c7e872b5..1e9fb2d1 100644 --- a/tests/api/backup.rs +++ b/tests/api/backup.rs @@ -31,7 +31,8 @@ pub fn simple_backup() { let srcdir = TreeFixture::new(); srcdir.create_file("hello"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); + let copy_stats = + backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(copy_stats.index_builder_stats.index_hunks, 1); assert_eq!(copy_stats.files, 1); assert_eq!(copy_stats.deduplicated_blocks, 0); @@ -46,8 +47,13 @@ pub fn simple_backup() { assert!(archive.band_exists(&BandId::zero()).unwrap()); assert!(archive.band_is_closed(&BandId::zero()).unwrap()); assert!(!archive.band_exists(&BandId::new(&[1])).unwrap()); - let copy_stats = - restore(&archive, restore_dir.path(), &RestoreOptions::default(), None).expect("restore"); + let copy_stats = restore( + &archive, + restore_dir.path(), + &RestoreOptions::default(), + None, + ) + .expect("restore"); assert_eq!(copy_stats.uncompressed_file_bytes, 8); } @@ -88,8 +94,13 @@ pub fn simple_backup_with_excludes() -> Result<()> { assert!(band_info.is_closed); assert!(band_info.end_time.is_some()); - let copy_stats = - restore(&archive, restore_dir.path(), &RestoreOptions::default(), None).expect("restore"); + let copy_stats = restore( + &archive, + restore_dir.path(), + &RestoreOptions::default(), + None, + ) + .expect("restore"); assert_eq!(copy_stats.uncompressed_file_bytes, 8); // TODO: Read back contents of that file. @@ -185,7 +196,8 @@ fn large_file() { let large_content = vec![b'a'; 4 << 20]; tf.create_file_with_contents("large", &large_content); - let backup_stats = backup(&af, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); + let backup_stats = + backup(&af, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(backup_stats.new_files, 1); // First 1MB should be new; remainder should be deduplicated. assert_eq!(backup_stats.uncompressed_bytes, 1 << 20); @@ -198,8 +210,13 @@ fn large_file() { // Try to restore it let rd = TempDir::new().unwrap(); let restore_archive = Archive::open_path(af.path()).unwrap(); - let restore_stats = - restore(&restore_archive, rd.path(), &RestoreOptions::default(), None).expect("restore"); + let restore_stats = restore( + &restore_archive, + rd.path(), + &RestoreOptions::default(), + None, + ) + .expect("restore"); assert_eq!(restore_stats.files, 1); let content = std::fs::read(rd.path().join("large")).unwrap(); @@ -259,7 +276,8 @@ pub fn symlink() { let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); srcdir.create_symlink("symlink", "/a/broken/destination"); - let copy_stats = backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); + let copy_stats = + backup(&af, &srcdir.live_tree(), &BackupOptions::default(), None).expect("backup"); assert_eq!(0, copy_stats.files); assert_eq!(1, copy_stats.symlinks); @@ -514,7 +532,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, - None + None, ) .unwrap(); assert_eq!(stats.new_files, 2); @@ -530,7 +548,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, - None + None, ) .unwrap(); assert_eq!(stats.unmodified_files, 1); @@ -552,7 +570,7 @@ fn detect_unchanged_from_stitched_index() { max_entries_per_hunk: 1, ..Default::default() }, - None + None, ) .unwrap(); assert_eq!(stats.unmodified_files, 2, "both files are unmodified"); diff --git a/tests/api/damaged.rs b/tests/api/damaged.rs index cd02098c..aa188ed5 100644 --- a/tests/api/damaged.rs +++ b/tests/api/damaged.rs @@ -31,9 +31,12 @@ fn missing_block() -> Result<()> { fn missing_block_skip_block_hashes() -> Result<()> { let archive = Archive::open_path(Path::new("testdata/damaged/missing-block"))?; - let validate_stats = archive.validate(&ValidateOptions { - skip_block_hashes: true, - }, None)?; + let validate_stats = archive.validate( + &ValidateOptions { + skip_block_hashes: true, + }, + None, + )?; assert!(validate_stats.has_problems()); assert_eq!(validate_stats.block_missing_count, 1); Ok(()) diff --git a/tests/api/delete.rs b/tests/api/delete.rs index f4a47134..38465539 100644 --- a/tests/api/delete.rs +++ b/tests/api/delete.rs @@ -21,7 +21,11 @@ fn delete_all_bands() { af.store_two_versions(); let stats = af - .delete_bands(&[BandId::new(&[0]), BandId::new(&[1])], &Default::default(), None) + .delete_bands( + &[BandId::new(&[0]), BandId::new(&[1])], + &Default::default(), + None, + ) .expect("delete_bands"); assert_eq!(stats.deleted_block_count, 2); diff --git a/tests/api/gc.rs b/tests/api/gc.rs index 73fa8e31..d2cf28b8 100644 --- a/tests/api/gc.rs +++ b/tests/api/gc.rs @@ -26,7 +26,8 @@ fn unreferenced_blocks() { .parse() .unwrap(); - let _copy_stats = backup(&archive, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); + let _copy_stats = + backup(&archive, &tf.live_tree(), &BackupOptions::default(), None).expect("backup"); // Delete the band and index std::fs::remove_dir_all(archive.path().join("b0000")).unwrap(); @@ -113,7 +114,7 @@ fn backup_prevented_by_gc_lock() -> Result<()> { break_lock: true, ..Default::default() }, - None + None, )?; // Backup should now succeed. diff --git a/tests/api/old_archives.rs b/tests/api/old_archives.rs index 9e385f7f..85fafaf9 100644 --- a/tests/api/old_archives.rs +++ b/tests/api/old_archives.rs @@ -136,7 +136,13 @@ fn restore_modify_backup() { let archive = open_old_archive(ver, "minimal"); - restore(&archive, working_tree.path(), &RestoreOptions::default(), None).expect("restore"); + restore( + &archive, + working_tree.path(), + &RestoreOptions::default(), + None, + ) + .expect("restore"); // Write back into a new copy of the archive, without modifying the // testdata in the source tree. @@ -162,7 +168,7 @@ fn restore_modify_backup() { &new_archive, &LiveTree::open(working_tree.path()).unwrap(), &Default::default(), - None + None, ) .expect("Backup modified tree"); diff --git a/tests/cli/versions.rs b/tests/cli/versions.rs index 5feee355..05a2fe2b 100644 --- a/tests/cli/versions.rs +++ b/tests/cli/versions.rs @@ -43,7 +43,7 @@ fn newest_first() { "--newest", "--utc", "testdata/archive/simple/v0.6.10", - "-R" + "-R", ]) .assert() .success() @@ -71,7 +71,12 @@ fn local_time() { #[test] fn short() { run_conserve() - .args(&["versions", "--short", "testdata/archive/simple/v0.6.10", "-R"]) + .args(&[ + "versions", + "--short", + "testdata/archive/simple/v0.6.10", + "-R", + ]) .assert() .success() .stdout( @@ -91,7 +96,7 @@ fn tree_sizes() { "--sizes", "--utc", "testdata/archive/simple/v0.6.10", - "-R" + "-R", ]) .assert() .success() From fe579e519a787815b323d4607eeef98731591278 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 12 Aug 2022 17:16:43 +0200 Subject: [PATCH 29/39] Fixed the restore test for linux --- tests/api/restore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/restore.rs b/tests/api/restore.rs index 737530b1..3a208c6a 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -133,7 +133,7 @@ fn restore_symlink() { backup(&af, &srcdir.live_tree(), &Default::default(), None).unwrap(); let restore_dir = TempDir::new().unwrap(); - restore(&af, restore_dir.path(), &Default::default()).unwrap(); + restore(&af, restore_dir.path(), &Default::default(), None).unwrap(); let restored_symlink_path = restore_dir.path().join("symlink"); let sym_meta = symlink_metadata(&restored_symlink_path).unwrap(); From e1574310df2aae9cde39f7e355152c8ff620414b Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 12 Aug 2022 17:22:31 +0200 Subject: [PATCH 30/39] Fixed clippy errors --- src/archive.rs | 16 ++++++++-------- src/backup.rs | 4 ++-- src/band.rs | 4 ++-- src/bin/conserve/main.rs | 22 ++++++++++------------ src/bin/conserve/show.rs | 8 ++++---- src/restore.rs | 4 ++-- src/stitch.rs | 2 +- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 12899039..eda3cd2a 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -210,8 +210,8 @@ impl Archive { band_ids: &[BandId], monitor: Option<&dyn ReferencedBlocksMonitor>, ) -> Result> { - let mut default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&mut default_monitor); + let default_monitor = DefaultMonitor {}; + let monitor = monitor.unwrap_or(&default_monitor); let archive = self.clone(); let current_count = AtomicUsize::new(0); @@ -253,8 +253,8 @@ impl Archive { options: &DeleteOptions, monitor: Option<&dyn DeleteMonitor>, ) -> Result { - let mut default_monitor = DefaultMonitor {}; - let monitor_ = monitor.unwrap_or(&mut default_monitor); + let default_monitor = DefaultMonitor {}; + let monitor_ = monitor.unwrap_or(&default_monitor); let mut stats = DeleteStats::default(); let start = Instant::now(); @@ -360,8 +360,8 @@ impl Archive { options: &ValidateOptions, monitor: Option<&dyn ValidateMonitor>, ) -> Result { - let mut default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&mut default_monitor); + let default_monitor = DefaultMonitor {}; + let monitor = monitor.unwrap_or(&default_monitor); let start = Instant::now(); monitor.validate_archive(); @@ -390,7 +390,7 @@ impl Archive { .keys() .filter(|&bh| !present_blocks.contains(bh)) { - monitor.validate_block_missing(&block_hash, &BlockMissingReason::NotExisting); + monitor.validate_block_missing(block_hash, &BlockMissingReason::NotExisting); stats.block_missing_count += 1; } } else { @@ -437,7 +437,7 @@ impl Archive { other_kind => { monitor.validate_archive_problem( &ValidateArchiveProblem::UnexpectedFileType { - name: name, + name, kind: other_kind, }, ); diff --git a/src/backup.rs b/src/backup.rs index 72e6a82e..477381ee 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -57,8 +57,8 @@ pub fn backup( ) -> Result { let _span = tracing::span!(Level::DEBUG, "backup"); - let mut default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&mut default_monitor); + let default_monitor = DefaultMonitor {}; + let monitor = monitor.unwrap_or(&default_monitor); let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; diff --git a/src/band.rs b/src/band.rs index 69f5019f..7518cb1f 100644 --- a/src/band.rs +++ b/src/band.rs @@ -248,8 +248,8 @@ impl Band { Ok(()) } - pub fn transport(&self) -> &Box { - &self.transport + pub fn transport(&self) -> &dyn Transport { + self.transport.as_ref() } } diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 374666e5..ed4e1ef5 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -392,8 +392,7 @@ impl Command { break_lock, no_stats, } => { - let mut monitor = - NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); + let monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( @@ -402,7 +401,7 @@ impl Command { dry_run: *dry_run, break_lock: *break_lock, }, - Some(&mut monitor), + Some(&monitor), )?; if !no_stats { for line in format!("{}", stats).lines() { @@ -454,9 +453,9 @@ impl Command { overwrite: *force_overwrite, }; - let mut monitor = + let monitor = NutmegMonitor::new(RestoreProgressModel::new(*verbose), !args.no_progress); - let stats = restore(&archive, destination, &options, Some(&mut monitor))?; + let stats = restore(&archive, destination, &options, Some(&monitor))?; if !no_stats { info!("Restore complete."); for line in format!("{}", stats).lines() { @@ -472,15 +471,14 @@ impl Command { } => { let excludes = ExcludeBuilder::from_args(exclude, exclude_from)?.build()?; - let mut monitor = - NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); + let monitor = NutmegMonitor::new(SizeProgressModel::default(), !args.no_progress); let size = if let Some(archive) = &stos.archive { stored_tree_from_opt(archive, &stos.backup)? - .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? + .size(excludes, Some(&monitor as &dyn TreeSizeMonitor<_>))? .file_bytes } else { LiveTree::open(stos.source.as_ref().unwrap())? - .size(excludes, Some(&mut monitor as &dyn TreeSizeMonitor<_>))? + .size(excludes, Some(&monitor as &dyn TreeSizeMonitor<_>))? .file_bytes }; drop(monitor); @@ -500,10 +498,10 @@ impl Command { skip_block_hashes: *quick, }; - let mut monitor = + let monitor = NutmegMonitor::new(ValidateProgressModel::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)? - .validate(&options, Some(&mut monitor as &dyn ValidateMonitor))?; + .validate(&options, Some(&monitor as &dyn ValidateMonitor))?; drop(monitor); if !no_stats { @@ -558,7 +556,7 @@ fn initialize_log(args: &Args) -> std::result::Result { let file = args .log_file .as_ref() - .map(|file| PathBuf::from_str(&file)) + .map(|file| PathBuf::from_str(file)) .transpose() .map_err(|_| "Unparseable log file path".to_string())?; diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index af256ade..c06fec9b 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -259,10 +259,10 @@ impl BackupMonitor for NutmegMonitor { if let Some(diff_kind) = result.as_ref() { let verbose = self.update_model(|model| { match diff_kind { - &DiffKind::Changed => model.entries_changed += 1, - &DiffKind::New => model.entries_new += 1, - &DiffKind::Unchanged => model.entries_unchanged += 1, - &DiffKind::Deleted => model.entries_deleted += 1, + DiffKind::Changed => model.entries_changed += 1, + DiffKind::New => model.entries_new += 1, + DiffKind::Unchanged => model.entries_unchanged += 1, + DiffKind::Deleted => model.entries_deleted += 1, }; model.verbose diff --git a/src/restore.rs b/src/restore.rs index b8e79e32..19ae8afa 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -61,8 +61,8 @@ pub fn restore( options: &RestoreOptions, monitor: Option<&dyn RestoreMonitor>, ) -> Result { - let mut default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&mut default_monitor); + let default_monitor = DefaultMonitor {}; + let monitor = monitor.unwrap_or(&default_monitor); let st = archive.open_stored_tree(options.band_selection.clone())?; let mut rt = if options.overwrite { diff --git a/src/stitch.rs b/src/stitch.rs index 78bec76f..400adfd7 100644 --- a/src/stitch.rs +++ b/src/stitch.rs @@ -267,7 +267,7 @@ mod test { let lt = tf.live_tree(); let af = ScratchArchive::new(); - backup(&af, <, &BackupOptions::default()).expect("backup should work"); + backup(&af, <, &BackupOptions::default(), None).expect("backup should work"); af.transport().remove_file("b0000/BANDTAIL").unwrap(); let band_ids = af.list_band_ids().expect("should list bands"); From 55bf2182739bc7da0e88ef75d5bc85cc3960d569 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Mon, 15 Aug 2022 13:06:12 +0200 Subject: [PATCH 31/39] Renaming DefaultMonitor to NullMonitor and using a single const instance --- src/archive.rs | 13 ++++--------- src/backup.rs | 5 ++--- src/blockdir.rs | 2 +- src/monitor.rs | 18 ++++++++++-------- src/restore.rs | 5 ++--- src/tree.rs | 5 ++--- 6 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index eda3cd2a..25f2fcce 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -30,7 +30,7 @@ use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; use crate::misc::remove_item; -use crate::monitor::{DefaultMonitor, DeleteMonitor, ReferencedBlocksMonitor}; +use crate::monitor::{DeleteMonitor, ReferencedBlocksMonitor, NULL_MONITOR}; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; @@ -203,15 +203,12 @@ impl Archive { } /// Returns all blocks referenced by all bands. - /// - /// Shows a progress bar as they're collected. pub fn referenced_blocks( &self, band_ids: &[BandId], monitor: Option<&dyn ReferencedBlocksMonitor>, ) -> Result> { - let default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&default_monitor); + let monitor = monitor.unwrap_or(&NULL_MONITOR); let archive = self.clone(); let current_count = AtomicUsize::new(0); @@ -253,8 +250,7 @@ impl Archive { options: &DeleteOptions, monitor: Option<&dyn DeleteMonitor>, ) -> Result { - let default_monitor = DefaultMonitor {}; - let monitor_ = monitor.unwrap_or(&default_monitor); + let monitor_ = monitor.unwrap_or(&NULL_MONITOR); let mut stats = DeleteStats::default(); let start = Instant::now(); @@ -360,8 +356,7 @@ impl Archive { options: &ValidateOptions, monitor: Option<&dyn ValidateMonitor>, ) -> Result { - let default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&default_monitor); + let monitor = monitor.unwrap_or(&NULL_MONITOR); let start = Instant::now(); monitor.validate_archive(); diff --git a/src/backup.rs b/src/backup.rs index 477381ee..606bd5fb 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -22,7 +22,7 @@ use tracing::{debug, Level}; use crate::blockdir::Address; use crate::io::read_with_retries; -use crate::monitor::DefaultMonitor; +use crate::monitor::{NULL_MONITOR}; use crate::stats::BackupStats; use crate::stitch::IterStitchedIndexHunks; use crate::tree::ReadTree; @@ -57,8 +57,7 @@ pub fn backup( ) -> Result { let _span = tracing::span!(Level::DEBUG, "backup"); - let default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&default_monitor); + let monitor = monitor.unwrap_or(&NULL_MONITOR); let start = Instant::now(); let mut writer = BackupWriter::begin(archive)?; diff --git a/src/blockdir.rs b/src/blockdir.rs index 5172089d..6fc8cc33 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -21,7 +21,6 @@ //! //! The structure is: archive > blockdir > subdir > file. -use rayon::prelude::*; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::io; @@ -32,6 +31,7 @@ use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; use serde::{Deserialize, Serialize}; use tracing::{error, warn}; +use rayon::prelude::*; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; diff --git a/src/monitor.rs b/src/monitor.rs index 8284086e..9a32dbb7 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -77,15 +77,17 @@ pub trait RestoreMonitor { /// Default monitor which does nothing. /// Will be used when no monitor has been specified by the caller. -pub(crate) struct DefaultMonitor {} - -impl BackupMonitor for DefaultMonitor {} -impl ValidateMonitor for DefaultMonitor {} -impl TreeSizeMonitor for DefaultMonitor {} -impl ReferencedBlocksMonitor for DefaultMonitor {} -impl DeleteMonitor for DefaultMonitor { +pub(crate) struct NullMonitor {} +pub(crate) const NULL_MONITOR: NullMonitor = NullMonitor{}; + +impl BackupMonitor for NullMonitor {} +impl ValidateMonitor for NullMonitor {} +impl TreeSizeMonitor for NullMonitor {} +impl ReferencedBlocksMonitor for NullMonitor {} +impl DeleteMonitor for NullMonitor { fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor { self } } -impl RestoreMonitor for DefaultMonitor {} +impl RestoreMonitor for NullMonitor {} + \ No newline at end of file diff --git a/src/restore.rs b/src/restore.rs index 19ae8afa..45c4092d 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -27,7 +27,7 @@ use tracing::{debug, warn}; use crate::band::BandSelectionPolicy; use crate::entry::Entry; use crate::io::{directory_is_empty, ensure_dir_exists}; -use crate::monitor::{DefaultMonitor, RestoreMonitor}; +use crate::monitor::{RestoreMonitor, NULL_MONITOR}; use crate::stats::RestoreStats; use crate::unix_time::UnixTime; use crate::*; @@ -61,8 +61,7 @@ pub fn restore( options: &RestoreOptions, monitor: Option<&dyn RestoreMonitor>, ) -> Result { - let default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&default_monitor); + let monitor = monitor.unwrap_or(&NULL_MONITOR); let st = archive.open_stored_tree(options.band_selection.clone())?; let mut rt = if options.overwrite { diff --git a/src/tree.rs b/src/tree.rs index 6fa77c15..85e2c1d5 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -15,7 +15,7 @@ use std::ops::Range; -use crate::monitor::DefaultMonitor; +use crate::monitor::NULL_MONITOR; use crate::stats::Sizes; use crate::*; @@ -51,8 +51,7 @@ pub trait ReadTree { where Self: Sized, { - let mut default_monitor = DefaultMonitor {}; - let monitor = monitor.unwrap_or(&mut default_monitor as &dyn TreeSizeMonitor); + let monitor = monitor.unwrap_or(&NULL_MONITOR as &dyn TreeSizeMonitor); let mut tot = 0u64; for e in self.iter_entries(Apath::root(), exclude)? { From e65b8d13981115e1ad19f375e643491d3aa34b82 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Mon, 15 Aug 2022 13:35:24 +0200 Subject: [PATCH 32/39] Implementing the simple review points --- src/bin/conserve/main.rs | 11 ++++------- src/bin/conserve/show.rs | 28 ++++++++++++++-------------- src/blockdir.rs | 16 ++++++++++------ src/lib.rs | 2 +- src/monitor.rs | 8 ++++---- src/validate.rs | 37 +++++++++++++++++++++++-------------- 6 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index ed4e1ef5..d0304a44 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -574,10 +574,7 @@ fn initialize_log(args: &Args) -> std::result::Result { terminal_raw: args.log_raw, })?; - if args.log_level == Some(tracing::Level::TRACE) { - trace!("tracing enabled"); - } - + trace!("tracing enabled"); Ok(guard) } @@ -603,14 +600,14 @@ fn main() -> ExitCode { cause = c; } - // // TODO: Perhaps always log the traceback to a log file. - // // NOTE(WolverinDEV): May always log this as trace level? + // NOTE(WolverinDEV): Reenable this as soon the feature backtrace lands in stable. + // When doing so, logging trough tracing::trace would be recommanded. + // (See https://github.com/sourcefrog/conserve/pull/173#discussion_r945195070) // if let Some(bt) = e.backtrace() { // if std::env::var("RUST_BACKTRACE") == Ok("1".to_string()) { // println!("{}", bt); // } // } - // Avoid Rust redundantly printing the error. ExitCode::Failed } Ok(code) => code, diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index c06fec9b..de40bf4a 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -26,7 +26,7 @@ use conserve::ui::duration_to_hms; use conserve::{ bytes_to_human_mb, Archive, BackupMonitor, Band, BandProblem, BandSelectionPolicy, BlockMissingReason, DeleteMonitor, DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, - ReadTree, ReferencedBlocksMonitor, RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, + ReadTree, ReferencedBlocksMonitor, RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, BandValidateError, ValidateStats, BlockLengths, }; use nutmeg::{Model, View}; use thousands::Separable; @@ -152,11 +152,11 @@ pub fn show_diff>(diff: D) -> Result<()> { } enum NutmegMonitorState { - View { + NutmegProgress { view: Arc>>, _log_guard: ViewLogGuard, }, - Simple { + NoProgress { state: Mutex, }, } @@ -174,12 +174,12 @@ impl NutmegMonitor { ))); let log_guard = log::update_terminal_target(view.clone()); - NutmegMonitorState::View { + NutmegMonitorState::NutmegProgress { view, _log_guard: log_guard, } } else { - NutmegMonitorState::Simple { + NutmegMonitorState::NoProgress { state: Mutex::new(initial_state), } }; @@ -189,11 +189,11 @@ impl NutmegMonitor { fn update_model R, R>(&self, update_fn: F) -> R { match &self.state { - NutmegMonitorState::View { view, .. } => { + NutmegMonitorState::NutmegProgress { view, .. } => { let view = view.lock().expect("lock() should not fail"); view.update(update_fn) } - NutmegMonitorState::Simple { state } => { + NutmegMonitorState::NoProgress { state } => { let mut state = state.lock().expect("lock() should not fail"); update_fn(&mut *state) } @@ -292,7 +292,7 @@ enum ValidateProgressState { bands_total: usize, start: Instant, }, - ListBlockes { + ListBlocks { discovered: usize, }, ReadBlocks { @@ -334,7 +334,7 @@ impl nutmeg::Model for ValidateProgressModel { nutmeg::estimate_remaining(start, *bands_done, *bands_total) ) } - ValidateProgressState::ListBlockes { discovered } => { + ValidateProgressState::ListBlocks { discovered } => { format!("Listing blocks ({} blocks discovered)", discovered) } ValidateProgressState::ReadBlocks { @@ -429,7 +429,7 @@ impl ValidateMonitor for NutmegMonitor { fn validate_band_result( &self, _band_id: &conserve::BandId, - _result: &conserve::BandValidateResult, + _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, ) { self.update_model(|model| { if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { @@ -459,7 +459,7 @@ impl ValidateMonitor for NutmegMonitor { } self.update_model(|model| { - model.state = ValidateProgressState::ListBlockes { + model.state = ValidateProgressState::ListBlocks { discovered: current_count, } }); @@ -481,7 +481,7 @@ impl ValidateMonitor for NutmegMonitor { fn read_block_result( &self, _block_hash: &conserve::BlockHash, - result: &Result<(Vec, Sizes)>, + result: &Result, ) { self.update_model(|model| { if let ValidateProgressState::ReadBlocks { @@ -490,8 +490,8 @@ impl ValidateMonitor for NutmegMonitor { .. } = &mut model.state { - if let Ok((bytes, _sizes)) = result { - *bytes_done += bytes.len(); + if let Ok(sizes) = result { + *bytes_done += sizes.uncompressed as usize; } else { // TODO: Add a fail counter. } diff --git a/src/blockdir.rs b/src/blockdir.rs index 6fc8cc33..845f0c44 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -281,12 +281,16 @@ impl BlockDir { .into_par_iter() .map(|hash| { let result = self.get_block_content(&hash); - // TODO(MH): Should we realy provide the block contents? May only return the size or the read error. - // This would also allow to read the blocks in parallel and use a simple iterator where - // we call the monitor. - monitor.read_block_result(&hash, &result); - - result.map(|(bytes, _sizes)| (hash, bytes.len())).ok() + match result { + Ok((bytes, sizes)) => { + monitor.read_block_result(&hash, &Ok(sizes)); + Some((hash, bytes.len())) + }, + Err(error) => { + monitor.read_block_result(&hash, &Err(error)); + None + } + } }) .collect(); diff --git a/src/lib.rs b/src/lib.rs index e34789fb..13293049 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -74,7 +74,7 @@ pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; -pub use crate::validate::{BandProblem, BandValidateResult, BlockMissingReason, ValidateOptions}; +pub use crate::validate::{BandProblem, BandValidateError, BlockLengths, BlockMissingReason, ValidateOptions}; pub type Result = std::result::Result; diff --git a/src/monitor.rs b/src/monitor.rs index 9a32dbb7..799a0850 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -1,7 +1,7 @@ use crate::{ archive::ValidateArchiveProblem, stats::Sizes, BackupStats, Band, BandId, BandProblem, - BandValidateResult, BlockHash, BlockMissingReason, DiffKind, Error, IndexEntry, LiveEntry, - ReadTree, Result, + BandValidateError, BlockHash, BlockMissingReason, DiffKind, Error, IndexEntry, LiveEntry, + ReadTree, Result, ValidateStats, validate::BlockLengths, }; /// Monitor the backup progress. @@ -27,7 +27,7 @@ pub trait ValidateMonitor: Sync { fn validate_band(&self, _band_id: &BandId) {} fn validate_band_problem(&self, _band: &Band, _problem: &BandProblem) {} - fn validate_band_result(&self, _band_id: &BandId, _result: &BandValidateResult) {} + fn validate_band_result(&self, _band_id: &BandId, _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>) {} fn validate_block_missing(&self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} fn validate_blocks(&self) {} @@ -37,7 +37,7 @@ pub trait ValidateMonitor: Sync { fn list_block_names_finished(&self) {} fn read_blocks(&self, _count: usize) {} - fn read_block_result(&self, _block_hash: &BlockHash, _result: &Result<(Vec, Sizes)>) {} + fn read_block_result(&self, _block_hash: &BlockHash, _result: &Result) {} fn read_blocks_finished(&self) {} } diff --git a/src/validate.rs b/src/validate.rs index c2c3a1b2..fbb10a74 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -24,17 +24,22 @@ pub struct ValidateOptions { pub skip_block_hashes: bool, } -/// Band validation result. -pub enum BandValidateResult { +/// Band validation error. +pub enum BandValidateError { MetadataError(Error), OpenError(Error), TreeOpenError(Error), TreeValidateError(Error), +} - Valid(BlockLengths, ValidateStats), +impl Into> for BandValidateError { + fn into(self) -> std::result::Result { + Err(self) + } } + pub enum BlockMissingReason { /// The target bock can not be found. NotExisting, @@ -87,13 +92,17 @@ pub(crate) fn validate_bands( monitor.validate_band_result(band_id, &result); match result { - BandValidateResult::MetadataError(_) => stats.band_metadata_problems += 1, - BandValidateResult::OpenError(_) => stats.band_open_errors += 1, - BandValidateResult::TreeOpenError(_) => stats.tree_open_errors += 1, - BandValidateResult::TreeValidateError(_) => stats.tree_validate_errors += 1, - BandValidateResult::Valid(st_block_lens, st_stats) => { + Ok((st_block_lens, st_stats)) => { stats += st_stats; block_lens.update(st_block_lens); + }, + Err(error) => { + match error { + BandValidateError::MetadataError(_) => stats.band_metadata_problems += 1, + BandValidateError::OpenError(_) => stats.band_open_errors += 1, + BandValidateError::TreeOpenError(_) => stats.tree_open_errors += 1, + BandValidateError::TreeValidateError(_) => stats.tree_validate_errors += 1, + } } } } @@ -106,25 +115,25 @@ pub(crate) fn validate_band( stats: &mut ValidateStats, band_id: &BandId, monitor: &dyn ValidateMonitor, -) -> BandValidateResult { +) -> std::result::Result<(BlockLengths, ValidateStats), BandValidateError> { let band = match Band::open(archive, band_id) { Ok(band) => band, - Err(error) => return BandValidateResult::OpenError(error), + Err(error) => return BandValidateError::OpenError(error).into(), }; if let Err(error) = band.validate(stats, monitor) { - return BandValidateResult::MetadataError(error); + return BandValidateError::MetadataError(error).into(); } let stored_tree = match archive.open_stored_tree(BandSelectionPolicy::Specified(band_id.clone())) { Ok(tree) => tree, - Err(error) => return BandValidateResult::TreeOpenError(error), + Err(error) => return BandValidateError::TreeOpenError(error).into(), }; match validate_stored_tree(&stored_tree) { - Ok(result) => BandValidateResult::Valid(result.0, result.1), - Err(error) => BandValidateResult::TreeValidateError(error), + Ok(result) => Ok(result), + Err(error) => BandValidateError::TreeValidateError(error).into(), } } From 820815230f8c82313b943e4045c2ea680fc681f1 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Mon, 15 Aug 2022 13:41:21 +0200 Subject: [PATCH 33/39] Using From instead of Into --- src/validate.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/validate.rs b/src/validate.rs index fbb10a74..53271a78 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -34,9 +34,9 @@ pub enum BandValidateError { TreeValidateError(Error), } -impl Into> for BandValidateError { - fn into(self) -> std::result::Result { - Err(self) +impl From for std::result::Result { + fn from(error: BandValidateError) -> Self { + Err(error) } } From 7859ce2b5763f9a48ba86f8447acc42e17c87602 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Mon, 15 Aug 2022 13:55:49 +0200 Subject: [PATCH 34/39] Seperating nutmeg monitors into a seperate file --- src/backup.rs | 2 +- src/bin/conserve/main.rs | 8 +- src/bin/conserve/monitor.rs | 548 +++++++++++++++++++++++++++++++++++ src/bin/conserve/show.rs | 552 +----------------------------------- src/blockdir.rs | 4 +- src/lib.rs | 4 +- src/monitor.rs | 16 +- src/validate.rs | 14 +- 8 files changed, 577 insertions(+), 571 deletions(-) create mode 100644 src/bin/conserve/monitor.rs diff --git a/src/backup.rs b/src/backup.rs index 606bd5fb..9bf81b37 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -22,7 +22,7 @@ use tracing::{debug, Level}; use crate::blockdir::Address; use crate::io::read_with_retries; -use crate::monitor::{NULL_MONITOR}; +use crate::monitor::NULL_MONITOR; use crate::stats::BackupStats; use crate::stitch::IterStitchedIndexHunks; use crate::tree::ReadTree; diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index d0304a44..402c8f47 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -20,11 +20,12 @@ use std::str::FromStr; use clap::{Parser, StructOpt, Subcommand}; use log::{LogGuard, LoggingOptions}; -use show::{show_diff, show_versions, ShowVersionsOptions}; -use show::{ +use monitor::ValidateProgressModel; +use monitor::{ BackupProgressModel, DeleteProcessState, NutmegMonitor, ReferencedBlocksProgressModel, RestoreProgressModel, SizeProgressModel, }; +use show::{show_diff, show_versions, ShowVersionsOptions}; use tracing::{error, info, trace, warn, Level}; use conserve::backup::BackupOptions; @@ -32,9 +33,8 @@ use conserve::ReadTree; use conserve::RestoreOptions; use conserve::*; -use crate::show::ValidateProgressModel; - mod log; +mod monitor; mod show; #[derive(Debug, Parser)] diff --git a/src/bin/conserve/monitor.rs b/src/bin/conserve/monitor.rs new file mode 100644 index 00000000..7fe3bac0 --- /dev/null +++ b/src/bin/conserve/monitor.rs @@ -0,0 +1,548 @@ +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use conserve::archive::ValidateArchiveProblem; +use conserve::stats::Sizes; +use conserve::{ + BackupMonitor, Band, BandProblem, BandValidateError, BlockLengths, BlockMissingReason, + DeleteMonitor, DiffKind, Entry, IndexEntry, Kind, ReadTree, ReferencedBlocksMonitor, + RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, ValidateStats, +}; +use nutmeg::{Model, View}; +use thousands::Separable; +use tracing::{error, info, warn}; + +use crate::log::{self, ViewLogGuard}; + +enum NutmegMonitorState { + NutmegProgress { + view: Arc>>, + _log_guard: ViewLogGuard, + }, + NoProgress { + state: Mutex, + }, +} + +pub struct NutmegMonitor { + state: NutmegMonitorState, +} + +impl NutmegMonitor { + pub fn new(initial_state: T, progress_enabled: bool) -> Self { + let state = if progress_enabled { + let view = Arc::new(Mutex::new(nutmeg::View::new( + initial_state, + nutmeg::Options::default().progress_enabled(progress_enabled), + ))); + + let log_guard = log::update_terminal_target(view.clone()); + NutmegMonitorState::NutmegProgress { + view, + _log_guard: log_guard, + } + } else { + NutmegMonitorState::NoProgress { + state: Mutex::new(initial_state), + } + }; + + Self { state } + } + + fn update_model R, R>(&self, update_fn: F) -> R { + match &self.state { + NutmegMonitorState::NutmegProgress { view, .. } => { + let view = view.lock().expect("lock() should not fail"); + view.update(update_fn) + } + NutmegMonitorState::NoProgress { state } => { + let mut state = state.lock().expect("lock() should not fail"); + update_fn(&mut *state) + } + } + } +} + +// Considerations if we're trying properly extimate the remaining progress. +// +// This causes us to walk the source tree twice, which is probably an acceptable option +// since it's nice to see realistic overall progress. We could keep all the entries +// in memory, and maybe we should, but it might get unreasonably big. +// if options.measure_first { +// progress_bar.set_phase("Measure source tree".to_owned()); +// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it +// // again a second time? But, that'll potentially use memory proportional to tree size, which +// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were +// // deleted or changed while this is running. +// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); +// } + +#[derive(Default)] +pub struct BackupProgressModel { + pub verbose: bool, + filename: String, + + scanned_file_bytes: u64, + scanned_dirs: usize, + scanned_files: usize, + + entries_new: usize, + entries_changed: usize, + entries_unchanged: usize, + entries_deleted: usize, +} + +impl nutmeg::Model for BackupProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", + self.scanned_dirs, + self.scanned_files, + self.scanned_file_bytes / 1_000_000, + self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, + self.filename + ) + } +} + +impl BackupMonitor for NutmegMonitor { + fn copy(&self, entry: &conserve::LiveEntry) { + self.update_model(|model| { + model.filename = entry.apath().to_string(); + match entry.kind() { + Kind::Dir => model.scanned_dirs += 1, + Kind::File => model.scanned_files += 1, + _ => (), + } + }); + } + + fn copy_result(&self, entry: &conserve::LiveEntry, result: &Option) { + if let Some(diff_kind) = result.as_ref() { + let verbose = self.update_model(|model| { + match diff_kind { + DiffKind::Changed => model.entries_changed += 1, + DiffKind::New => model.entries_new += 1, + DiffKind::Unchanged => model.entries_unchanged += 1, + DiffKind::Deleted => model.entries_deleted += 1, + }; + + model.verbose + }); + + if verbose { + info!("{} {}", diff_kind.as_sigil(), entry.apath()); + } + } + + if let Some(size) = entry.size() { + self.update_model(|model| model.scanned_file_bytes += size); + } + } + + fn copy_error(&self, entry: &conserve::LiveEntry, _error: &conserve::Error) { + if let Some(size) = entry.size() { + self.update_model(|model| model.scanned_file_bytes += size); + } + } +} + +enum ValidateProgressState { + CountBands, + ValidateBands { + bands_done: usize, + bands_total: usize, + start: Instant, + }, + ListBlocks { + discovered: usize, + }, + ReadBlocks { + total_blocks: usize, + blocks_done: usize, + bytes_done: usize, + start: Instant, + }, +} + +pub struct ValidateProgressModel { + bands_total: Option, + state: ValidateProgressState, +} + +impl Default for ValidateProgressModel { + fn default() -> Self { + Self { + bands_total: None, + state: ValidateProgressState::CountBands {}, + } + } +} + +impl nutmeg::Model for ValidateProgressModel { + fn render(&mut self, _width: usize) -> String { + match &self.state { + ValidateProgressState::CountBands => "Counting bands".to_string(), + ValidateProgressState::ValidateBands { + bands_done, + bands_total, + start, + } => { + format!( + "Check index {}/{}, {} done, {} remaining", + bands_done, + bands_total, + nutmeg::percent_done(*bands_done, *bands_total), + nutmeg::estimate_remaining(start, *bands_done, *bands_total) + ) + } + ValidateProgressState::ListBlocks { discovered } => { + format!("Listing blocks ({} blocks discovered)", discovered) + } + ValidateProgressState::ReadBlocks { + total_blocks, + blocks_done, + bytes_done, + start, + } => { + format!( + "Check block {}/{}: {} done, {} MB checked, {} remaining", + *blocks_done, + *total_blocks, + nutmeg::percent_done(*blocks_done, *total_blocks), + *bytes_done / 1_000_000, + nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) + ) + } + } + } +} + +impl ValidateMonitor for NutmegMonitor { + fn validate_archive(&self) { + info!("Check archive top-level directory..."); + } + + fn validate_archive_problem(&self, problem: &ValidateArchiveProblem) { + match problem { + ValidateArchiveProblem::UnexpectedFileType { name, kind } => { + error!( + "Unexpected file kind in archive directory: {:?} of kind {:?}", + name, kind + ); + } + ValidateArchiveProblem::DirectoryListError { error } => { + error!("Error listing archive directory: {:?}", error); + } + ValidateArchiveProblem::UnexpectedFiles { path, files } => { + error!( + "Unexpected files in archive directory {:?}: {:?}", + path, files + ); + } + ValidateArchiveProblem::DuplicateBand { path, directory } => { + error!("Duplicated band directory in {:?}: {:?}", path, directory); + } + ValidateArchiveProblem::UnexpectedDirectory { path, directory } => { + error!("Unexpected directory in {:?}: {:?}", path, directory); + } + } + } + + fn count_bands(&self) { + info!("Count bands..."); + } + + fn count_bands_result(&self, bands: &[conserve::BandId]) { + info!("Checking {} bands...", bands.len()); + + self.update_model(|model| model.bands_total = Some(bands.len())); + } + + fn validate_bands(&self) { + self.update_model(|model| { + let bands_total = model.bands_total.expect("bands have been counted"); + model.state = ValidateProgressState::ValidateBands { + bands_done: 0, + bands_total, + start: Instant::now(), + }; + }); + } + + fn validate_band_problem(&self, band: &Band, problem: &conserve::BandProblem) { + match problem { + BandProblem::MissingHeadFile { .. } => { + warn!("No band head file in {:?}", band.transport()) + } + BandProblem::UnexpectedFiles { files } => warn!( + "Unexpected files in band directory {:?}: {:?}", + band.transport(), + files + ), + BandProblem::UnexpectedDirectories { directories } => warn!( + "Incongruous directories in band directory {:?}: {:?}", + band.transport(), + directories + ), + } + } + + fn validate_band_result( + &self, + _band_id: &conserve::BandId, + _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, + ) { + self.update_model(|model| { + if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { + *bands_done += 1; + } else { + panic!("Expected state ValidateProgressState::ValidateBands"); + } + }); + } + + fn validate_bands_finished(&self) { + // We can't use logging while locked_view is held since we would deadlock. + let elapsed = self.update_model(|model| { + if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { + start.elapsed() + } else { + panic!("Expected state ValidateProgressState::ValidateBands"); + } + }); + + info!("Finished validating bands in {:#?}.", elapsed); + } + + fn list_block_names(&self, current_count: usize) { + if current_count == 0 { + info!("Count blocks..."); + } + + self.update_model(|model| { + model.state = ValidateProgressState::ListBlocks { + discovered: current_count, + } + }); + } + + fn read_blocks(&self, count: usize) { + info!("Check {} blocks...", count.separate_with_commas()); + + self.update_model(|model| { + model.state = ValidateProgressState::ReadBlocks { + total_blocks: count, + blocks_done: 0, + bytes_done: 0, + start: Instant::now(), + } + }); + } + + fn read_block_result(&self, _block_hash: &conserve::BlockHash, result: &Result) { + self.update_model(|model| { + if let ValidateProgressState::ReadBlocks { + blocks_done, + bytes_done, + .. + } = &mut model.state + { + if let Ok(sizes) = result { + *bytes_done += sizes.uncompressed as usize; + } else { + // TODO: Add a fail counter. + } + + *blocks_done += 1; + } else { + panic!("Expected state ValidateProgressState::ReadBlocks"); + } + }); + } + + fn validate_block_missing( + &self, + block_hash: &conserve::BlockHash, + reason: &conserve::BlockMissingReason, + ) { + match reason { + BlockMissingReason::NotExisting => warn!("Block {:?} is missing", block_hash), + BlockMissingReason::InvalidRange => warn!("Block {:?} is too short", block_hash), + } + } +} + +#[derive(Default)] +pub struct SizeProgressModel { + files: usize, + total_bytes: u64, +} +impl nutmeg::Model for SizeProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Measuring... {} files, {} MB", + self.files, + self.total_bytes / 1_000_000 + ) + } +} + +impl TreeSizeMonitor for NutmegMonitor { + fn entry_discovered(&self, _entry: &::Entry, size: &Option) { + self.update_model(|model| { + model.files += 1; + model.total_bytes += size.unwrap_or(0); + }); + } +} + +pub enum DeleteProcessState { + ListReferencedBlocks { count: usize }, + FindPresentBlocks { count: usize }, + MeasureUnreferencedBlocks { count: usize, target: usize }, + DeleteBands { count: usize, target: usize }, + DeleteBlocks { count: usize, target: usize }, +} + +impl Default for DeleteProcessState { + fn default() -> Self { + DeleteProcessState::ListReferencedBlocks { count: 0 } + } +} + +impl Model for DeleteProcessState { + fn render(&mut self, _width: usize) -> String { + match self { + DeleteProcessState::ListReferencedBlocks { count } => { + format!("Find referenced blocks in band ({} discovered)", count) + } + DeleteProcessState::FindPresentBlocks { count } => { + format!("Find present blocks ({} discovered)", count) + } + DeleteProcessState::MeasureUnreferencedBlocks { count, target } => { + format!("Measure unreferenced blocks ({}/{})", count, target) + } + DeleteProcessState::DeleteBands { count, target } => { + format!("Delete bands ({}/{})", count, target) + } + DeleteProcessState::DeleteBlocks { count, target } => { + format!("Delete blocks ({}/{})", count, target) + } + } + } +} + +impl DeleteMonitor for NutmegMonitor { + fn referenced_blocks_monitor(&self) -> &dyn conserve::ReferencedBlocksMonitor { + self + } + + fn find_present_blocks(&self, current_count: usize) { + self.update_model(|view| { + *view = DeleteProcessState::FindPresentBlocks { + count: current_count, + }; + }); + } + + fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { + self.update_model(|view| { + *view = DeleteProcessState::MeasureUnreferencedBlocks { + count: current_count, + target: target_count, + }; + }); + } + + fn delete_bands(&self, current_count: usize, target_count: usize) { + self.update_model(|view| { + *view = DeleteProcessState::DeleteBands { + count: current_count, + target: target_count, + }; + }); + } + + fn delete_blocks(&self, current_count: usize, target_count: usize) { + self.update_model(|view| { + *view = DeleteProcessState::DeleteBlocks { + count: current_count, + target: target_count, + }; + }); + } +} + +impl ReferencedBlocksMonitor for NutmegMonitor { + fn list_referenced_blocks(&self, current_count: usize) { + self.update_model(|view| { + *view = DeleteProcessState::ListReferencedBlocks { + count: current_count, + }; + }); + } +} + +pub struct RestoreProgressModel { + print_filenames: bool, + filename: String, + bytes_done: u64, +} + +impl RestoreProgressModel { + pub fn new(print_filenames: bool) -> Self { + Self { + print_filenames, + filename: "".to_string(), + bytes_done: 0, + } + } +} + +impl nutmeg::Model for RestoreProgressModel { + fn render(&mut self, _width: usize) -> String { + format!( + "Restoring: {} MB\n{}", + self.bytes_done / 1_000_000, + self.filename + ) + } +} + +impl RestoreMonitor for NutmegMonitor { + fn restore_entry(&self, entry: &IndexEntry) { + let print_filename = self.update_model(|view| { + view.filename = entry.apath().to_string(); + view.print_filenames + }); + + if print_filename { + info!("{}", entry.apath()); + } + } + + fn restore_entry_result(&self, entry: &IndexEntry, _result: &Result<()>) { + if let Some(bytes) = entry.size() { + self.update_model(|view| view.bytes_done += bytes); + } + } +} + +#[derive(Default)] +pub struct ReferencedBlocksProgressModel { + count: usize, +} + +impl Model for ReferencedBlocksProgressModel { + fn render(&mut self, _width: usize) -> String { + format!("Find referenced blocks in band ({} discovered)", self.count) + } +} + +impl ReferencedBlocksMonitor for NutmegMonitor { + fn list_referenced_blocks(&self, current_count: usize) { + self.update_model(|model| model.count = current_count); + } +} diff --git a/src/bin/conserve/show.rs b/src/bin/conserve/show.rs index de40bf4a..a7518b86 100644 --- a/src/bin/conserve/show.rs +++ b/src/bin/conserve/show.rs @@ -17,22 +17,13 @@ //! file (typically stdout). use std::borrow::Cow; -use std::sync::{Arc, Mutex}; -use std::time::Instant; -use conserve::archive::ValidateArchiveProblem; -use conserve::stats::Sizes; use conserve::ui::duration_to_hms; use conserve::{ - bytes_to_human_mb, Archive, BackupMonitor, Band, BandProblem, BandSelectionPolicy, - BlockMissingReason, DeleteMonitor, DiffEntry, DiffKind, Entry, Exclude, IndexEntry, Kind, - ReadTree, ReferencedBlocksMonitor, RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, BandValidateError, ValidateStats, BlockLengths, + bytes_to_human_mb, Archive, Band, BandSelectionPolicy, DiffEntry, Exclude, IndexEntry, + ReadTree, Result, }; -use nutmeg::{Model, View}; -use thousands::Separable; -use tracing::{error, info, warn}; - -use crate::log::{self, ViewLogGuard}; +use tracing::{info, warn}; /// ISO timestamp, for https://docs.rs/chrono/0.4.11/chrono/format/strftime/. const TIMESTAMP_FORMAT: &str = "%F %T"; @@ -150,540 +141,3 @@ pub fn show_diff>(diff: D) -> Result<()> { Ok(()) } - -enum NutmegMonitorState { - NutmegProgress { - view: Arc>>, - _log_guard: ViewLogGuard, - }, - NoProgress { - state: Mutex, - }, -} - -pub struct NutmegMonitor { - state: NutmegMonitorState, -} - -impl NutmegMonitor { - pub fn new(initial_state: T, progress_enabled: bool) -> Self { - let state = if progress_enabled { - let view = Arc::new(Mutex::new(nutmeg::View::new( - initial_state, - nutmeg::Options::default().progress_enabled(progress_enabled), - ))); - - let log_guard = log::update_terminal_target(view.clone()); - NutmegMonitorState::NutmegProgress { - view, - _log_guard: log_guard, - } - } else { - NutmegMonitorState::NoProgress { - state: Mutex::new(initial_state), - } - }; - - Self { state } - } - - fn update_model R, R>(&self, update_fn: F) -> R { - match &self.state { - NutmegMonitorState::NutmegProgress { view, .. } => { - let view = view.lock().expect("lock() should not fail"); - view.update(update_fn) - } - NutmegMonitorState::NoProgress { state } => { - let mut state = state.lock().expect("lock() should not fail"); - update_fn(&mut *state) - } - } - } -} - -// Considerations if we're trying properly extimate the remaining progress. -// -// This causes us to walk the source tree twice, which is probably an acceptable option -// since it's nice to see realistic overall progress. We could keep all the entries -// in memory, and maybe we should, but it might get unreasonably big. -// if options.measure_first { -// progress_bar.set_phase("Measure source tree".to_owned()); -// // TODO: Maybe read all entries for the source tree in to memory now, rather than walking it -// // again a second time? But, that'll potentially use memory proportional to tree size, which -// // I'd like to avoid, and also perhaps make it more likely we grumble about files that were -// // deleted or changed while this is running. -// progress_bar.set_bytes_total(source.size()?.file_bytes as u64); -// } - -#[derive(Default)] -pub struct BackupProgressModel { - pub verbose: bool, - filename: String, - - scanned_file_bytes: u64, - scanned_dirs: usize, - scanned_files: usize, - - entries_new: usize, - entries_changed: usize, - entries_unchanged: usize, - entries_deleted: usize, -} - -impl nutmeg::Model for BackupProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Scanned {} directories, {} files, {} MB\n{} new entries, {} changed, {} deleted, {} unchanged\n{}", - self.scanned_dirs, - self.scanned_files, - self.scanned_file_bytes / 1_000_000, - self.entries_new, self.entries_changed, self.entries_deleted, self.entries_unchanged, - self.filename - ) - } -} - -impl BackupMonitor for NutmegMonitor { - fn copy(&self, entry: &conserve::LiveEntry) { - self.update_model(|model| { - model.filename = entry.apath().to_string(); - match entry.kind() { - Kind::Dir => model.scanned_dirs += 1, - Kind::File => model.scanned_files += 1, - _ => (), - } - }); - } - - fn copy_result(&self, entry: &conserve::LiveEntry, result: &Option) { - if let Some(diff_kind) = result.as_ref() { - let verbose = self.update_model(|model| { - match diff_kind { - DiffKind::Changed => model.entries_changed += 1, - DiffKind::New => model.entries_new += 1, - DiffKind::Unchanged => model.entries_unchanged += 1, - DiffKind::Deleted => model.entries_deleted += 1, - }; - - model.verbose - }); - - if verbose { - info!("{} {}", diff_kind.as_sigil(), entry.apath()); - } - } - - if let Some(size) = entry.size() { - self.update_model(|model| model.scanned_file_bytes += size); - } - } - - fn copy_error(&self, entry: &conserve::LiveEntry, _error: &conserve::Error) { - if let Some(size) = entry.size() { - self.update_model(|model| model.scanned_file_bytes += size); - } - } -} - -enum ValidateProgressState { - CountBands, - ValidateBands { - bands_done: usize, - bands_total: usize, - start: Instant, - }, - ListBlocks { - discovered: usize, - }, - ReadBlocks { - total_blocks: usize, - blocks_done: usize, - bytes_done: usize, - start: Instant, - }, -} - -pub struct ValidateProgressModel { - bands_total: Option, - state: ValidateProgressState, -} - -impl Default for ValidateProgressModel { - fn default() -> Self { - Self { - bands_total: None, - state: ValidateProgressState::CountBands {}, - } - } -} - -impl nutmeg::Model for ValidateProgressModel { - fn render(&mut self, _width: usize) -> String { - match &self.state { - ValidateProgressState::CountBands => "Counting bands".to_string(), - ValidateProgressState::ValidateBands { - bands_done, - bands_total, - start, - } => { - format!( - "Check index {}/{}, {} done, {} remaining", - bands_done, - bands_total, - nutmeg::percent_done(*bands_done, *bands_total), - nutmeg::estimate_remaining(start, *bands_done, *bands_total) - ) - } - ValidateProgressState::ListBlocks { discovered } => { - format!("Listing blocks ({} blocks discovered)", discovered) - } - ValidateProgressState::ReadBlocks { - total_blocks, - blocks_done, - bytes_done, - start, - } => { - format!( - "Check block {}/{}: {} done, {} MB checked, {} remaining", - *blocks_done, - *total_blocks, - nutmeg::percent_done(*blocks_done, *total_blocks), - *bytes_done / 1_000_000, - nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) - ) - } - } - } -} - -impl ValidateMonitor for NutmegMonitor { - fn validate_archive(&self) { - info!("Check archive top-level directory..."); - } - - fn validate_archive_problem(&self, problem: &ValidateArchiveProblem) { - match problem { - ValidateArchiveProblem::UnexpectedFileType { name, kind } => { - error!( - "Unexpected file kind in archive directory: {:?} of kind {:?}", - name, kind - ); - } - ValidateArchiveProblem::DirectoryListError { error } => { - error!("Error listing archive directory: {:?}", error); - } - ValidateArchiveProblem::UnexpectedFiles { path, files } => { - error!( - "Unexpected files in archive directory {:?}: {:?}", - path, files - ); - } - ValidateArchiveProblem::DuplicateBand { path, directory } => { - error!("Duplicated band directory in {:?}: {:?}", path, directory); - } - ValidateArchiveProblem::UnexpectedDirectory { path, directory } => { - error!("Unexpected directory in {:?}: {:?}", path, directory); - } - } - } - - fn count_bands(&self) { - info!("Count bands..."); - } - - fn count_bands_result(&self, bands: &[conserve::BandId]) { - info!("Checking {} bands...", bands.len()); - - self.update_model(|model| model.bands_total = Some(bands.len())); - } - - fn validate_bands(&self) { - self.update_model(|model| { - let bands_total = model.bands_total.expect("bands have been counted"); - model.state = ValidateProgressState::ValidateBands { - bands_done: 0, - bands_total, - start: Instant::now(), - }; - }); - } - - fn validate_band_problem(&self, band: &Band, problem: &conserve::BandProblem) { - match problem { - BandProblem::MissingHeadFile { .. } => { - warn!("No band head file in {:?}", band.transport()) - } - BandProblem::UnexpectedFiles { files } => warn!( - "Unexpected files in band directory {:?}: {:?}", - band.transport(), - files - ), - BandProblem::UnexpectedDirectories { directories } => warn!( - "Incongruous directories in band directory {:?}: {:?}", - band.transport(), - directories - ), - } - } - - fn validate_band_result( - &self, - _band_id: &conserve::BandId, - _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, - ) { - self.update_model(|model| { - if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { - *bands_done += 1; - } else { - panic!("Expected state ValidateProgressState::ValidateBands"); - } - }); - } - - fn validate_bands_finished(&self) { - // We can't use logging while locked_view is held since we would deadlock. - let elapsed = self.update_model(|model| { - if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { - start.elapsed() - } else { - panic!("Expected state ValidateProgressState::ValidateBands"); - } - }); - - info!("Finished validating bands in {:#?}.", elapsed); - } - - fn list_block_names(&self, current_count: usize) { - if current_count == 0 { - info!("Count blocks..."); - } - - self.update_model(|model| { - model.state = ValidateProgressState::ListBlocks { - discovered: current_count, - } - }); - } - - fn read_blocks(&self, count: usize) { - info!("Check {} blocks...", count.separate_with_commas()); - - self.update_model(|model| { - model.state = ValidateProgressState::ReadBlocks { - total_blocks: count, - blocks_done: 0, - bytes_done: 0, - start: Instant::now(), - } - }); - } - - fn read_block_result( - &self, - _block_hash: &conserve::BlockHash, - result: &Result, - ) { - self.update_model(|model| { - if let ValidateProgressState::ReadBlocks { - blocks_done, - bytes_done, - .. - } = &mut model.state - { - if let Ok(sizes) = result { - *bytes_done += sizes.uncompressed as usize; - } else { - // TODO: Add a fail counter. - } - - *blocks_done += 1; - } else { - panic!("Expected state ValidateProgressState::ReadBlocks"); - } - }); - } - - fn validate_block_missing( - &self, - block_hash: &conserve::BlockHash, - reason: &conserve::BlockMissingReason, - ) { - match reason { - BlockMissingReason::NotExisting => warn!("Block {:?} is missing", block_hash), - BlockMissingReason::InvalidRange => warn!("Block {:?} is too short", block_hash), - } - } -} - -#[derive(Default)] -pub struct SizeProgressModel { - files: usize, - total_bytes: u64, -} -impl nutmeg::Model for SizeProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Measuring... {} files, {} MB", - self.files, - self.total_bytes / 1_000_000 - ) - } -} - -impl TreeSizeMonitor for NutmegMonitor { - fn entry_discovered(&self, _entry: &::Entry, size: &Option) { - self.update_model(|model| { - model.files += 1; - model.total_bytes += size.unwrap_or(0); - }); - } -} - -pub enum DeleteProcessState { - ListReferencedBlocks { count: usize }, - FindPresentBlocks { count: usize }, - MeasureUnreferencedBlocks { count: usize, target: usize }, - DeleteBands { count: usize, target: usize }, - DeleteBlocks { count: usize, target: usize }, -} - -impl Default for DeleteProcessState { - fn default() -> Self { - DeleteProcessState::ListReferencedBlocks { count: 0 } - } -} - -impl Model for DeleteProcessState { - fn render(&mut self, _width: usize) -> String { - match self { - DeleteProcessState::ListReferencedBlocks { count } => { - format!("Find referenced blocks in band ({} discovered)", count) - } - DeleteProcessState::FindPresentBlocks { count } => { - format!("Find present blocks ({} discovered)", count) - } - DeleteProcessState::MeasureUnreferencedBlocks { count, target } => { - format!("Measure unreferenced blocks ({}/{})", count, target) - } - DeleteProcessState::DeleteBands { count, target } => { - format!("Delete bands ({}/{})", count, target) - } - DeleteProcessState::DeleteBlocks { count, target } => { - format!("Delete blocks ({}/{})", count, target) - } - } - } -} - -impl DeleteMonitor for NutmegMonitor { - fn referenced_blocks_monitor(&self) -> &dyn conserve::ReferencedBlocksMonitor { - self - } - - fn find_present_blocks(&self, current_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::FindPresentBlocks { - count: current_count, - }; - }); - } - - fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::MeasureUnreferencedBlocks { - count: current_count, - target: target_count, - }; - }); - } - - fn delete_bands(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::DeleteBands { - count: current_count, - target: target_count, - }; - }); - } - - fn delete_blocks(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::DeleteBlocks { - count: current_count, - target: target_count, - }; - }); - } -} - -impl ReferencedBlocksMonitor for NutmegMonitor { - fn list_referenced_blocks(&self, current_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::ListReferencedBlocks { - count: current_count, - }; - }); - } -} - -pub struct RestoreProgressModel { - print_filenames: bool, - filename: String, - bytes_done: u64, -} - -impl RestoreProgressModel { - pub fn new(print_filenames: bool) -> Self { - Self { - print_filenames, - filename: "".to_string(), - bytes_done: 0, - } - } -} - -impl nutmeg::Model for RestoreProgressModel { - fn render(&mut self, _width: usize) -> String { - format!( - "Restoring: {} MB\n{}", - self.bytes_done / 1_000_000, - self.filename - ) - } -} - -impl RestoreMonitor for NutmegMonitor { - fn restore_entry(&self, entry: &IndexEntry) { - let print_filename = self.update_model(|view| { - view.filename = entry.apath().to_string(); - view.print_filenames - }); - - if print_filename { - info!("{}", entry.apath()); - } - } - - fn restore_entry_result(&self, entry: &IndexEntry, _result: &Result<()>) { - if let Some(bytes) = entry.size() { - self.update_model(|view| view.bytes_done += bytes); - } - } -} - -#[derive(Default)] -pub struct ReferencedBlocksProgressModel { - count: usize, -} - -impl Model for ReferencedBlocksProgressModel { - fn render(&mut self, _width: usize) -> String { - format!("Find referenced blocks in band ({} discovered)", self.count) - } -} - -impl ReferencedBlocksMonitor for NutmegMonitor { - fn list_referenced_blocks(&self, current_count: usize) { - self.update_model(|model| model.count = current_count); - } -} diff --git a/src/blockdir.rs b/src/blockdir.rs index 845f0c44..26cc9b64 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -29,9 +29,9 @@ use std::sync::Arc; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use tracing::{error, warn}; -use rayon::prelude::*; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; @@ -285,7 +285,7 @@ impl BlockDir { Ok((bytes, sizes)) => { monitor.read_block_result(&hash, &Ok(sizes)); Some((hash, bytes.len())) - }, + } Err(error) => { monitor.read_block_result(&hash, &Err(error)); None diff --git a/src/lib.rs b/src/lib.rs index 13293049..7b78f777 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -74,7 +74,9 @@ pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; pub use crate::stored_tree::StoredTree; pub use crate::transport::{open_transport, Transport}; pub use crate::tree::{ReadBlocks, ReadTree, TreeSize}; -pub use crate::validate::{BandProblem, BandValidateError, BlockLengths, BlockMissingReason, ValidateOptions}; +pub use crate::validate::{ + BandProblem, BandValidateError, BlockLengths, BlockMissingReason, ValidateOptions, +}; pub type Result = std::result::Result; diff --git a/src/monitor.rs b/src/monitor.rs index 799a0850..20bcc4e8 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -1,7 +1,7 @@ use crate::{ - archive::ValidateArchiveProblem, stats::Sizes, BackupStats, Band, BandId, BandProblem, - BandValidateError, BlockHash, BlockMissingReason, DiffKind, Error, IndexEntry, LiveEntry, - ReadTree, Result, ValidateStats, validate::BlockLengths, + archive::ValidateArchiveProblem, stats::Sizes, validate::BlockLengths, BackupStats, Band, + BandId, BandProblem, BandValidateError, BlockHash, BlockMissingReason, DiffKind, Error, + IndexEntry, LiveEntry, ReadTree, Result, ValidateStats, }; /// Monitor the backup progress. @@ -27,7 +27,12 @@ pub trait ValidateMonitor: Sync { fn validate_band(&self, _band_id: &BandId) {} fn validate_band_problem(&self, _band: &Band, _problem: &BandProblem) {} - fn validate_band_result(&self, _band_id: &BandId, _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>) {} + fn validate_band_result( + &self, + _band_id: &BandId, + _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, + ) { + } fn validate_block_missing(&self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} fn validate_blocks(&self) {} @@ -78,7 +83,7 @@ pub trait RestoreMonitor { /// Default monitor which does nothing. /// Will be used when no monitor has been specified by the caller. pub(crate) struct NullMonitor {} -pub(crate) const NULL_MONITOR: NullMonitor = NullMonitor{}; +pub(crate) const NULL_MONITOR: NullMonitor = NullMonitor {}; impl BackupMonitor for NullMonitor {} impl ValidateMonitor for NullMonitor {} @@ -90,4 +95,3 @@ impl DeleteMonitor for NullMonitor { } } impl RestoreMonitor for NullMonitor {} - \ No newline at end of file diff --git a/src/validate.rs b/src/validate.rs index 53271a78..c39748d9 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -95,15 +95,13 @@ pub(crate) fn validate_bands( Ok((st_block_lens, st_stats)) => { stats += st_stats; block_lens.update(st_block_lens); - }, - Err(error) => { - match error { - BandValidateError::MetadataError(_) => stats.band_metadata_problems += 1, - BandValidateError::OpenError(_) => stats.band_open_errors += 1, - BandValidateError::TreeOpenError(_) => stats.tree_open_errors += 1, - BandValidateError::TreeValidateError(_) => stats.tree_validate_errors += 1, - } } + Err(error) => match error { + BandValidateError::MetadataError(_) => stats.band_metadata_problems += 1, + BandValidateError::OpenError(_) => stats.band_open_errors += 1, + BandValidateError::TreeOpenError(_) => stats.tree_open_errors += 1, + BandValidateError::TreeValidateError(_) => stats.tree_validate_errors += 1, + }, } } From 8c8c244f94ca952cf73ddfb54769f5552e1227c9 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 24 Aug 2022 17:36:23 +0200 Subject: [PATCH 35/39] Simplifying process updates via monitors --- src/archive.rs | 77 ++++---- src/band.rs | 6 +- src/bin/conserve/main.rs | 6 +- src/bin/conserve/monitor.rs | 359 +++++++++++++++--------------------- src/blockdir.rs | 23 ++- src/lib.rs | 2 +- src/monitor.rs | 92 +++++---- src/validate.rs | 8 +- 8 files changed, 269 insertions(+), 304 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 25f2fcce..1ad20138 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -30,7 +30,7 @@ use crate::errors::Error; use crate::jsonio::{read_json, write_json}; use crate::kind::Kind; use crate::misc::remove_item; -use crate::monitor::{DeleteMonitor, ReferencedBlocksMonitor, NULL_MONITOR}; +use crate::monitor::{DeleteMonitor, ReferencedBlocksMonitor, NULL_MONITOR, ValidateProgress, DeleteProgress}; use crate::stats::ValidateStats; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, Transport}; @@ -209,6 +209,7 @@ impl Archive { monitor: Option<&dyn ReferencedBlocksMonitor>, ) -> Result> { let monitor = monitor.unwrap_or(&NULL_MONITOR); + monitor.progress(ReferencedBlocksProgress::ReferencedBlocks { discovered: 0 }); let archive = self.clone(); let current_count = AtomicUsize::new(0); @@ -216,15 +217,15 @@ impl Archive { let result = band_ids .par_iter() .inspect(|_| { - monitor.list_referenced_blocks(current_count.fetch_add(1, Ordering::Relaxed) + 1); + monitor.progress(ReferencedBlocksProgress::ReferencedBlocks { discovered: current_count.fetch_add(1, Ordering::Relaxed) + 1 }); }) .map(move |band_id| Band::open(&archive, band_id).expect("Failed to open band")) .flat_map_iter(|band| band.index().iter_entries()) .flat_map_iter(|entry| entry.addrs) .map(|addr| addr.hash) - .collect(); + .collect::>(); - monitor.list_referenced_blocks_finished(); + monitor.progress(ReferencedBlocksProgress::ReferencedBlocksFinished { total: result.len() }); Ok(result) } @@ -269,7 +270,7 @@ impl Archive { let referenced = self.referenced_blocks(&keep_band_ids, Some(monitor_.referenced_blocks_monitor()))?; let unref = { - monitor_.find_present_blocks(0); + monitor_.progress(DeleteProgress::FindPresentBlocks { discovered: 0 }); let mut present_block_index = 0; let unref = self @@ -277,12 +278,12 @@ impl Archive { .block_names()? .inspect(|_| { present_block_index += 1; - monitor_.find_present_blocks(present_block_index); + monitor_.progress(DeleteProgress::FindPresentBlocks { discovered: present_block_index }); }) .filter(|bh| !referenced.contains(bh)) .collect_vec(); - monitor_.find_present_blocks_finished(); + monitor_.progress(DeleteProgress::FindPresentBlocksFinished { total: present_block_index }); unref }; @@ -290,21 +291,21 @@ impl Archive { stats.unreferenced_block_count = unref_count; let total_bytes = { - monitor_.measure_unreferenced_blocks(0, unref_count); + monitor_.progress(DeleteProgress::MeasureUnreferencedBlocks { current: 0, total: unref_count }); let block_index = AtomicUsize::new(0); let total_bytes = unref .par_iter() .inspect(|_| { - monitor_.measure_unreferenced_blocks( - block_index.fetch_add(1, Ordering::Relaxed) + 1, - unref_count, - ); + monitor_.progress(DeleteProgress::MeasureUnreferencedBlocks { + current: block_index.fetch_add(1, Ordering::Relaxed) + 1, + total: unref_count + }); }) .map(|block_id| block_dir.compressed_size(block_id).unwrap_or_default()) .sum(); - monitor_.measure_unreferenced_blocks_finished(); + monitor_.progress(DeleteProgress::MeasureUnreferencedBlocksFinished { total: unref_count }); total_bytes }; @@ -314,36 +315,37 @@ impl Archive { delete_guard.check()?; { + monitor_.progress(DeleteProgress::DeleteBands { current: 0, total: delete_band_ids.len() }); + let mut delete_band_count = 0; - monitor_.delete_bands(0, delete_band_ids.len()); for band_id in delete_band_ids { delete_band_count += 1; - monitor_.delete_bands(delete_band_count, delete_band_ids.len()); + monitor_.progress(DeleteProgress::DeleteBands { current: delete_band_count, total: delete_band_ids.len() }); Band::delete(self, band_id)?; stats.deleted_band_count += 1; } - monitor_.delete_bands_finished(); + monitor_.progress(DeleteProgress::DeleteBandsFinished { total: delete_band_ids.len() }); } { - monitor_.delete_blocks(0, unref.len()); + monitor_.progress(DeleteProgress::DeleteBlocks { current: 0, total: unref.len() }); let delete_block_count = AtomicUsize::new(0); let error_count = unref .par_iter() .inspect(|_| { - monitor_.delete_blocks( - delete_block_count.fetch_add(1, Ordering::Relaxed) + 1, - unref.len(), - ); + monitor_.progress(DeleteProgress::DeleteBlocks { + current: delete_block_count.fetch_add(1, Ordering::Relaxed) + 1, + total: unref.len() + }); }) .filter(|block_hash| block_dir.delete_block(block_hash).is_err()) .count(); stats.deletion_errors += error_count; stats.deleted_block_count += unref_count - error_count; - monitor_.delete_blocks_finished(); + monitor_.progress(DeleteProgress::DeleteBlocksFinished { total: unref.len() }); } } @@ -359,22 +361,21 @@ impl Archive { let monitor = monitor.unwrap_or(&NULL_MONITOR); let start = Instant::now(); - monitor.validate_archive(); + monitor.progress(ValidateProgress::ValidateArchive); let mut stats = self.validate_archive_dir(monitor)?; - monitor.validate_archive_finished(); + monitor.progress(ValidateProgress::ValidateArchiveFinished); - monitor.count_bands(); + monitor.progress(ValidateProgress::CountBands); let band_ids = self.list_band_ids()?; - monitor.count_bands_result(&band_ids); + monitor.discovered_bands(&band_ids); + monitor.progress(ValidateProgress::CountBandsFinished); // 1. Walk all indexes, collecting a list of (block_hash, min_length) // values referenced by all the indexes. - monitor.validate_bands(); let (referenced_lens, ref_stats) = validate::validate_bands(self, &band_ids, monitor); stats += ref_stats; - monitor.validate_bands_finished(); - monitor.validate_blocks(); + monitor.progress(ValidateProgress::ValidateBlocks); if options.skip_block_hashes { // 3a. Check that all referenced blocks are present, without spending time reading their // content. @@ -385,7 +386,7 @@ impl Archive { .keys() .filter(|&bh| !present_blocks.contains(bh)) { - monitor.validate_block_missing(block_hash, &BlockMissingReason::NotExisting); + monitor.block_missing(block_hash, &BlockMissingReason::NotExisting); stats.block_missing_count += 1; } } else { @@ -398,17 +399,17 @@ impl Archive { if let Some(actual_len) = block_lengths.get(&block_hash) { if referenced_len > (*actual_len as u64) { monitor - .validate_block_missing(&block_hash, &BlockMissingReason::InvalidRange); + .block_missing(&block_hash, &BlockMissingReason::InvalidRange); // TODO: A separate counter; this is worse than just being missing stats.block_missing_count += 1; } } else { - monitor.validate_block_missing(&block_hash, &BlockMissingReason::NotExisting); + monitor.block_missing(&block_hash, &BlockMissingReason::NotExisting); stats.block_missing_count += 1; } } } - monitor.validate_blocks_finished(); + monitor.progress(ValidateProgress::ValidateBlocksFinished); stats.elapsed = start.elapsed(); Ok(stats) @@ -430,7 +431,7 @@ impl Archive { Kind::Dir => dirs.push(name), Kind::File => files.push(name), other_kind => { - monitor.validate_archive_problem( + monitor.archive_problem( &ValidateArchiveProblem::UnexpectedFileType { name, kind: other_kind, @@ -440,7 +441,7 @@ impl Archive { } }, Err(source) => { - monitor.validate_archive_problem(&ValidateArchiveProblem::DirectoryListError { + monitor.archive_problem(&ValidateArchiveProblem::DirectoryListError { error: source, }); stats.io_errors += 1; @@ -451,7 +452,7 @@ impl Archive { if !files.is_empty() { // TODO: Ignore .DS_Store stats.unexpected_files += 1; - monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedFiles { + monitor.archive_problem(&ValidateArchiveProblem::UnexpectedFiles { path: format!("{:?}", self.transport), files, }); @@ -463,7 +464,7 @@ impl Archive { if let Ok(b) = d.parse() { if bs.contains(&b) { stats.structure_problems += 1; - monitor.validate_archive_problem(&ValidateArchiveProblem::DuplicateBand { + monitor.archive_problem(&ValidateArchiveProblem::DuplicateBand { path: format!("{:?}", self.transport), directory: d.clone(), }); @@ -472,7 +473,7 @@ impl Archive { } } else { stats.structure_problems += 1; - monitor.validate_archive_problem(&ValidateArchiveProblem::UnexpectedDirectory { + monitor.archive_problem(&ValidateArchiveProblem::UnexpectedDirectory { path: format!("{:?}", self.transport), directory: d.clone(), }); diff --git a/src/band.rs b/src/band.rs index 7518cb1f..06928d67 100644 --- a/src/band.rs +++ b/src/band.rs @@ -225,7 +225,7 @@ impl Band { let band_head_filename = BAND_HEAD_FILENAME.to_string(); if !files.contains(&band_head_filename) { monitor - .validate_band_problem(self, &BandProblem::MissingHeadFile { band_head_filename }); + .band_problem(self, &BandProblem::MissingHeadFile { band_head_filename }); stats.missing_band_heads += 1; } remove_item(&mut files, &BAND_HEAD_FILENAME); @@ -233,12 +233,12 @@ impl Band { remove_item(&mut dirs, &INDEX_DIR); if !files.is_empty() { - monitor.validate_band_problem(self, &BandProblem::UnexpectedFiles { files }); + monitor.band_problem(self, &BandProblem::UnexpectedFiles { files }); stats.unexpected_files += 1; } if !dirs.is_empty() { - monitor.validate_band_problem( + monitor.band_problem( self, &BandProblem::UnexpectedDirectories { directories: dirs }, ); diff --git a/src/bin/conserve/main.rs b/src/bin/conserve/main.rs index 402c8f47..91c39433 100644 --- a/src/bin/conserve/main.rs +++ b/src/bin/conserve/main.rs @@ -22,7 +22,7 @@ use clap::{Parser, StructOpt, Subcommand}; use log::{LogGuard, LoggingOptions}; use monitor::ValidateProgressModel; use monitor::{ - BackupProgressModel, DeleteProcessState, NutmegMonitor, ReferencedBlocksProgressModel, + BackupProgressModel, DeleteProcessModel, NutmegMonitor, ReferencedBlocksProgressModel, RestoreProgressModel, SizeProgressModel, }; use show::{show_diff, show_versions, ShowVersionsOptions}; @@ -353,7 +353,7 @@ impl Command { break_lock, no_stats, } => { - let monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); + let monitor = NutmegMonitor::new(DeleteProcessModel::default(), !args.no_progress); let stats = Archive::open(open_transport(archive)?)?.delete_bands( backup, &DeleteOptions { @@ -392,7 +392,7 @@ impl Command { break_lock, no_stats, } => { - let monitor = NutmegMonitor::new(DeleteProcessState::default(), !args.no_progress); + let monitor = NutmegMonitor::new(DeleteProcessModel::default(), !args.no_progress); let archive = Archive::open(open_transport(archive)?)?; let stats = archive.delete_bands( diff --git a/src/bin/conserve/monitor.rs b/src/bin/conserve/monitor.rs index 7fe3bac0..67824c9a 100644 --- a/src/bin/conserve/monitor.rs +++ b/src/bin/conserve/monitor.rs @@ -6,11 +6,11 @@ use conserve::stats::Sizes; use conserve::{ BackupMonitor, Band, BandProblem, BandValidateError, BlockLengths, BlockMissingReason, DeleteMonitor, DiffKind, Entry, IndexEntry, Kind, ReadTree, ReferencedBlocksMonitor, - RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, ValidateStats, + RestoreMonitor, Result, TreeSizeMonitor, ValidateMonitor, ValidateStats, ValidateProgress, DeleteProgress, ReferencedBlocksProgress, }; use nutmeg::{Model, View}; use thousands::Separable; -use tracing::{error, info, warn}; +use tracing::{error, info, warn, debug}; use crate::log::{self, ViewLogGuard}; @@ -62,6 +62,19 @@ impl NutmegMonitor { } } } + + fn inspect_model R, R>(&self, inspect_fn: F) -> R { + match &self.state { + NutmegMonitorState::NutmegProgress { view, .. } => { + let view = view.lock().expect("lock() should not fail"); + view.inspect_model(inspect_fn) + }, + NutmegMonitorState::NoProgress { state } => { + let mut state = state.lock().expect("lock() should not fail"); + inspect_fn(&mut *state) + } + } + } } // Considerations if we're trying properly extimate the remaining progress. @@ -148,83 +161,94 @@ impl BackupMonitor for NutmegMonitor { } } -enum ValidateProgressState { - CountBands, - ValidateBands { - bands_done: usize, - bands_total: usize, - start: Instant, - }, - ListBlocks { - discovered: usize, - }, - ReadBlocks { - total_blocks: usize, - blocks_done: usize, - bytes_done: usize, - start: Instant, - }, -} - +#[derive(Debug, Default)] pub struct ValidateProgressModel { - bands_total: Option, - state: ValidateProgressState, -} + progress: Option, -impl Default for ValidateProgressModel { - fn default() -> Self { - Self { - bands_total: None, - state: ValidateProgressState::CountBands {}, - } - } + bands_validated: usize, + bands_start: Option, + + read_blocks_count: usize, + read_blocks_bytes: usize, } impl nutmeg::Model for ValidateProgressModel { fn render(&mut self, _width: usize) -> String { - match &self.state { - ValidateProgressState::CountBands => "Counting bands".to_string(), - ValidateProgressState::ValidateBands { - bands_done, - bands_total, - start, - } => { - format!( - "Check index {}/{}, {} done, {} remaining", - bands_done, - bands_total, - nutmeg::percent_done(*bands_done, *bands_total), - nutmeg::estimate_remaining(start, *bands_done, *bands_total) - ) - } - ValidateProgressState::ListBlocks { discovered } => { - format!("Listing blocks ({} blocks discovered)", discovered) - } - ValidateProgressState::ReadBlocks { - total_blocks, - blocks_done, - bytes_done, - start, - } => { + let state = match &self.progress { + Some(state) => state, + None => return "Validating, please wait...".to_string() + }; + + match state { + ValidateProgress::CountBands => "Counting bands".to_string(), + ValidateProgress::CountBandsFinished => "Finished counting bands".to_string(), + + ValidateProgress::ValidateArchive => "Validating archive integrity".to_string(), + ValidateProgress::ValidateArchiveFinished => "Finished validating archive integrity".to_string(), + + ValidateProgress::ValidateBlocks => format!("Validating blocks"), + ValidateProgress::ValidateBlocksFinished => format!("Blocks validated"), + + ValidateProgress::ValidateBands { current, total } => format!("Validating band {}/{}", current, total), + ValidateProgress::ValidateBandsFinished { total } => format!("{} bands validated", total), + + ValidateProgress::ListBlockNames { discovered } => format!("Listing blocks ({} blocks discovered)", discovered), + ValidateProgress::ListBlockNamesFinished { total } => format!("Discovered {} blocks", total), + + ValidateProgress::BlockRead { total, .. } => { + // Note: We're using our own read block counter (`read_blocks_count`) since the current argument in ValidateProgress::BlockRead + // is not garanteed to be in sequential due to multithreading. + let start = self.bands_start.get_or_insert_with(|| Instant::now()); format!( "Check block {}/{}: {} done, {} MB checked, {} remaining", - *blocks_done, - *total_blocks, - nutmeg::percent_done(*blocks_done, *total_blocks), - *bytes_done / 1_000_000, - nutmeg::estimate_remaining(start, *blocks_done, *total_blocks) + self.read_blocks_count, + total, + nutmeg::percent_done(self.read_blocks_count, *total), + self.read_blocks_bytes / 1_000_000, + nutmeg::estimate_remaining(start, self.read_blocks_count, *total) ) - } + }, + ValidateProgress::BlockReadFinished { total } => format!("Finished reading {} blocks", total), } } } impl ValidateMonitor for NutmegMonitor { - fn validate_archive(&self) { - info!("Check archive top-level directory..."); + fn progress(&self, state: ValidateProgress) { + match &state { + ValidateProgress::CountBands => info!("Count bands..."), + ValidateProgress::ValidateArchive => info!("Check archive top-level directory..."), + ValidateProgress::ListBlockNames { discovered } => { + if *discovered == 0 { + info!("Count blocks..."); + } + }, + ValidateProgress::ValidateBands { .. } => { + self.update_model(|state| { + if state.bands_start.is_none() { + state.bands_start = Some(Instant::now()); + } + }); + }, + ValidateProgress::ValidateBandsFinished { .. } => { + let start = self.inspect_model( + |state| state.bands_start.unwrap_or(Instant::now()) + ); + info!("Finished validating bands in {:#?}.", start.elapsed()); + }, + ValidateProgress::BlockRead { current, .. } => { + if *current == 0 { + info!("Check {} blocks...", current.separate_with_commas()); + } + } + _ => {} + } + + debug!("{:?}", &state); + self.update_model(|model| model.progress = Some(state)); } - fn validate_archive_problem(&self, problem: &ValidateArchiveProblem) { + fn archive_problem(&self, problem: &ValidateArchiveProblem) { match problem { ValidateArchiveProblem::UnexpectedFileType { name, kind } => { error!( @@ -250,28 +274,11 @@ impl ValidateMonitor for NutmegMonitor { } } - fn count_bands(&self) { - info!("Count bands..."); - } - - fn count_bands_result(&self, bands: &[conserve::BandId]) { + fn discovered_bands(&self, bands: &[conserve::BandId]) { info!("Checking {} bands...", bands.len()); - - self.update_model(|model| model.bands_total = Some(bands.len())); } - fn validate_bands(&self) { - self.update_model(|model| { - let bands_total = model.bands_total.expect("bands have been counted"); - model.state = ValidateProgressState::ValidateBands { - bands_done: 0, - bands_total, - start: Instant::now(), - }; - }); - } - - fn validate_band_problem(&self, band: &Band, problem: &conserve::BandProblem) { + fn band_problem(&self, band: &Band, problem: &conserve::BandProblem) { match problem { BandProblem::MissingHeadFile { .. } => { warn!("No band head file in {:?}", band.transport()) @@ -289,80 +296,28 @@ impl ValidateMonitor for NutmegMonitor { } } - fn validate_band_result( + fn band_validate_result( &self, _band_id: &conserve::BandId, _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, ) { self.update_model(|model| { - if let ValidateProgressState::ValidateBands { bands_done, .. } = &mut model.state { - *bands_done += 1; - } else { - panic!("Expected state ValidateProgressState::ValidateBands"); - } - }); - } - - fn validate_bands_finished(&self) { - // We can't use logging while locked_view is held since we would deadlock. - let elapsed = self.update_model(|model| { - if let ValidateProgressState::ValidateBands { start, .. } = &mut model.state { - start.elapsed() - } else { - panic!("Expected state ValidateProgressState::ValidateBands"); - } + model.bands_validated += 1; }); - - info!("Finished validating bands in {:#?}.", elapsed); } - fn list_block_names(&self, current_count: usize) { - if current_count == 0 { - info!("Count blocks..."); - } - + fn block_read_result(&self, _block_hash: &conserve::BlockHash, result: &Result) { self.update_model(|model| { - model.state = ValidateProgressState::ListBlocks { - discovered: current_count, - } - }); - } - - fn read_blocks(&self, count: usize) { - info!("Check {} blocks...", count.separate_with_commas()); - - self.update_model(|model| { - model.state = ValidateProgressState::ReadBlocks { - total_blocks: count, - blocks_done: 0, - bytes_done: 0, - start: Instant::now(), - } - }); - } - - fn read_block_result(&self, _block_hash: &conserve::BlockHash, result: &Result) { - self.update_model(|model| { - if let ValidateProgressState::ReadBlocks { - blocks_done, - bytes_done, - .. - } = &mut model.state - { - if let Ok(sizes) = result { - *bytes_done += sizes.uncompressed as usize; - } else { - // TODO: Add a fail counter. - } - - *blocks_done += 1; + model.read_blocks_count += 1; + if let Ok(sizes) = result { + model.read_blocks_bytes += sizes.uncompressed as usize; } else { - panic!("Expected state ValidateProgressState::ReadBlocks"); + // TODO: Add a fail counter. } }); } - fn validate_block_missing( + fn block_missing( &self, block_hash: &conserve::BlockHash, reason: &conserve::BlockMissingReason, @@ -398,90 +353,62 @@ impl TreeSizeMonitor for NutmegMonitor { } } -pub enum DeleteProcessState { - ListReferencedBlocks { count: usize }, - FindPresentBlocks { count: usize }, - MeasureUnreferencedBlocks { count: usize, target: usize }, - DeleteBands { count: usize, target: usize }, - DeleteBlocks { count: usize, target: usize }, +// ReferencedBlocksProgress + +pub enum DeleteProcessModel { + Unset, + List(ReferencedBlocksProgress), + Delete(DeleteProgress), } -impl Default for DeleteProcessState { +impl Default for DeleteProcessModel { fn default() -> Self { - DeleteProcessState::ListReferencedBlocks { count: 0 } + Self::Unset } } -impl Model for DeleteProcessState { + +impl Model for DeleteProcessModel { fn render(&mut self, _width: usize) -> String { match self { - DeleteProcessState::ListReferencedBlocks { count } => { - format!("Find referenced blocks in band ({} discovered)", count) - } - DeleteProcessState::FindPresentBlocks { count } => { - format!("Find present blocks ({} discovered)", count) - } - DeleteProcessState::MeasureUnreferencedBlocks { count, target } => { - format!("Measure unreferenced blocks ({}/{})", count, target) - } - DeleteProcessState::DeleteBands { count, target } => { - format!("Delete bands ({}/{})", count, target) - } - DeleteProcessState::DeleteBlocks { count, target } => { - format!("Delete blocks ({}/{})", count, target) - } + Self::List(state) => { + match state { + ReferencedBlocksProgress::ReferencedBlocks { discovered } => format!("Find referenced blocks in band ({} discovered)", discovered), + ReferencedBlocksProgress::ReferencedBlocksFinished { total } => format!("Discovered {} referenced blocks in band", total) + } + }, + Self::Delete(state) => { + match state { + DeleteProgress::FindPresentBlocks { discovered } => format!("Find present blocks ({} discovered)", discovered), + DeleteProgress::FindPresentBlocksFinished { total } => format!("Found {} present blocks", total), + + DeleteProgress::MeasureUnreferencedBlocks { current, total } => format!("Measure unreferenced blocks ({}/{})", current, total), + DeleteProgress::MeasureUnreferencedBlocksFinished { .. } => format!("Measured unreferenced blocks"), + + DeleteProgress::DeleteBands { current, total } => format!("Delete bands ({}/{})", current, total), + DeleteProgress::DeleteBandsFinished { total } => format!("Deleted {} bands", total), + + DeleteProgress::DeleteBlocks { current, total } => format!("Delete blocks ({}/{})", current, total), + DeleteProgress::DeleteBlocksFinished { total } => format!("Deleted {} blocks", total) + } + }, + Self::Unset => "Deleting, please wait...".to_string() } } } -impl DeleteMonitor for NutmegMonitor { +impl DeleteMonitor for NutmegMonitor { fn referenced_blocks_monitor(&self) -> &dyn conserve::ReferencedBlocksMonitor { self } - - fn find_present_blocks(&self, current_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::FindPresentBlocks { - count: current_count, - }; - }); - } - - fn measure_unreferenced_blocks(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::MeasureUnreferencedBlocks { - count: current_count, - target: target_count, - }; - }); - } - - fn delete_bands(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::DeleteBands { - count: current_count, - target: target_count, - }; - }); - } - - fn delete_blocks(&self, current_count: usize, target_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::DeleteBlocks { - count: current_count, - target: target_count, - }; - }); + fn progress(&self, state: DeleteProgress) { + self.update_model(|model| *model = DeleteProcessModel::Delete(state)); } } -impl ReferencedBlocksMonitor for NutmegMonitor { - fn list_referenced_blocks(&self, current_count: usize) { - self.update_model(|view| { - *view = DeleteProcessState::ListReferencedBlocks { - count: current_count, - }; - }); +impl ReferencedBlocksMonitor for NutmegMonitor { + fn progress(&self, state: ReferencedBlocksProgress) { + self.update_model(|model| *model = DeleteProcessModel::List(state)); } } @@ -532,17 +459,25 @@ impl RestoreMonitor for NutmegMonitor { #[derive(Default)] pub struct ReferencedBlocksProgressModel { - count: usize, + progress: Option, } impl Model for ReferencedBlocksProgressModel { fn render(&mut self, _width: usize) -> String { - format!("Find referenced blocks in band ({} discovered)", self.count) + let state = match &self.progress { + Some(state) => state, + None => return "Listing referenced blocks, please wait...".to_string() + }; + + match state { + ReferencedBlocksProgress::ReferencedBlocks { discovered } => format!("Find referenced blocks in band ({} discovered)", discovered), + ReferencedBlocksProgress::ReferencedBlocksFinished { total } => format!("Discovered {} referenced blocks in band", total) + } } } impl ReferencedBlocksMonitor for NutmegMonitor { - fn list_referenced_blocks(&self, current_count: usize) { - self.update_model(|model| model.count = current_count); + fn progress(&self, state: ReferencedBlocksProgress) { + self.update_model(|model| model.progress = Some(state)); } } diff --git a/src/blockdir.rs b/src/blockdir.rs index 26cc9b64..4b7f84c2 100644 --- a/src/blockdir.rs +++ b/src/blockdir.rs @@ -26,6 +26,7 @@ use std::convert::TryInto; use std::io; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use blake2_rfc::blake2b; use blake2_rfc::blake2b::Blake2b; @@ -36,6 +37,7 @@ use tracing::{error, warn}; use crate::blockhash::BlockHash; use crate::compress::snappy::{Compressor, Decompressor}; use crate::kind::Kind; +use crate::monitor::ValidateProgress; use crate::stats::{BackupStats, Sizes, ValidateStats}; use crate::transport::local::LocalTransport; use crate::transport::{DirEntry, ListDirNames, Transport}; @@ -245,16 +247,16 @@ impl BlockDir { pub fn block_names_set(&self, monitor: &dyn ValidateMonitor) -> Result> { let mut block_count = 0usize; - monitor.list_block_names(block_count); + monitor.progress(ValidateProgress::ListBlockNames { discovered: block_count }); let result = self .iter_block_dir_entries()? .filter_map(|de| de.name.parse().ok()) .inspect(|_| { block_count += 1; - monitor.list_block_names(block_count); + monitor.progress(ValidateProgress::ListBlockNames { discovered: block_count }); }) .collect(); - monitor.list_block_names_finished(); + monitor.progress(ValidateProgress::ListBlockNamesFinished { total: block_count }); Ok(result) } @@ -271,7 +273,9 @@ impl BlockDir { // directories of the right length. // TODO: Test having a block with the right compression but the wrong contents. let blocks = self.block_names_set(monitor)?; - monitor.read_blocks(blocks.len()); + let block_count_read = AtomicUsize::new(0); + let block_count = blocks.len(); + monitor.progress(ValidateProgress::BlockRead { current: 0, total: block_count }); stats.block_read_count = blocks.len().try_into().unwrap(); @@ -279,15 +283,18 @@ impl BlockDir { // failed, where the usize gives the uncompressed data size. let results: Vec> = blocks .into_par_iter() - .map(|hash| { + .map(move |hash| { let result = self.get_block_content(&hash); + let read_count = block_count_read.fetch_add(1, Ordering::Relaxed) + 1; + monitor.progress(ValidateProgress::BlockRead { current: read_count, total: block_count }); + match result { Ok((bytes, sizes)) => { - monitor.read_block_result(&hash, &Ok(sizes)); + monitor.block_read_result(&hash, &Ok(sizes)); Some((hash, bytes.len())) } Err(error) => { - monitor.read_block_result(&hash, &Err(error)); + monitor.block_read_result(&hash, &Err(error)); None } } @@ -301,7 +308,7 @@ impl BlockDir { .flatten() // keep only Some values .collect(); - monitor.read_blocks_finished(); + monitor.progress(ValidateProgress::BlockReadFinished { total: block_count }); Ok(len_map) } diff --git a/src/lib.rs b/src/lib.rs index 7b78f777..6445053d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,7 +67,7 @@ pub use crate::merge::{MergeTrees, MergedEntryKind}; pub use crate::misc::bytes_to_human_mb; pub use crate::monitor::{ BackupMonitor, DeleteMonitor, ReferencedBlocksMonitor, RestoreMonitor, TreeSizeMonitor, - ValidateMonitor, + ValidateMonitor, ValidateProgress, DeleteProgress, ReferencedBlocksProgress }; pub use crate::restore::{restore, RestoreOptions, RestoreTree}; pub use crate::stats::{BackupStats, DeleteStats, RestoreStats, ValidateStats}; diff --git a/src/monitor.rs b/src/monitor.rs index 20bcc4e8..d458e56b 100644 --- a/src/monitor.rs +++ b/src/monitor.rs @@ -13,37 +13,44 @@ pub trait BackupMonitor { fn finished(&self, _stats: &BackupStats) {} } +#[derive(Debug, Clone)] +pub enum ValidateProgress { + CountBands, + CountBandsFinished, + + ValidateArchive, + ValidateArchiveFinished, + + ValidateBands { current: usize, total: usize }, + ValidateBandsFinished { total: usize }, + + ListBlockNames { discovered: usize }, + ListBlockNamesFinished { total: usize }, + + BlockRead { current: usize, total: usize }, + BlockReadFinished { total: usize }, + + ValidateBlocks, + ValidateBlocksFinished, +} + + /// Monitor the validation progress. pub trait ValidateMonitor: Sync { - fn count_bands(&self) {} - fn count_bands_result(&self, _bands: &[BandId]) {} - - fn validate_archive(&self) {} - fn validate_archive_problem(&self, _problem: &ValidateArchiveProblem) {} - fn validate_archive_finished(&self) {} + /// Will be called with the current state of validating the target archive. + fn progress(&self, _state: ValidateProgress) {} - fn validate_bands(&self) {} - fn validate_bands_finished(&self) {} + fn discovered_bands(&self, _bands: &[BandId]) {} - fn validate_band(&self, _band_id: &BandId) {} - fn validate_band_problem(&self, _band: &Band, _problem: &BandProblem) {} - fn validate_band_result( + fn archive_problem(&self, _problem: &ValidateArchiveProblem) {} + fn band_problem(&self, _band: &Band, _problem: &BandProblem) {} + fn band_validate_result( &self, _band_id: &BandId, _result: &std::result::Result<(BlockLengths, ValidateStats), BandValidateError>, - ) { - } - - fn validate_block_missing(&self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} - fn validate_blocks(&self) {} - fn validate_blocks_finished(&self) {} - - fn list_block_names(&self, _current_count: usize) {} - fn list_block_names_finished(&self) {} - - fn read_blocks(&self, _count: usize) {} - fn read_block_result(&self, _block_hash: &BlockHash, _result: &Result) {} - fn read_blocks_finished(&self) {} + ) { } + fn block_missing(&self, _block_hash: &BlockHash, _reason: &BlockMissingReason) {} + fn block_read_result(&self, _block_hash: &BlockHash, _result: &Result) {} } /// Monitor for iterating trees. @@ -51,27 +58,38 @@ pub trait TreeSizeMonitor { fn entry_discovered(&self, _entry: &T::Entry, _size: &Option) {} } +#[derive(Debug, Clone)] +pub enum ReferencedBlocksProgress { + ReferencedBlocks { discovered: usize }, + ReferencedBlocksFinished { total: usize } +} + /// Monitor for iterating referenced blocks. pub trait ReferencedBlocksMonitor: Sync { - fn list_referenced_blocks(&self, _current_count: usize) {} - fn list_referenced_blocks_finished(&self) {} + fn progress(&self, _state: ReferencedBlocksProgress) {} } -/// Monitor for deleting backups/blocks. -pub trait DeleteMonitor: Sync { - fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor; +#[derive(Debug, Clone)] +pub enum DeleteProgress { + FindPresentBlocks { discovered: usize }, + FindPresentBlocksFinished { total: usize }, - fn find_present_blocks(&self, _current_count: usize) {} - fn find_present_blocks_finished(&self) {} + MeasureUnreferencedBlocks { current: usize, total: usize }, + MeasureUnreferencedBlocksFinished { total: usize }, - fn measure_unreferenced_blocks(&self, _current_count: usize, _target_count: usize) {} - fn measure_unreferenced_blocks_finished(&self) {} + DeleteBands { current: usize, total: usize }, + DeleteBandsFinished { total: usize }, - fn delete_bands(&self, _current_count: usize, _target_count: usize) {} - fn delete_bands_finished(&self) {} + DeleteBlocks { current: usize, total: usize }, + DeleteBlocksFinished { total: usize }, +} + +/// Monitor for deleting backups/blocks. +pub trait DeleteMonitor: Sync { + fn referenced_blocks_monitor(&self) -> &dyn ReferencedBlocksMonitor; - fn delete_blocks(&self, _current_count: usize, _target_count: usize) {} - fn delete_blocks_finished(&self) {} + fn progress(&self, _state: DeleteProgress) {} + // TODO: May encountered delete errors? } /// Monitor the progress of restoring files. diff --git a/src/validate.rs b/src/validate.rs index c39748d9..42cdb740 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use crate::blockdir::Address; use crate::*; +use crate::monitor::ValidateProgress; pub struct BlockLengths(pub HashMap); @@ -85,11 +86,12 @@ pub(crate) fn validate_bands( ) -> (BlockLengths, ValidateStats) { let mut stats = ValidateStats::default(); let mut block_lens = BlockLengths::new(); + let mut band_index = 0; for band_id in band_ids { - monitor.validate_band(band_id); + monitor.progress(ValidateProgress::ValidateBands { current: band_index, total: band_ids.len() }); let result = validate_band(archive, &mut stats, band_id, monitor); - monitor.validate_band_result(band_id, &result); + monitor.band_validate_result(band_id, &result); match result { Ok((st_block_lens, st_stats)) => { @@ -103,8 +105,10 @@ pub(crate) fn validate_bands( BandValidateError::TreeValidateError(_) => stats.tree_validate_errors += 1, }, } + band_index += 1; } + monitor.progress(ValidateProgress::ValidateBandsFinished { total: band_ids.len() }); (block_lens, stats) } From df9092887bb3b74381192e8d5058eeaf12d4881b Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 5 Jan 2023 01:10:43 +0100 Subject: [PATCH 36/39] Fixing unix builds --- src/restore.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/restore.rs b/src/restore.rs index 30093aba..0c398714 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -19,10 +19,10 @@ use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; use filetime::set_file_handle_times; +use tracing::{debug, warn}; #[cfg(unix)] -use filetime::set_symlink_file_times; -use tracing::{debug, warn}; +use nix::unistd; use crate::band::BandSelectionPolicy; use crate::entry::Entry; @@ -223,7 +223,7 @@ impl RestoreTree { .unix_mode() .set_permissions(&path) .map_err(|e| { - ui::show_error(&e); + error!("Failed to set permissions on {path}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); @@ -244,7 +244,7 @@ impl RestoreTree { // TODO: use `std::os::unix::fs::chown(path, uid, gid)?;` once stable unistd::chown(&path, uid_opt, gid_opt) .map_err(|e| { - ui::show_error(&e); + error!("Failed to change owner on {path}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); From 6f4097a7f2dabd611156e8f9b82ff5893ba709c0 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 5 Jan 2023 01:29:27 +0100 Subject: [PATCH 37/39] Fixed unix tests --- src/restore.rs | 10 ++++++---- tests/api/restore.rs | 16 ++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/restore.rs b/src/restore.rs index 0c398714..dc301b6c 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; use filetime::set_file_handle_times; -use tracing::{debug, warn}; +use tracing::{debug, warn, error}; #[cfg(unix)] use nix::unistd; @@ -167,7 +167,7 @@ impl RestoreTree { #[cfg(unix)] for (path, unix_mode) in self.dir_unix_modes { if let Err(err) = unix_mode.set_permissions(path) { - ui::problem(&format!("Failed to set directory permissions: {err:?}")); + error!("Failed to set directory permissions: {err:?}"); } } for (path, time) in self.dir_mtimes { @@ -223,7 +223,7 @@ impl RestoreTree { .unix_mode() .set_permissions(&path) .map_err(|e| { - error!("Failed to set permissions on {path}: {}", ui::format_error_causes(&e)); + error!("Failed to set permissions on {path:?}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); @@ -244,7 +244,7 @@ impl RestoreTree { // TODO: use `std::os::unix::fs::chown(path, uid, gid)?;` once stable unistd::chown(&path, uid_opt, gid_opt) .map_err(|e| { - error!("Failed to change owner on {path}: {}", ui::format_error_causes(&e)); + error!("Failed to change owner on {path:?}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); @@ -257,6 +257,8 @@ impl RestoreTree { #[cfg(unix)] fn copy_symlink(&mut self, entry: &E) -> Result<()> { use std::os::unix::fs as unix_fs; + use filetime::set_symlink_file_times; + if let Some(ref target) = entry.symlink_target() { let path = self.rooted_path(entry.apath()); if let Err(source) = unix_fs::symlink(target, &path) { diff --git a/tests/api/restore.rs b/tests/api/restore.rs index eab8a02e..15b8dbe0 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -12,16 +12,8 @@ //! Tests focussed on restore. -#[cfg(unix)] -use std::fs::{read_link, symlink_metadata}; -use std::path::PathBuf; - -use filetime::{set_symlink_file_times, FileTime}; -use tempfile::TempDir; - use conserve::test_fixtures::ScratchArchive; use conserve::test_fixtures::TreeFixture; -use conserve::unix_time::UnixTime; use conserve::*; #[test] @@ -119,6 +111,14 @@ fn exclude_files() { #[test] #[cfg(unix)] fn restore_symlink() { + use std::path::PathBuf; + use std::fs::{read_link, symlink_metadata}; + + use tempfile::TempDir; + use filetime::{set_symlink_file_times, FileTime}; + + use conserve::unix_time::UnixTime; + let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); From 5131148b70d424990f1c45794cbc6545d290aef7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 5 Jan 2023 01:29:27 +0100 Subject: [PATCH 38/39] Using raw output for unix_permissions.rx test --- src/restore.rs | 10 ++++++---- tests/api/restore.rs | 16 ++++++++-------- tests/cli/unix_permissions.rs | 16 +++++++++------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/restore.rs b/src/restore.rs index 0c398714..dc301b6c 100644 --- a/src/restore.rs +++ b/src/restore.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf}; use std::{fs, time::Instant}; use filetime::set_file_handle_times; -use tracing::{debug, warn}; +use tracing::{debug, warn, error}; #[cfg(unix)] use nix::unistd; @@ -167,7 +167,7 @@ impl RestoreTree { #[cfg(unix)] for (path, unix_mode) in self.dir_unix_modes { if let Err(err) = unix_mode.set_permissions(path) { - ui::problem(&format!("Failed to set directory permissions: {err:?}")); + error!("Failed to set directory permissions: {err:?}"); } } for (path, time) in self.dir_mtimes { @@ -223,7 +223,7 @@ impl RestoreTree { .unix_mode() .set_permissions(&path) .map_err(|e| { - error!("Failed to set permissions on {path}: {}", ui::format_error_causes(&e)); + error!("Failed to set permissions on {path:?}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); @@ -244,7 +244,7 @@ impl RestoreTree { // TODO: use `std::os::unix::fs::chown(path, uid, gid)?;` once stable unistd::chown(&path, uid_opt, gid_opt) .map_err(|e| { - error!("Failed to change owner on {path}: {}", ui::format_error_causes(&e)); + error!("Failed to change owner on {path:?}: {}", ui::format_error_causes(&e)); stats.errors += 1; }) .ok(); @@ -257,6 +257,8 @@ impl RestoreTree { #[cfg(unix)] fn copy_symlink(&mut self, entry: &E) -> Result<()> { use std::os::unix::fs as unix_fs; + use filetime::set_symlink_file_times; + if let Some(ref target) = entry.symlink_target() { let path = self.rooted_path(entry.apath()); if let Err(source) = unix_fs::symlink(target, &path) { diff --git a/tests/api/restore.rs b/tests/api/restore.rs index eab8a02e..15b8dbe0 100644 --- a/tests/api/restore.rs +++ b/tests/api/restore.rs @@ -12,16 +12,8 @@ //! Tests focussed on restore. -#[cfg(unix)] -use std::fs::{read_link, symlink_metadata}; -use std::path::PathBuf; - -use filetime::{set_symlink_file_times, FileTime}; -use tempfile::TempDir; - use conserve::test_fixtures::ScratchArchive; use conserve::test_fixtures::TreeFixture; -use conserve::unix_time::UnixTime; use conserve::*; #[test] @@ -119,6 +111,14 @@ fn exclude_files() { #[test] #[cfg(unix)] fn restore_symlink() { + use std::path::PathBuf; + use std::fs::{read_link, symlink_metadata}; + + use tempfile::TempDir; + use filetime::{set_symlink_file_times, FileTime}; + + use conserve::unix_time::UnixTime; + let af = ScratchArchive::new(); let srcdir = TreeFixture::new(); diff --git a/tests/cli/unix_permissions.rs b/tests/cli/unix_permissions.rs index af5d5071..c7538742 100644 --- a/tests/cli/unix_permissions.rs +++ b/tests/cli/unix_permissions.rs @@ -19,6 +19,7 @@ fn backup_unix_permissions() { // conserve init run_conserve() + .arg("-R") .arg("init") .arg(&arch_dir) .assert() @@ -70,7 +71,7 @@ fn backup_unix_permissions() { // backup run_conserve() - .args(["backup", "-v", "-l"]) + .args(["-R", "backup", "-v", "-l"]) .arg(&arch_dir) .arg(&data_dir) .assert() @@ -84,7 +85,7 @@ fn backup_unix_permissions() { // verify file permissions in stored archive run_conserve() - .args(["ls", "-l"]) + .args(["-R", "ls", "-l"]) .arg(&arch_dir) .assert() .success() @@ -101,7 +102,7 @@ fn backup_unix_permissions() { // verify permissions are restored correctly run_conserve() - .args(["restore", "-v", "-l"]) + .args(["-R", "restore", "-v", "-l"]) .arg(&arch_dir) .arg(&*restore_dir) .assert() @@ -126,6 +127,7 @@ fn backup_user_and_permissions() { // conserve init run_conserve() + .arg("-R") .arg("init") .arg(&arch_dir) .assert() @@ -175,7 +177,7 @@ fn backup_user_and_permissions() { // verify ls command run_conserve() - .args(["ls", "-l", "--source"]) + .args(["-R", "ls", "-l", "--source"]) .arg(&src) .assert() .success() @@ -184,7 +186,7 @@ fn backup_user_and_permissions() { // backup run_conserve() - .args(["backup"]) + .args(["-R", "backup"]) .arg(&arch_dir) .arg(&src) .assert() @@ -196,7 +198,7 @@ fn backup_user_and_permissions() { // restore run_conserve() - .args(["restore", "-v", "-l", "--no-progress"]) + .args(["-R", "restore", "-v", "-l", "--no-progress"]) .arg(&arch_dir) .arg(restore_dir.path()) .assert() @@ -240,7 +242,7 @@ fn backup_user_and_permissions() { fn list_testdata_with_permissions() { let archive_path = Path::new("testdata/archive/minimal/v0.6.17"); run_conserve() - .args(["ls", "-l"]) + .args(["-R", "ls", "-l"]) .arg(archive_path) .assert() .success() From 426b4e39476b1a3ee9a57279d034c0c2e4215207 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 5 Jan 2023 01:41:39 +0100 Subject: [PATCH 39/39] Removing duplicate new line for restore file list --- src/bin/conserve/monitor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/conserve/monitor.rs b/src/bin/conserve/monitor.rs index 717721d9..a53bbe50 100644 --- a/src/bin/conserve/monitor.rs +++ b/src/bin/conserve/monitor.rs @@ -502,7 +502,7 @@ impl RestoreMonitor for NutmegMonitor { FileListVerbosity::NameOnly => info!("{}", entry.apath()), FileListVerbosity::Full => { info!( - "{} {} {}\n", + "{} {} {}", entry.unix_mode(), entry.owner(), entry.apath()