Skip to content

Commit

Permalink
optimize file read in Config::verify
Browse files Browse the repository at this point in the history
`Config::verify` refactored to improve the efficiency and
memory usage of file hashing.

Signed-off-by: onur-ozkan <work@onurozkan.dev>
  • Loading branch information
onur-ozkan committed Oct 11, 2023
1 parent 7ed044c commit d32a733
Showing 1 changed file with 27 additions and 10 deletions.
37 changes: 27 additions & 10 deletions src/bootstrap/download.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::{
env,
ffi::{OsStr, OsString},
fs::{self, File},
io::{BufRead, BufReader, BufWriter, ErrorKind, Write},
io::{BufRead, BufReader, BufWriter, ErrorKind, Read, Write},
path::{Path, PathBuf},
process::{Command, Stdio},
};
Expand Down Expand Up @@ -324,21 +324,38 @@ impl Config {
use sha2::Digest;

self.verbose(&format!("verifying {}", path.display()));

if self.dry_run() {
return false;
}

let mut hasher = sha2::Sha256::new();
// FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
// Consider using streaming IO instead?
let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) };
hasher.update(&contents);
let found = hex::encode(hasher.finalize().as_slice());
let verified = found == expected;
if !verified && !self.dry_run() {

let file = t!(File::open(path));
let mut reader = BufReader::new(file);
let mut buffer = [0; 4096]; // read in chunks of 4KB

loop {
let read_len = t!(reader.read(&mut buffer));
// break if EOF
if read_len == 0 {
break;
}
hasher.update(&buffer[0..read_len]);
}

let checksum = hex::encode(hasher.finalize().as_slice());
let verified = checksum == expected;

if !verified {
println!(
"invalid checksum: \n\
found: {found}\n\
found: {checksum}\n\
expected: {expected}",
);
}
return verified;

verified
}
}

Expand Down

0 comments on commit d32a733

Please sign in to comment.