Added a download timestamp
This commit is contained in:
@@ -589,6 +589,18 @@ pub async fn download_dump(
|
|||||||
let url = format!("{DUMP_BASE_URL}{timestamp}/{filename}");
|
let url = format!("{DUMP_BASE_URL}{timestamp}/{filename}");
|
||||||
let target_path = target_dir.join(filename);
|
let target_path = target_dir.join(filename);
|
||||||
|
|
||||||
|
// Skip if we already have this file from the same dump timestamp
|
||||||
|
let stamp_path = target_dir.join(format!("{filename}.timestamp"));
|
||||||
|
if target_path.exists() {
|
||||||
|
if let Ok(existing_stamp) = std::fs::read_to_string(&stamp_path) {
|
||||||
|
if existing_stamp.trim() == timestamp {
|
||||||
|
progress(&format!("Skipping {filename} (already downloaded from {timestamp})"));
|
||||||
|
tracing::info!(file = %filename, timestamp = %timestamp, "dump file already up to date, skipping download");
|
||||||
|
return Ok(target_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
progress(&format!("Downloading {filename}..."));
|
progress(&format!("Downloading {filename}..."));
|
||||||
tracing::info!(url = %url, target = %target_path.display(), "downloading MB dump");
|
tracing::info!(url = %url, target = %target_path.display(), "downloading MB dump");
|
||||||
|
|
||||||
@@ -634,6 +646,9 @@ pub async fn download_dump(
|
|||||||
"download complete"
|
"download complete"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Write timestamp marker so we can skip this file on re-runs
|
||||||
|
let _ = std::fs::write(&stamp_path, timestamp);
|
||||||
|
|
||||||
Ok(target_path)
|
Ok(target_path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user