Properly use blake3 mmap

Cargo.lock

@@ -170,6 +170,8 @@ dependencies = [
  "cc",
  "cfg-if",
  "constant_time_eq",
+ "memmap2",
+ "rayon-core",
  "serde",
 ]
 
@@ -266,6 +268,31 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
 [[package]]
 name = "darling"
 version = "0.20.10"
@@ -790,6 +817,15 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "memmap2"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -1055,6 +1091,16 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.7"

Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 bincode = { version = "2.0.0-rc.3", features = ["serde"] }
-blake3 = { version = "1.5.4", features = ["serde"] }
+blake3 = { version = "1.5.4", features = ["mmap", "rayon", "serde"] }
 chrono = { version = "0.4.38", features = ["serde"] }
 log = "0.4"
 maud = { version = "0.26", features = ["rocket"] }

src/database.rs

@@ -57,9 +57,6 @@ pub struct MochiFile {
     /// The original name of the file
     name: String,
 
-    /// Size of the file in bytes
-    size: usize,
-
     /// The location on disk (for deletion and management)
     filename: PathBuf,
 
@@ -80,7 +77,6 @@ impl MochiFile {
     /// Create a new file that expires in `expiry`.
     pub fn new_with_expiry(
         name: &str,
-        size: usize,
         hash: Hash,
         filename: PathBuf,
         expire_duration: TimeDelta
@@ -90,7 +86,6 @@ impl MochiFile {
 
         Self {
             name: name.to_string(),
-            size,
             filename,
             hash,
             upload_datetime: current,

src/main.rs

@@ -8,7 +8,7 @@ use chrono::{DateTime, TimeDelta, Utc};
 use database::{clean_loop, Database, MochiFile};
 use log::info;
 use rocket::{
-    data::{Limits, ToByteUnit}, form::Form, fs::{FileServer, Options, TempFile}, get, http::{ContentType, RawStr}, post, response::{content::{RawCss, RawJavaScript}, status::NotFound, Redirect}, routes, serde::{json::Json, Serialize}, tokio::{self, fs::File, io::AsyncReadExt}, Config, FromForm, State
+    data::{Limits, ToByteUnit}, form::Form, fs::{FileServer, Options, TempFile}, get, http::{ContentType, RawStr}, post, response::{content::{RawCss, RawJavaScript}, status::NotFound, Redirect}, routes, serde::{json::Json, Serialize}, tokio, Config, FromForm, State
 };
 use settings::Settings;
 use strings::{parse_time_string, to_pretty_time};
@@ -155,14 +155,13 @@ async fn handle_upload(
 
     let filename = get_id(
         raw_name,
-        hash.0
+        hash
     );
     out_path.push(filename.clone());
 
     let constructed_file = MochiFile::new_with_expiry(
         raw_name,
-        hash.1,
+        hash,
-        hash.0,
         out_path.clone(),
         expire_time
     );
@@ -175,8 +174,8 @@ async fn handle_upload(
             status: true,
             response: "File already exists",
             name: constructed_file.name().clone(),
-            url: "files/".to_string() + &filename,
+            url: filename,
-            hash: hash.0.to_hex()[0..10].to_string(),
+            hash: hash.to_hex()[0..10].to_string(),
             expires: Some(constructed_file.get_expiry()),
             ..Default::default()
         }))
@@ -191,8 +190,8 @@ async fn handle_upload(
     Ok(Json(ClientResponse {
         status: true,
         name: constructed_file.name().clone(),
-        url: "files/".to_string() + &filename,
+        url: filename,
-        hash: hash.0.to_hex()[0..10].to_string(),
+        hash: hash.to_hex()[0..10].to_string(),
         expires: Some(constructed_file.get_expiry()),
         ..Default::default()
     }))
@@ -207,11 +206,11 @@ struct ClientResponse {
 
     pub response: &'static str,
 
-    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(skip_serializing_if = "str::is_empty")]
     pub name: String,
-    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(skip_serializing_if = "str::is_empty")]
     pub url: String,
-    #[serde(skip_serializing_if = "String::is_empty")]
+    #[serde(skip_serializing_if = "str::is_empty")]
     pub hash: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub expires: Option<DateTime<Utc>>,
@@ -223,20 +222,11 @@ fn get_id(name: &str, hash: Hash) -> String {
 }
 
 /// Get the Blake3 hash of a file, without reading it all into memory, and also get the size
-async fn hash_file<P: AsRef<Path>>(input: &P) -> Result<(Hash, usize), std::io::Error> {
+async fn hash_file<P: AsRef<Path>>(input: &P) -> Result<Hash, std::io::Error> {
-    let mut file = File::open(input).await?;
-    let mut buf = vec![0; 5000000];
     let mut hasher = blake3::Hasher::new();
+    hasher.update_mmap_rayon(input)?;
 
-    let mut total = 0;
+    Ok(hasher.finalize())
-    let mut bytes_read = None;
-    while bytes_read != Some(0) {
-        bytes_read = Some(file.read(&mut buf).await?);
-        total += bytes_read.unwrap();
-        hasher.update(&buf[..bytes_read.unwrap()]);
-    }
-
-    Ok((hasher.finalize(), total))
 }
 
 /// An endpoint to obtain information about the server's capabilities

web/request.js

@@ -57,7 +57,7 @@ function makeFinished(progressBar, progressText, linkRow, linkAddress, hash) {
     progressText.textContent = "";
     const link = progressText.appendChild(document.createElement("a"));
     link.textContent = hash;
-    link.href = linkAddress;
+    link.href = "/files/" + linkAddress;
     link.target = "_blank";
 
     let button = linkRow.appendChild(document.createElement("button"));
@@ -68,7 +68,7 @@ function makeFinished(progressBar, progressText, linkRow, linkAddress, hash) {
             clearTimeout(buttonTimeout)
         }
         navigator.clipboard.writeText(
-            encodeURI(window.location.protocol + "//" + window.location.host + "/" + linkAddress)
+            encodeURI(window.location.protocol + "//" + window.location.host + "/files/" + linkAddress)
         )
         button.textContent = "✅";
         buttonTimeout = setTimeout(function() {