refactor hashing
This commit is contained in:
parent
cc255f1eaf
commit
d6034633bf
73
src/build.rs
73
src/build.rs
|
@ -29,16 +29,11 @@ pub(crate) fn build_content(
|
||||||
pending: &[&Output],
|
pending: &[&Output],
|
||||||
hole: &[&Output],
|
hole: &[&Output],
|
||||||
hash: Option<HashMap<Utf8PathBuf, Utf8PathBuf>>,
|
hash: Option<HashMap<Utf8PathBuf, Utf8PathBuf>>,
|
||||||
) -> HashMap<Utf8PathBuf, Utf8PathBuf> {
|
) {
|
||||||
let now = std::time::Instant::now();
|
let now = std::time::Instant::now();
|
||||||
let hashes = render_all(ctx, pending, hole, hash);
|
render_all(ctx, pending, hole, hash);
|
||||||
println!("Elapsed: {:.2?}", now.elapsed());
|
println!("Elapsed: {:.2?}", now.elapsed());
|
||||||
copy_recursively(Path::new(".hash"), Path::new("dist/hash")).unwrap();
|
copy_recursively(Path::new(".hash"), Path::new("dist/hash")).unwrap();
|
||||||
let mut lmao = HashMap::<Utf8PathBuf, Utf8PathBuf>::new();
|
|
||||||
for hash in hashes {
|
|
||||||
lmao.insert(hash.file, hash.hash);
|
|
||||||
}
|
|
||||||
lmao
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn build_static() {
|
pub(crate) fn build_static() {
|
||||||
|
@ -89,10 +84,10 @@ fn render_all(
|
||||||
pending: &[&Output],
|
pending: &[&Output],
|
||||||
hole: &[&Output],
|
hole: &[&Output],
|
||||||
hash: Option<HashMap<Utf8PathBuf, Utf8PathBuf>>,
|
hash: Option<HashMap<Utf8PathBuf, Utf8PathBuf>>,
|
||||||
) -> Vec<Hashed> {
|
) {
|
||||||
pending
|
pending
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|item| {
|
.map(|item| {
|
||||||
let file = match &item.kind {
|
let file = match &item.kind {
|
||||||
OutputKind::Asset(a) => Some(&a.meta.path),
|
OutputKind::Asset(a) => Some(&a.meta.path),
|
||||||
OutputKind::Virtual(_) => None,
|
OutputKind::Virtual(_) => None,
|
||||||
|
@ -112,17 +107,20 @@ fn render_all(
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn store_hash_png(data: &[u8]) -> Utf8PathBuf {
|
fn store_hash(buffer: &[u8]) -> Utf8PathBuf {
|
||||||
let store = Utf8Path::new(".hash");
|
let store = Utf8Path::new(".hash");
|
||||||
let ident = sha256::digest(data);
|
let hash = sha256::digest(buffer);
|
||||||
let store_hash = store.join(&ident).with_extension("webp");
|
let img = image::load_from_memory(buffer).expect("Couldn't load image");
|
||||||
|
let store_hash = store.join(&hash).with_extension("webp");
|
||||||
|
|
||||||
if !store_hash.exists() {
|
if !store_hash.exists() {
|
||||||
let img = image::load_from_memory(data).expect("Couldn't load image");
|
|
||||||
let dim = (img.width(), img.height());
|
let dim = (img.width(), img.height());
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
let encoder = image::codecs::webp::WebPEncoder::new_lossless(&mut out);
|
let encoder = image::codecs::webp::WebPEncoder::new_lossless(&mut out);
|
||||||
encoder.encode(&img.to_rgba8(), dim.0, dim.1, image::ColorType::Rgba8).expect("Encoding error");
|
|
||||||
|
encoder
|
||||||
|
.encode(&img.to_rgba8(), dim.0, dim.1, image::ColorType::Rgba8)
|
||||||
|
.expect("Encoding error");
|
||||||
|
|
||||||
fs::create_dir_all(store).unwrap();
|
fs::create_dir_all(store).unwrap();
|
||||||
fs::write(store_hash, out).expect("Couldn't output optimized image");
|
fs::write(store_hash, out).expect("Couldn't output optimized image");
|
||||||
|
@ -130,17 +128,45 @@ fn store_hash_png(data: &[u8]) -> Utf8PathBuf {
|
||||||
|
|
||||||
Utf8Path::new("/")
|
Utf8Path::new("/")
|
||||||
.join("hash")
|
.join("hash")
|
||||||
.join(ident)
|
.join(hash)
|
||||||
.with_extension("webp")
|
.with_extension("webp")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn store_hash_all(items: &[&Output]) -> Vec<Hashed> {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter_map(|item| match item.kind {
|
||||||
|
OutputKind::Asset(ref asset) => match asset.kind {
|
||||||
|
AssetKind::Image => {
|
||||||
|
let buffer = std::fs::read(&asset.meta.path).expect("Couldn't read file");
|
||||||
|
let format = image::guess_format(&buffer).expect("Couldn't read format");
|
||||||
|
|
||||||
|
if matches!(format, image::ImageFormat::Gif) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash = store_hash(&buffer);
|
||||||
|
println!("Hashing image {} as {}", asset.meta.path, hash);
|
||||||
|
|
||||||
|
Some(Hashed {
|
||||||
|
file: item.path.to_owned(),
|
||||||
|
hash,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
},
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct Hashed {
|
pub(crate) struct Hashed {
|
||||||
pub file: Utf8PathBuf,
|
pub file: Utf8PathBuf,
|
||||||
pub hash: Utf8PathBuf,
|
pub hash: Utf8PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render(item: &Output, sack: Sack) -> Option<Hashed> {
|
fn render(item: &Output, sack: Sack) {
|
||||||
let dist = Utf8Path::new("dist");
|
let dist = Utf8Path::new("dist");
|
||||||
let o = dist.join(&item.path);
|
let o = dist.join(&item.path);
|
||||||
fs::create_dir_all(o.parent().unwrap()).unwrap();
|
fs::create_dir_all(o.parent().unwrap()).unwrap();
|
||||||
|
@ -154,24 +180,12 @@ fn render(item: &Output, sack: Sack) -> Option<Hashed> {
|
||||||
let mut file = File::create(&o).unwrap();
|
let mut file = File::create(&o).unwrap();
|
||||||
file.write_all(closure(&sack).as_bytes()).unwrap();
|
file.write_all(closure(&sack).as_bytes()).unwrap();
|
||||||
println!("HTML: {} -> {}", i, o);
|
println!("HTML: {} -> {}", i, o);
|
||||||
None
|
|
||||||
}
|
}
|
||||||
AssetKind::Bibtex(_) => None,
|
AssetKind::Bibtex(_) => (),
|
||||||
AssetKind::Image => {
|
AssetKind::Image => {
|
||||||
let hash = match item.path.extension() {
|
|
||||||
Some("png") => Some(store_hash_png(&std::fs::read(i).unwrap())),
|
|
||||||
Some("") => None,
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
fs::create_dir_all(o.parent().unwrap()).unwrap();
|
fs::create_dir_all(o.parent().unwrap()).unwrap();
|
||||||
fs::copy(i, &o).unwrap();
|
fs::copy(i, &o).unwrap();
|
||||||
println!("Image: {} -> {}", i, o);
|
println!("Image: {} -> {}", i, o);
|
||||||
|
|
||||||
hash.map(|hash| Hashed {
|
|
||||||
file: item.path.to_owned(),
|
|
||||||
hash,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -179,7 +193,6 @@ fn render(item: &Output, sack: Sack) -> Option<Hashed> {
|
||||||
let mut file = File::create(&o).unwrap();
|
let mut file = File::create(&o).unwrap();
|
||||||
file.write_all(closure(&sack).as_bytes()).unwrap();
|
file.write_all(closure(&sack).as_bytes()).unwrap();
|
||||||
println!("Virtual: -> {}", o);
|
println!("Virtual: -> {}", o);
|
||||||
None
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
28
src/main.rs
28
src/main.rs
|
@ -10,13 +10,14 @@ use std::collections::{HashMap, HashSet};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
|
|
||||||
|
use build::Hashed;
|
||||||
use camino::{Utf8Path, Utf8PathBuf};
|
use camino::{Utf8Path, Utf8PathBuf};
|
||||||
use chrono::{DateTime, Datelike, Utc};
|
use chrono::{DateTime, Datelike, Utc};
|
||||||
use clap::{Parser, ValueEnum};
|
use clap::{Parser, ValueEnum};
|
||||||
use gray_matter::engine::YAML;
|
use gray_matter::engine::YAML;
|
||||||
use gray_matter::Matter;
|
use gray_matter::Matter;
|
||||||
use hypertext::{Raw, Renderable};
|
use hypertext::{Raw, Renderable};
|
||||||
use pipeline::{Asset, AssetKind, Content, FileItemKind, Output, PipelineItem};
|
use pipeline::{Asset, AssetKind, Content, FileItemKind, Output, OutputKind, PipelineItem};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
use crate::pipeline::Virtual;
|
use crate::pipeline::Virtual;
|
||||||
|
@ -201,23 +202,33 @@ impl Source {
|
||||||
fn build(ctx: &BuildContext, sources: &[Source], special: Vec<Output>) -> Vec<Output> {
|
fn build(ctx: &BuildContext, sources: &[Source], special: Vec<Output>) -> Vec<Output> {
|
||||||
crate::build::clean_dist();
|
crate::build::clean_dist();
|
||||||
|
|
||||||
let sources: Vec<_> = sources
|
let content: Vec<Output> = sources
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(Source::get)
|
.flat_map(Source::get)
|
||||||
.map(to_bundle)
|
.map(to_bundle)
|
||||||
.filter_map(Option::from)
|
.filter_map(Option::from)
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let assets: Vec<_> = sources.iter().chain(special.iter()).collect();
|
let images: Vec<&Output> = content
|
||||||
|
.iter()
|
||||||
|
.filter(|&e| match e.kind {
|
||||||
|
OutputKind::Asset(ref a) => matches!(a.kind, AssetKind::Image),
|
||||||
|
_ => false,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
let lmao = crate::build::build_content(ctx, &assets, &assets, None);
|
let hashes = crate::build::store_hash_all(&images);
|
||||||
crate::build::build_content(ctx, &assets, &assets, Some(lmao));
|
let hashes = HashMap::from_iter(hashes.into_iter().map(|Hashed { file, hash } | (file, hash)));
|
||||||
|
|
||||||
|
let assets: Vec<_> = content.iter().chain(special.iter()).collect();
|
||||||
|
|
||||||
|
crate::build::build_content(ctx, &assets, &assets, Some(hashes));
|
||||||
crate::build::build_static();
|
crate::build::build_static();
|
||||||
crate::build::build_styles();
|
crate::build::build_styles();
|
||||||
crate::build::build_pagefind();
|
crate::build::build_pagefind();
|
||||||
crate::build::build_js();
|
crate::build::build_js();
|
||||||
|
|
||||||
sources.into_iter().chain(special).collect()
|
content.into_iter().chain(special).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse_frontmatter<D>(raw: &str) -> (D, String)
|
pub fn parse_frontmatter<D>(raw: &str) -> (D, String)
|
||||||
|
@ -265,7 +276,10 @@ where
|
||||||
parsed.clone(),
|
parsed.clone(),
|
||||||
lib,
|
lib,
|
||||||
dir.clone(),
|
dir.clone(),
|
||||||
sack.hash.as_ref().map(ToOwned::to_owned).unwrap_or_default(),
|
sack.hash
|
||||||
|
.as_ref()
|
||||||
|
.map(ToOwned::to_owned)
|
||||||
|
.unwrap_or_default(),
|
||||||
);
|
);
|
||||||
T::render(matter.clone(), sack, Raw(parsed), outline, bib)
|
T::render(matter.clone(), sack, Raw(parsed), outline, bib)
|
||||||
.render()
|
.render()
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
use camino::{Utf8Path, Utf8PathBuf};
|
use camino::Utf8PathBuf;
|
||||||
use hayagriva::{
|
use hayagriva::{
|
||||||
archive::ArchivedStyle,
|
archive::ArchivedStyle,
|
||||||
citationberg::{IndependentStyle, Locale, Style},
|
citationberg::{IndependentStyle, Locale, Style},
|
||||||
|
|
|
@ -2,7 +2,7 @@ use std::collections::HashMap;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::io::Result;
|
use std::io::Result;
|
||||||
use std::net::{TcpListener, TcpStream};
|
use std::net::{TcpListener, TcpStream};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::Path;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::sync::mpsc::Sender;
|
use std::sync::mpsc::Sender;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
Loading…
Reference in a new issue