refactor: asset pipeline

src/gen/load.rs

@@ -2,53 +2,69 @@ use std::collections::HashSet ;
 
 use camino::Utf8PathBuf;
 use glob::glob;
+use hayagriva::Library;
+
+use crate::html::Linkable;
+
+use super::Sack;
 
 
+/// Whether the item should be treated as a content page, converted into a standalone HTML page, or
+/// as a bundled asset.
 #[derive(Debug)]
-pub enum SourceKind {
+pub enum StaticItemKind {
+    /// Convert to `index.html`
     Index,
-    Asset,
+    /// Convert to a bundled asset
+    Bundle,
 }
 
+/// Metadata for a single item consumed by SSG.
 #[derive(Debug)]
-pub struct Source {
-    pub kind: SourceKind,
+pub struct StaticItem {
+    /// Kind of an item
+    pub kind: StaticItemKind,
+    /// Original extension for the source file
     pub ext: String,
-    pub dirs: Utf8PathBuf,
-    pub path: Utf8PathBuf,
+    pub dir: Utf8PathBuf,
+    pub src: Utf8PathBuf,
 }
 
+pub enum AssetKind {
+    Html(Box<dyn Fn(&Sack) -> String>),
+    Image,
+    Other,
+    Bib(Library),
+}
 
-fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> Source {
-    let dir = path.parent().unwrap();
-    let ext = path.extension().unwrap();
+pub struct Asset {
+    pub kind: AssetKind,
+    pub out: Utf8PathBuf,
+    pub meta: StaticItem,
+    pub link: Option<Linkable>,
+}
 
-    if !exts.contains(ext) {
-        return Source {
-            kind: SourceKind::Asset,
-            ext: ext.to_owned(),
-            dirs: dir.to_owned(),
-            path,
-        };
+pub enum PipelineItem {
+    Skip(StaticItem),
+    Take(Asset),
+}
+
+impl From<StaticItem> for PipelineItem {
+    fn from(value: StaticItem) -> Self {
+        Self::Skip(value)
     }
+}
 
-    let dirs = match path.file_stem().unwrap() {
-        "index" => dir.to_owned(),
-        name    => dir.join(name),
-    };
-
-    Source {
-        kind: SourceKind::Index,
-        ext: ext.to_owned(),
-        dirs,
-        path,
+impl From<Asset> for PipelineItem {
+    fn from(value: Asset) -> Self {
+        Self::Take(value)
     }
 }
 
 
-pub fn gather(pattern: &str, exts: &HashSet<&'static str>) -> Vec<Source> {
+pub fn gather(pattern: &str, exts: &HashSet<&'static str>) -> Vec<PipelineItem> {
     glob(pattern)
-        .unwrap()
+        .expect("Invalid glob pattern")
         .filter_map(|path| {
             let path = path.unwrap();
             let path = Utf8PathBuf::from_path_buf(path).expect("Filename is not valid UTF8");
@@ -58,5 +74,33 @@ pub fn gather(pattern: &str, exts: &HashSet<&'static str>) -> Vec<Source> {
                 false => Some(to_source(path, exts))
             }
         })
+        .map(Into::into)
         .collect()
 }
+
+
+fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> StaticItem {
+    let dir = path.parent().unwrap();
+    let ext = path.extension().unwrap();
+
+    if !exts.contains(ext) {
+        return StaticItem {
+            kind: StaticItemKind::Bundle,
+            ext: ext.to_owned(),
+            dir: dir.to_owned(),
+            src: path,
+        };
+    }
+
+    let dirs = match path.file_stem().unwrap() {
+        "index" => dir.to_owned(),
+        name    => dir.join(name),
+    };
+
+    StaticItem {
+        kind: StaticItemKind::Index,
+        ext: ext.to_owned(),
+        dir: dirs,
+        src: path,
+    }
+}

src/gen/mod.rs

@@ -6,8 +6,8 @@ use camino::Utf8PathBuf;
 use hayagriva::Library;
 use hypertext::Renderable;
 
-pub use load::{gather, Source, SourceKind};
-pub use render::{render, Asset, AssetKind, Virtual, Item};
+pub use load::{gather, StaticItem, StaticItemKind, Asset, AssetKind, PipelineItem};
+pub use render::{render, Virtual, Item};
 pub use sack::{TreePage, Sack};
 
 use crate::{html::Linkable, text::md::Outline};

src/gen/render.rs

@@ -1,38 +1,12 @@
-use std::fmt::Debug;
 use std::fs::{self, File};
 use std::io::Write;
 
 use camino::{Utf8Path, Utf8PathBuf};
 
-use crate::html::Linkable;
 use crate::Sack;
 
+use super::{Asset, AssetKind};
 
-pub enum AssetKind {
-    Html(Box<dyn Fn(&Sack) -> String>),
-    Image,
-    Unknown,
-    Bib(hayagriva::Library),
-}
-
-impl Debug for AssetKind {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Html(ptr) => f.debug_tuple("Html").field(&format!("{:p}", *ptr)).finish(),
-            Self::Image => write!(f, "Image"),
-            Self::Unknown => write!(f, "Unknown"),
-            Self::Bib(bib) => f.debug_tuple("Bib").field(bib).finish(),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct Asset {
-    pub kind: AssetKind,
-    pub out: Utf8PathBuf,
-    pub link: Option<Linkable>,
-    pub meta: super::Source,
-}
 
 pub struct Virtual(pub Utf8PathBuf, pub Box<dyn Fn(&Sack) -> String>);
 
@@ -85,7 +59,7 @@ pub fn render(items: &[Item]) {
 fn render_real(item: &Asset, sack: &Sack) {
     match &item.kind {
         AssetKind::Html(render) => {
-            let i = &item.meta.path;
+            let i = &item.meta.src;
             let o = Utf8Path::new("dist").join(&item.out);
 
             fs::create_dir_all(o.parent().unwrap()).unwrap();
@@ -96,15 +70,15 @@ fn render_real(item: &Asset, sack: &Sack) {
             println!("HTML: {} -> {}", i, o);
         },
         AssetKind::Image => {
-            let i = &item.meta.path;
+            let i = &item.meta.src;
             let o = Utf8Path::new("dist").join(&item.out);
             fs::create_dir_all(o.parent().unwrap()).unwrap();
             fs::copy(i, &o).unwrap();
             println!("Image: {} -> {}", i, o);
         },
         AssetKind::Bib(_) => (),
-        AssetKind::Unknown => {
-            let i = &item.meta.path;
+        AssetKind::Other => {
+            let i = &item.meta.src;
             let o = Utf8Path::new("dist").join(&item.out);
             fs::create_dir_all(o.parent().unwrap()).unwrap();
             fs::copy(i, &o).unwrap();

src/gen/sack.rs

@@ -35,7 +35,6 @@ impl TreePage {
 
 
 /// This struct allows for querying the website hierarchy.
-#[derive(Debug)]
 pub struct Sack<'a> {
     assets: &'a [&'a Asset],
     path: &'a Utf8PathBuf,

src/main.rs

@@ -4,11 +4,12 @@ use std::fs;
 
 use camino::{Utf8Path, Utf8PathBuf};
 use chrono::Datelike;
-use gen::{Asset, Sack, Content};
+use gen::{Asset, AssetKind, Content, PipelineItem, Sack, StaticItemKind};
 use hayagriva::Library;
 use html::{Link, LinkDate, Linkable};
 use hypertext::{Raw, Renderable};
 use once_cell::sync::Lazy;
+use serde::Deserialize;
 use text::md::Outline;
 
 mod md;
@@ -171,74 +172,73 @@ fn to_list(list: Vec<LinkDate>) -> String {
     html::list("", &groups).render().into()
 }
 
-
-fn transform<T>(meta: gen::Source) -> Asset
+fn to_index<T>(item: PipelineItem) -> PipelineItem
     where
-        T: for<'de> serde::Deserialize<'de>,
-        T: Content + 'static,
+        T: for<'de> Deserialize<'de> + Content + 'static,
 {
-    let dir = meta.dirs.strip_prefix("content").unwrap();
+    let meta = match item {
+        PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Index) => meta,
+        _ => return item,
+    };
 
-    match meta.kind {
-        gen::SourceKind::Index => match meta.ext.as_str() {
-            "md" | "mdx" | "lhs" => {
-                let path = dir.join("index.html");
+    let dir = meta.dir.strip_prefix("content").unwrap();
+    match meta.ext.as_str() {
+        "md" | "mdx" | "lhs" => {
+            let path = dir.join("index.html");
 
-                let data = fs::read_to_string(&meta.path).unwrap();
-                let (fm, md) = md::preflight::<T>(&data);
-                let link = T::as_link(&fm, Utf8Path::new("/").join(dir));
+            let data = fs::read_to_string(&meta.src).unwrap();
+            let (fm, md) = md::preflight::<T>(&data);
+            let link = T::as_link(&fm, Utf8Path::new("/").join(dir));
 
-                let call = move |sack: &Sack| {
-                    let lib = sack.get_library();
-                    let (outline, html, bib) = T::render(&md, lib);
-                    T::transform(&fm, Raw(html), outline, sack, bib).render().into()
-                };
+            let call = move |sack: &Sack| {
+                let lib = sack.get_library();
+                let (outline, html, bib) = T::render(&md, lib);
+                T::transform(&fm, Raw(html), outline, sack, bib).render().into()
+            };
 
-                gen::Asset {
-                    kind: gen::AssetKind::Html(Box::new(call)),
-                    out: path,
-                    link,
-                    meta,
-                }
-            },
-            _ => gen::Asset {
-                kind: gen::AssetKind::Unknown,
-                out: dir.join(meta.path.file_name().unwrap()).to_owned(),
-                link: None,
+            gen::Asset {
+                kind: gen::AssetKind::Html(Box::new(call)),
+                out: path,
+                link,
                 meta,
-            }
+            }.into()
         },
-        gen::SourceKind::Asset => {
-            let loc = dir.join(meta.path.file_name().unwrap()).to_owned();
-            match meta.ext.as_str() {
-                "jpg" | "png" | "gif" => gen::Asset {
-                    kind: gen::AssetKind::Image,
-                    out: loc,
-                    link: None,
-                    meta,
-                },
-                "bib" => {
-                    let data = fs::read_to_string(&meta.path).unwrap();
-                    let data = hayagriva::io::from_biblatex_str(&data).unwrap();
-
-                    gen::Asset {
-                        kind: gen::AssetKind::Bib(data),
-                        out: loc,
-                        link: None,
-                        meta,
-                    }
-                },
-                _ => gen::Asset {
-                    kind: gen::AssetKind::Unknown,
-                    out: loc,
-                    link: None,
-                    meta,
-                },
-            }
-        }
+        _ => meta.into(),
     }
 }
 
+fn to_bundle(item: PipelineItem) -> PipelineItem {
+    let meta = match item {
+        PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Bundle) => meta,
+        _ => return item,
+    };
+
+    let dir = meta.dir.strip_prefix("content").unwrap();
+    let out = dir.join(meta.src.file_name().unwrap()).to_owned();
+
+    match meta.ext.as_str() {
+        "jpg" | "png" | "gif" => gen::Asset {
+            kind: gen::AssetKind::Image,
+            out,
+            link: None,
+            meta,
+        }.into(),
+        "bib" => {
+            let data = fs::read_to_string(&meta.src).unwrap();
+            let data = hayagriva::io::from_biblatex_str(&data).unwrap();
+
+            Asset {
+                kind: AssetKind::Bib(data),
+                out,
+                link: None,
+                meta,
+            }.into()
+        },
+        _ => meta.into(),
+    }
+}
+
+
 fn main() {
     if fs::metadata("dist").is_ok() {
         println!("Cleaning dist");
@@ -247,7 +247,36 @@ fn main() {
 
     fs::create_dir("dist").unwrap();
 
+    let assets: Vec<Asset> = vec![
+        gen::gather("content/about.md", &["md"].into())
+            .into_iter()
+            .map(to_index::<md::Post> as fn(PipelineItem) -> PipelineItem),
+        gen::gather("content/posts/**/*", &["md", "mdx"].into())
+            .into_iter()
+            .map(to_index::<md::Post>),
+        gen::gather("content/slides/**/*", &["md", "lhs"].into())
+            .into_iter()
+            .map(to_index::<md::Slide>),
+        gen::gather("content/wiki/**/*", &["md"].into())
+            .into_iter()
+            .map(to_index::<md::Wiki>),
+    ]
+        .into_iter()
+        .flatten()
+        .map(to_bundle)
+        .filter_map(|item| match item {
+            PipelineItem::Skip(skip) => {
+                println!("Skipping {}", skip.src);
+                None
+            },
+            PipelineItem::Take(take) => Some(take),
+        })
+        .collect();
+
     let assets: Vec<Vec<gen::Item>> = vec![
+        assets.into_iter()
+            .map(Into::into)
+            .collect(),
         vec![
             gen::Virtual::new("map/index.html", |_| html::map().render().to_owned().into()).into(),
             gen::Virtual::new("search/index.html", |_| html::search().render().to_owned().into()).into(),
@@ -259,11 +288,11 @@ fn main() {
                 })),
                 out: "index.html".into(),
                 link: None,
-                meta: gen::Source {
-                    kind: gen::SourceKind::Index,
+                meta: gen::StaticItem {
+                    kind: gen::StaticItemKind::Index,
                     ext: "md".into(),
-                    dirs: "".into(),
-                    path: "content/index.md".into()
+                    dir: "".into(),
+                    src: "content/index.md".into()
                 }
             }.into(),
             gen::Virtual("posts/index.html".into(), Box::new(|all|
@@ -273,26 +302,6 @@ fn main() {
                 to_list(all.get_links("slides/**/*.html"))
             )).into(),
         ],
-        gen::gather("content/about.md", &["md"].into())
-            .into_iter()
-            .map(transform::<md::Post>)
-            .map(Into::into)
-            .collect(),
-        gen::gather("content/posts/**/*", &["md", "mdx"].into())
-            .into_iter()
-            .map(transform::<md::Post>)
-            .map(Into::into)
-            .collect(),
-        gen::gather("content/slides/**/*", &["md", "lhs"].into())
-            .into_iter()
-            .map(transform::<md::Slide>)
-            .map(Into::into)
-            .collect(),
-        gen::gather("content/wiki/**/*", &["md"].into())
-            .into_iter()
-            .map(transform::<md::Wiki>)
-            .map(Into::into)
-            .collect(),
     ];
 
     let all: Vec<gen::Item> = assets