refactor: content pipeline

src/gen/load.rs

@@ -1,6 +1,6 @@
-use std::collections::HashSet ;
+use std::{collections::HashSet, fs::{self, File}, io::Write} ;
 
-use camino::Utf8PathBuf;
+use camino::{Utf8Path, Utf8PathBuf};
 use glob::glob;
 use hayagriva::Library;
 
@@ -9,10 +9,10 @@ use crate::html::Linkable;
 use super::Sack;
 
 
-/// Whether the item should be treated as a content page, converted into a standalone HTML page, or
+/// Marks whether the item should be treated as a content page, converted into a standalone HTML
-/// as a bundled asset.
+/// page, or as a bundled asset.
 #[derive(Debug)]
-pub enum StaticItemKind {
+pub enum FileItemKind {
     /// Convert to `index.html`
     Index,
     /// Convert to a bundled asset
@@ -21,42 +21,81 @@ pub enum StaticItemKind {
 
 /// Metadata for a single item consumed by SSG.
 #[derive(Debug)]
-pub struct StaticItem {
+pub struct FileItem {
     /// Kind of an item
-    pub kind: StaticItemKind,
+    pub kind: FileItemKind,
-    /// Original extension for the source file
+    /// Original source file location
-    pub ext: String,
+    pub path: Utf8PathBuf,
-    pub dir: Utf8PathBuf,
-    pub src: Utf8PathBuf,
 }
 
+/// Marks how the asset should be processed by the SSG
 pub enum AssetKind {
+    /// Data renderable to HTML
     Html(Box<dyn Fn(&Sack) -> String>),
+    /// Bibliographical data
+    Bibtex(Library),
+    /// Images
     Image,
-    Other,
-    Bib(Library),
 }
 
+/// Asset renderable by the SSG
 pub struct Asset {
+    /// Kind of a processed asset
     pub kind: AssetKind,
-    pub out: Utf8PathBuf,
+    /// File metadata
-    pub meta: StaticItem,
+    pub meta: FileItem,
+}
+
+/// Dynamically generated asset not related to any disk file.
+pub struct Dynamic(pub Box<dyn Fn(&Sack) -> String>);
+
+impl Dynamic {
+    pub fn new(call: impl Fn(&Sack) -> String + 'static) -> Self {
+        Self(Box::new(call))
+    }
+}
+
+pub enum OutputKind {
+    Real(Asset),
+    Fake(Dynamic),
+}
+
+impl From<Asset> for OutputKind {
+    fn from(value: Asset) -> Self {
+        OutputKind::Real(value)
+    }
+}
+
+impl From<Dynamic> for OutputKind {
+    fn from(value: Dynamic) -> Self {
+        OutputKind::Fake(value)
+    }
+}
+
+/// Renderable output
+pub struct Output {
+    pub kind: OutputKind,
+    pub path: Utf8PathBuf,
+    /// Optional link to outputted page.
     pub link: Option<Linkable>,
 }
 
+/// Variants used for filtering static assets.
 pub enum PipelineItem {
-    Skip(StaticItem),
+    /// Unclaimed file, unrecognized file extensions.
-    Take(Asset),
+    Skip(FileItem),
+    /// Data ready to be processed by SSG.
+    Take(Output),
 }
 
-impl From<StaticItem> for PipelineItem {
+impl From<FileItem> for PipelineItem {
-    fn from(value: StaticItem) -> Self {
+    fn from(value: FileItem) -> Self {
         Self::Skip(value)
     }
 }
 
-impl From<Asset> for PipelineItem {
+impl From<Output> for PipelineItem {
-    fn from(value: Asset) -> Self {
+    fn from(value: Output) -> Self {
         Self::Take(value)
     }
 }
@@ -79,28 +118,53 @@ pub fn gather(pattern: &str, exts: &HashSet<&'static str>) -> Vec<PipelineItem>
 }
 
 
-fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> StaticItem {
+fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> FileItem {
-    let dir = path.parent().unwrap();
+    let hit = path.extension().map_or(false, |ext| exts.contains(ext));
-    let ext = path.extension().unwrap();
 
-    if !exts.contains(ext) {
+    let kind = match hit {
-        return StaticItem {
+        true  => FileItemKind::Index,
-            kind: StaticItemKind::Bundle,
+        false => FileItemKind::Bundle,
-            ext: ext.to_owned(),
-            dir: dir.to_owned(),
-            src: path,
-        };
-    }
-
-    let dirs = match path.file_stem().unwrap() {
-        "index" => dir.to_owned(),
-        name    => dir.join(name),
     };
 
-    StaticItem {
+    FileItem {
-        kind: StaticItemKind::Index,
+        kind,
-        ext: ext.to_owned(),
+        path,
-        dir: dirs,
+    }
-        src: path,
+}
+
+
+pub fn render_all(items: &[Output]) {
+    for item in items {
+        render(item, &Sack::new(items, &item.path));
+    }
+}
+
+fn render(item: &Output, sack: &Sack) {
+    let o = Utf8Path::new("dist").join(&item.path);
+    fs::create_dir_all(o.parent().unwrap()).unwrap();
+
+    match item.kind {
+        OutputKind::Real(ref real) => {
+            let i = &real.meta.path;
+
+            match &real.kind {
+                AssetKind::Html(closure) => {
+                    let mut file = File::create(&o).unwrap();
+                    file.write_all(closure(sack).as_bytes()).unwrap();
+                    println!("HTML: {} -> {}", i, o);
+                },
+                AssetKind::Bibtex(_) => { },
+                AssetKind::Image => {
+                    fs::create_dir_all(o.parent().unwrap()).unwrap();
+                    fs::copy(i, &o).unwrap();
+                    println!("Image: {} -> {}", i, o);
+                },
+            };
+        },
+        OutputKind::Fake(Dynamic(ref closure)) => {
+            let mut file = File::create(&o).unwrap();
+            file.write_all(closure(sack).as_bytes()).unwrap();
+            println!("Virtual: -> {}", o);
+        },
     }
 }

src/gen/mod.rs

@@ -1,19 +1,17 @@
 mod load;
-mod render;
 mod sack;
 
 use camino::Utf8PathBuf;
 use hayagriva::Library;
 use hypertext::Renderable;
 
-pub use load::{gather, StaticItem, StaticItemKind, Asset, AssetKind, PipelineItem};
+pub use load::{gather, render_all, FileItem, FileItemKind, Asset, AssetKind, PipelineItem, Dynamic, Output};
-pub use render::{render, Virtual, Item};
 pub use sack::{TreePage, Sack};
 
 use crate::{html::Linkable, text::md::Outline};
 
 
-/// Represents a piece of content that can be rendered into a page.
+/// Represents a piece of content that can be rendered as a page.
 pub trait Content {
     fn transform<'f, 'm, 's, 'html, T>(
         &'f self,

src/gen/render.rs

@@ -1,99 +0,0 @@
-use std::fs::{self, File};
-use std::io::Write;
-
-use camino::{Utf8Path, Utf8PathBuf};
-
-use crate::Sack;
-
-use super::{Asset, AssetKind};
-
-
-pub struct Virtual(pub Utf8PathBuf, pub Box<dyn Fn(&Sack) -> String>);
-
-impl Virtual {
-    pub fn new<P, F>(path: P, call: F) -> Self
-        where
-            P: AsRef<Utf8Path>,
-            F: Fn(&Sack) -> String + 'static
-    {
-        Self(path.as_ref().into(), Box::new(call))
-    }
-}
-
-pub enum Item {
-    Real(Asset),
-    Fake(Virtual),
-}
-
-impl From<Asset> for Item {
-    fn from(value: Asset) -> Self {
-        Item::Real(value)
-    }
-}
-
-impl From<Virtual> for Item {
-    fn from(value: Virtual) -> Self {
-        Item::Fake(value)
-    }
-}
-
-
-pub fn render(items: &[Item]) {
-    let assets: Vec<&Asset> = items
-        .iter()
-        .filter_map(|item| match item {
-            Item::Real(a) => Some(a),
-            Item::Fake(_) => None,
-        })
-        .collect();
-
-    for item in items {
-        match item {
-            Item::Real(real) => render_real(real, &Sack::new(&assets, &real.out)),
-            Item::Fake(fake) => render_fake(fake, &Sack::new(&assets, &fake.0)),
-        }
-    }
-}
-
-
-fn render_real(item: &Asset, sack: &Sack) {
-    match &item.kind {
-        AssetKind::Html(render) => {
-            let i = &item.meta.src;
-            let o = Utf8Path::new("dist").join(&item.out);
-
-            fs::create_dir_all(o.parent().unwrap()).unwrap();
-
-            let mut file = File::create(&o).unwrap();
-            file.write_all(render(sack).as_bytes()).unwrap();
-
-            println!("HTML: {} -> {}", i, o);
-        },
-        AssetKind::Image => {
-            let i = &item.meta.src;
-            let o = Utf8Path::new("dist").join(&item.out);
-            fs::create_dir_all(o.parent().unwrap()).unwrap();
-            fs::copy(i, &o).unwrap();
-            println!("Image: {} -> {}", i, o);
-        },
-        AssetKind::Bib(_) => (),
-        AssetKind::Other => {
-            let i = &item.meta.src;
-            let o = Utf8Path::new("dist").join(&item.out);
-            fs::create_dir_all(o.parent().unwrap()).unwrap();
-            fs::copy(i, &o).unwrap();
-            println!("Unknown: {} -> {}", i, o);
-        },
-    }
-}
-
-fn render_fake(item: &Virtual, sack: &Sack) {
-    let Virtual(out, render) = item;
-
-    let o = Utf8Path::new("dist").join(out);
-    fs::create_dir_all(o.parent().unwrap()).unwrap();
-
-    let mut file = File::create(&o).unwrap();
-    file.write_all(render(sack).as_bytes()).unwrap();
-    println!("Virtual: -> {}", o);
-}

src/gen/sack.rs

@@ -5,7 +5,7 @@ use hayagriva::Library;
 
 use crate::html::{Link, LinkDate, Linkable};
 
-use super::{Asset, AssetKind};
+use super::{load::{Output, OutputKind}, AssetKind};
 
 
 #[derive(Debug)]
@@ -34,22 +34,25 @@ impl TreePage {
 }
 
 
-/// This struct allows for querying the website hierarchy.
+/// This struct allows for querying the website hierarchy. Separate instance of this struct is
+/// passed to each closure contained by some rendered assets.
 pub struct Sack<'a> {
-    assets: &'a [&'a Asset],
+    /// Literally everything
+    hole: &'a [Output],
+    /// Current path for page
     path: &'a Utf8PathBuf,
 }
 
 impl<'a> Sack<'a> {
-    pub fn new(assets: &'a [&'a Asset], path: &'a Utf8PathBuf) -> Self {
+    pub fn new(hole: &'a [Output], path: &'a Utf8PathBuf) -> Self {
-        Self { assets, path }
+        Self { hole, path }
     }
 
     pub fn get_links(&self, path: &str) -> Vec<LinkDate> {
-        let pattern = glob::Pattern::new(path).unwrap();
+        let pattern = glob::Pattern::new(path).expect("Bad glob pattern");
-        self.assets.iter()
+        self.hole.iter()
-            .filter(|f| pattern.matches_path(f.out.as_ref()))
+            .filter(|item| pattern.matches_path(item.path.as_ref()))
-            .filter_map(|f| match &f.link {
+            .filter_map(|item| match &item.link {
                 Some(Linkable::Date(link)) => Some(link.clone()),
                 _ => None,
             })
@@ -57,10 +60,10 @@ impl<'a> Sack<'a> {
     }
 
     pub fn get_tree(&self, path: &str) -> TreePage {
-        let glob = glob::Pattern::new(path).unwrap();
+        let glob = glob::Pattern::new(path).expect("Bad glob pattern");
-        let list = self.assets.iter()
+        let list = self.hole.iter()
-            .filter(|f| glob.matches_path(f.out.as_ref()))
+            .filter(|item| glob.matches_path(item.path.as_ref()))
-            .filter_map(|f| match &f.link {
+            .filter_map(|item| match &item.link {
                 Some(Linkable::Link(link)) => Some(link.clone()),
                 _ => None,
             });
@@ -75,18 +78,22 @@ impl<'a> Sack<'a> {
 
     pub fn get_library(&self) -> Option<&Library> {
         let glob = format!("{}/*.bib", self.path.parent()?);
-        let glob = glob::Pattern::new(&glob).unwrap();
+        let glob = glob::Pattern::new(&glob).expect("Bad glob pattern");
         let opts = glob::MatchOptions {
             case_sensitive: true,
             require_literal_separator: true,
             require_literal_leading_dot: false,
         };
 
-        self.assets.iter()
+        self.hole.iter()
-            .filter(|asset| glob.matches_path_with(asset.out.as_ref(), opts))
+            .filter(|item| glob.matches_path_with(item.path.as_ref(), opts))
+            .filter_map(|asset| match asset.kind {
+                OutputKind::Real(ref real) => Some(real),
+                _ => None,
+            })
             .find_map(|asset| match asset.kind {
-                AssetKind::Bib(ref lib) => Some(lib),
+                AssetKind::Bibtex(ref lib) => Some(lib),
-                _ => None
+                _ => None,
             })
     }
 }

src/main.rs

@@ -4,7 +4,7 @@ use std::fs;
 
 use camino::{Utf8Path, Utf8PathBuf};
 use chrono::Datelike;
-use gen::{Asset, AssetKind, Content, PipelineItem, Sack, StaticItemKind};
+use gen::{Asset, AssetKind, Content, FileItemKind, Output, PipelineItem, Sack};
 use hayagriva::Library;
 use html::{Link, LinkDate, Linkable};
 use hypertext::{Raw, Renderable};
@@ -12,6 +12,8 @@ use once_cell::sync::Lazy;
 use serde::Deserialize;
 use text::md::Outline;
 
+use crate::gen::Dynamic;
+
 mod md;
 mod html;
 mod ts;
@@ -177,16 +179,23 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
         T: for<'de> Deserialize<'de> + Content + 'static,
 {
     let meta = match item {
-        PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Index) => meta,
+        PipelineItem::Skip(meta) if matches!(meta.kind, FileItemKind::Index) => meta,
         _ => return item,
     };
 
-    let dir = meta.dir.strip_prefix("content").unwrap();
+    // FIXME: clean this up
-    match meta.ext.as_str() {
+    let dir = meta.path.parent().unwrap();
-        "md" | "mdx" | "lhs" => {
+    let ext = meta.path.extension().unwrap();
+    let dir = dir.strip_prefix("content").unwrap();
+    let dir = match meta.path.file_stem().unwrap() {
+        "index" => dir.to_owned(),
+        name    => dir.join(name),
+    };
     let path = dir.join("index.html");
 
-            let data = fs::read_to_string(&meta.src).unwrap();
+    match ext {
+        "md" | "mdx" | "lhs" => {
+            let data = fs::read_to_string(&meta.path).unwrap();
             let (fm, md) = md::preflight::<T>(&data);
             let link = T::as_link(&fm, Utf8Path::new("/").join(dir));
 
@@ -196,11 +205,13 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
                 T::transform(&fm, Raw(html), outline, sack, bib).render().into()
             };
 
-            gen::Asset {
+            Output {
+                kind: Asset {
                     kind: gen::AssetKind::Html(Box::new(call)),
-                out: path,
-                link,
                     meta,
+                }.into(),
+                path,
+                link,
             }.into()
         },
         _ => meta.into(),
@@ -209,29 +220,33 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
 
 fn to_bundle(item: PipelineItem) -> PipelineItem {
     let meta = match item {
-        PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Bundle) => meta,
+        PipelineItem::Skip(meta) if matches!(meta.kind, FileItemKind::Bundle) => meta,
         _ => return item,
     };
 
-    let dir = meta.dir.strip_prefix("content").unwrap();
+    let dirs = meta.path.strip_prefix("content").unwrap().parent().unwrap();
-    let out = dir.join(meta.src.file_name().unwrap()).to_owned();
+    let path = dirs.join(meta.path.file_name().unwrap()).to_owned();
 
-    match meta.ext.as_str() {
+    match meta.path.extension().unwrap() {
-        "jpg" | "png" | "gif" => gen::Asset {
+        "jpg" | "png" | "gif" => Output {
-            kind: gen::AssetKind::Image,
+            kind: Asset {
-            out,
+                kind: AssetKind::Image,
-            link: None,
                 meta,
             }.into(),
+            path,
+            link: None,
+        }.into(),
         "bib" => {
-            let data = fs::read_to_string(&meta.src).unwrap();
+            let data = fs::read_to_string(&meta.path).unwrap();
             let data = hayagriva::io::from_biblatex_str(&data).unwrap();
 
-            Asset {
+            Output {
-                kind: AssetKind::Bib(data),
+                kind: Asset {
-                out,
+                    kind: AssetKind::Bibtex(data),
-                link: None,
                     meta,
+                }.into(),
+                path,
+                link: None,
             }.into()
         },
         _ => meta.into(),
@@ -247,7 +262,7 @@ fn main() {
 
     fs::create_dir("dist").unwrap();
 
-    let assets: Vec<Asset> = vec![
+    let assets: Vec<Output> = vec![
         gen::gather("content/about.md", &["md"].into())
             .into_iter()
             .map(to_index::<md::Post> as fn(PipelineItem) -> PipelineItem),
@@ -266,52 +281,60 @@ fn main() {
         .map(to_bundle)
         .filter_map(|item| match item {
             PipelineItem::Skip(skip) => {
-                println!("Skipping {}", skip.src);
+                println!("Skipping {}", skip.path);
                 None
             },
             PipelineItem::Take(take) => Some(take),
         })
         .collect();
 
-    let assets: Vec<Vec<gen::Item>> = vec![
+    let assets: Vec<Output> = vec![
-        assets.into_iter()
+        assets,
-            .map(Into::into)
-            .collect(),
         vec![
-            gen::Virtual::new("map/index.html", |_| html::map().render().to_owned().into()).into(),
+            Output {
-            gen::Virtual::new("search/index.html", |_| html::search().render().to_owned().into()).into(),
+                kind: Dynamic::new(|_| html::map().render().to_owned().into()).into(),
-            gen::Asset {
+                path: "map/index.html".into(),
+                link: None,
+            },
+            Output {
+                kind: Dynamic::new(|_| html::search().render().to_owned().into()).into(),
+                path: "search/index.html".into(),
+                link: None,
+            },
+            Output {
+                kind: Asset {
                     kind: gen::AssetKind::Html(Box::new(|_| {
                         let data = std::fs::read_to_string("content/index.md").unwrap();
                         let (_, html, bib) = text::md::parse(&data, None);
                         html::home(Raw(html)).render().to_owned().into()
-                })),
+                    })).into(),
-                out: "index.html".into(),
+                    meta: gen::FileItem {
-                link: None,
+                        kind: gen::FileItemKind::Index,
-                meta: gen::StaticItem {
+                        path: "content/index.md".into()
-                    kind: gen::StaticItemKind::Index,
-                    ext: "md".into(),
-                    dir: "".into(),
-                    src: "content/index.md".into()
                     }
                 }.into(),
-            gen::Virtual("posts/index.html".into(), Box::new(|all|
+                path: "index.html".into(),
-                to_list(all.get_links("posts/**/*.html"))
+                link: None,
-            )).into(),
+            }.into(),
-            gen::Virtual("slides/index.html".into(), Box::new(|all|
+            Output {
-                to_list(all.get_links("slides/**/*.html"))
+                kind: Dynamic::new(|sack| to_list(sack.get_links("posts/**/*.html"))).into(),
-            )).into(),
+                path: "posts/index.html".into(),
+                link: None,
+            },
+            Output {
+                kind: Dynamic::new(|sack| to_list(sack.get_links("slides/**/*.html"))).into(),
+                path: "slides/index.html".into(),
+                link: None,
+            },
         ],
-    ];
+    ]
-
-    let all: Vec<gen::Item> = assets
         .into_iter()
         .flatten()
         .collect();
 
     {
         let now = std::time::Instant::now();
-        gen::render(&all);
+        gen::render_all(&assets);
         println!("Elapsed: {:.2?}", now.elapsed());
     }