refactor: content pipeline

This commit is contained in:
Maciej Jur 2024-05-02 00:53:17 +02:00
parent 426edc3a03
commit fdb2d01136
Signed by: kamov
GPG key ID: 191CBFF5F72ECAFD
5 changed files with 210 additions and 217 deletions

View file

@ -1,6 +1,6 @@
use std::collections::HashSet ;
use std::{collections::HashSet, fs::{self, File}, io::Write} ;
use camino::Utf8PathBuf;
use camino::{Utf8Path, Utf8PathBuf};
use glob::glob;
use hayagriva::Library;
@ -9,10 +9,10 @@ use crate::html::Linkable;
use super::Sack;
/// Whether the item should be treated as a content page, converted into a standalone HTML page, or
/// as a bundled asset.
/// Marks whether the item should be treated as a content page, converted into a standalone HTML
/// page, or as a bundled asset.
#[derive(Debug)]
pub enum StaticItemKind {
pub enum FileItemKind {
/// Convert to `index.html`
Index,
/// Convert to a bundled asset
@ -21,42 +21,81 @@ pub enum StaticItemKind {
/// Metadata for a single item consumed by SSG.
#[derive(Debug)]
pub struct StaticItem {
pub struct FileItem {
/// Kind of an item
pub kind: StaticItemKind,
/// Original extension for the source file
pub ext: String,
pub dir: Utf8PathBuf,
pub src: Utf8PathBuf,
pub kind: FileItemKind,
/// Original source file location
pub path: Utf8PathBuf,
}
/// Marks how the asset should be processed by the SSG
pub enum AssetKind {
/// Data renderable to HTML
Html(Box<dyn Fn(&Sack) -> String>),
/// Bibliographical data
Bibtex(Library),
/// Images
Image,
Other,
Bib(Library),
}
/// Asset renderable by the SSG
pub struct Asset {
/// Kind of a processed asset
pub kind: AssetKind,
pub out: Utf8PathBuf,
pub meta: StaticItem,
/// File metadata
pub meta: FileItem,
}
/// Dynamically generated asset not related to any disk file.
pub struct Dynamic(pub Box<dyn Fn(&Sack) -> String>);
impl Dynamic {
pub fn new(call: impl Fn(&Sack) -> String + 'static) -> Self {
Self(Box::new(call))
}
}
pub enum OutputKind {
Real(Asset),
Fake(Dynamic),
}
impl From<Asset> for OutputKind {
fn from(value: Asset) -> Self {
OutputKind::Real(value)
}
}
impl From<Dynamic> for OutputKind {
fn from(value: Dynamic) -> Self {
OutputKind::Fake(value)
}
}
/// Renderable output
pub struct Output {
pub kind: OutputKind,
pub path: Utf8PathBuf,
/// Optional link to outputted page.
pub link: Option<Linkable>,
}
/// Variants used for filtering static assets.
pub enum PipelineItem {
Skip(StaticItem),
Take(Asset),
/// Unclaimed file, unrecognized file extensions.
Skip(FileItem),
/// Data ready to be processed by SSG.
Take(Output),
}
impl From<StaticItem> for PipelineItem {
fn from(value: StaticItem) -> Self {
impl From<FileItem> for PipelineItem {
fn from(value: FileItem) -> Self {
Self::Skip(value)
}
}
impl From<Asset> for PipelineItem {
fn from(value: Asset) -> Self {
impl From<Output> for PipelineItem {
fn from(value: Output) -> Self {
Self::Take(value)
}
}
@ -79,28 +118,53 @@ pub fn gather(pattern: &str, exts: &HashSet<&'static str>) -> Vec<PipelineItem>
}
fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> StaticItem {
let dir = path.parent().unwrap();
let ext = path.extension().unwrap();
fn to_source(path: Utf8PathBuf, exts: &HashSet<&'static str>) -> FileItem {
let hit = path.extension().map_or(false, |ext| exts.contains(ext));
if !exts.contains(ext) {
return StaticItem {
kind: StaticItemKind::Bundle,
ext: ext.to_owned(),
dir: dir.to_owned(),
src: path,
};
}
let dirs = match path.file_stem().unwrap() {
"index" => dir.to_owned(),
name => dir.join(name),
let kind = match hit {
true => FileItemKind::Index,
false => FileItemKind::Bundle,
};
StaticItem {
kind: StaticItemKind::Index,
ext: ext.to_owned(),
dir: dirs,
src: path,
FileItem {
kind,
path,
}
}
pub fn render_all(items: &[Output]) {
for item in items {
render(item, &Sack::new(items, &item.path));
}
}
fn render(item: &Output, sack: &Sack) {
let o = Utf8Path::new("dist").join(&item.path);
fs::create_dir_all(o.parent().unwrap()).unwrap();
match item.kind {
OutputKind::Real(ref real) => {
let i = &real.meta.path;
match &real.kind {
AssetKind::Html(closure) => {
let mut file = File::create(&o).unwrap();
file.write_all(closure(sack).as_bytes()).unwrap();
println!("HTML: {} -> {}", i, o);
},
AssetKind::Bibtex(_) => { },
AssetKind::Image => {
fs::create_dir_all(o.parent().unwrap()).unwrap();
fs::copy(i, &o).unwrap();
println!("Image: {} -> {}", i, o);
},
};
},
OutputKind::Fake(Dynamic(ref closure)) => {
let mut file = File::create(&o).unwrap();
file.write_all(closure(sack).as_bytes()).unwrap();
println!("Virtual: -> {}", o);
},
}
}

View file

@ -1,19 +1,17 @@
mod load;
mod render;
mod sack;
use camino::Utf8PathBuf;
use hayagriva::Library;
use hypertext::Renderable;
pub use load::{gather, StaticItem, StaticItemKind, Asset, AssetKind, PipelineItem};
pub use render::{render, Virtual, Item};
pub use load::{gather, render_all, FileItem, FileItemKind, Asset, AssetKind, PipelineItem, Dynamic, Output};
pub use sack::{TreePage, Sack};
use crate::{html::Linkable, text::md::Outline};
/// Represents a piece of content that can be rendered into a page.
/// Represents a piece of content that can be rendered as a page.
pub trait Content {
fn transform<'f, 'm, 's, 'html, T>(
&'f self,

View file

@ -1,99 +0,0 @@
use std::fs::{self, File};
use std::io::Write;
use camino::{Utf8Path, Utf8PathBuf};
use crate::Sack;
use super::{Asset, AssetKind};
pub struct Virtual(pub Utf8PathBuf, pub Box<dyn Fn(&Sack) -> String>);
impl Virtual {
pub fn new<P, F>(path: P, call: F) -> Self
where
P: AsRef<Utf8Path>,
F: Fn(&Sack) -> String + 'static
{
Self(path.as_ref().into(), Box::new(call))
}
}
pub enum Item {
Real(Asset),
Fake(Virtual),
}
impl From<Asset> for Item {
fn from(value: Asset) -> Self {
Item::Real(value)
}
}
impl From<Virtual> for Item {
fn from(value: Virtual) -> Self {
Item::Fake(value)
}
}
pub fn render(items: &[Item]) {
let assets: Vec<&Asset> = items
.iter()
.filter_map(|item| match item {
Item::Real(a) => Some(a),
Item::Fake(_) => None,
})
.collect();
for item in items {
match item {
Item::Real(real) => render_real(real, &Sack::new(&assets, &real.out)),
Item::Fake(fake) => render_fake(fake, &Sack::new(&assets, &fake.0)),
}
}
}
fn render_real(item: &Asset, sack: &Sack) {
match &item.kind {
AssetKind::Html(render) => {
let i = &item.meta.src;
let o = Utf8Path::new("dist").join(&item.out);
fs::create_dir_all(o.parent().unwrap()).unwrap();
let mut file = File::create(&o).unwrap();
file.write_all(render(sack).as_bytes()).unwrap();
println!("HTML: {} -> {}", i, o);
},
AssetKind::Image => {
let i = &item.meta.src;
let o = Utf8Path::new("dist").join(&item.out);
fs::create_dir_all(o.parent().unwrap()).unwrap();
fs::copy(i, &o).unwrap();
println!("Image: {} -> {}", i, o);
},
AssetKind::Bib(_) => (),
AssetKind::Other => {
let i = &item.meta.src;
let o = Utf8Path::new("dist").join(&item.out);
fs::create_dir_all(o.parent().unwrap()).unwrap();
fs::copy(i, &o).unwrap();
println!("Unknown: {} -> {}", i, o);
},
}
}
fn render_fake(item: &Virtual, sack: &Sack) {
let Virtual(out, render) = item;
let o = Utf8Path::new("dist").join(out);
fs::create_dir_all(o.parent().unwrap()).unwrap();
let mut file = File::create(&o).unwrap();
file.write_all(render(sack).as_bytes()).unwrap();
println!("Virtual: -> {}", o);
}

View file

@ -5,7 +5,7 @@ use hayagriva::Library;
use crate::html::{Link, LinkDate, Linkable};
use super::{Asset, AssetKind};
use super::{load::{Output, OutputKind}, AssetKind};
#[derive(Debug)]
@ -34,22 +34,25 @@ impl TreePage {
}
/// This struct allows for querying the website hierarchy.
/// This struct allows for querying the website hierarchy. Separate instance of this struct is
/// passed to each closure contained by some rendered assets.
pub struct Sack<'a> {
assets: &'a [&'a Asset],
/// Literally everything
hole: &'a [Output],
/// Current path for page
path: &'a Utf8PathBuf,
}
impl<'a> Sack<'a> {
pub fn new(assets: &'a [&'a Asset], path: &'a Utf8PathBuf) -> Self {
Self { assets, path }
pub fn new(hole: &'a [Output], path: &'a Utf8PathBuf) -> Self {
Self { hole, path }
}
pub fn get_links(&self, path: &str) -> Vec<LinkDate> {
let pattern = glob::Pattern::new(path).unwrap();
self.assets.iter()
.filter(|f| pattern.matches_path(f.out.as_ref()))
.filter_map(|f| match &f.link {
let pattern = glob::Pattern::new(path).expect("Bad glob pattern");
self.hole.iter()
.filter(|item| pattern.matches_path(item.path.as_ref()))
.filter_map(|item| match &item.link {
Some(Linkable::Date(link)) => Some(link.clone()),
_ => None,
})
@ -57,10 +60,10 @@ impl<'a> Sack<'a> {
}
pub fn get_tree(&self, path: &str) -> TreePage {
let glob = glob::Pattern::new(path).unwrap();
let list = self.assets.iter()
.filter(|f| glob.matches_path(f.out.as_ref()))
.filter_map(|f| match &f.link {
let glob = glob::Pattern::new(path).expect("Bad glob pattern");
let list = self.hole.iter()
.filter(|item| glob.matches_path(item.path.as_ref()))
.filter_map(|item| match &item.link {
Some(Linkable::Link(link)) => Some(link.clone()),
_ => None,
});
@ -75,18 +78,22 @@ impl<'a> Sack<'a> {
pub fn get_library(&self) -> Option<&Library> {
let glob = format!("{}/*.bib", self.path.parent()?);
let glob = glob::Pattern::new(&glob).unwrap();
let glob = glob::Pattern::new(&glob).expect("Bad glob pattern");
let opts = glob::MatchOptions {
case_sensitive: true,
require_literal_separator: true,
require_literal_leading_dot: false,
};
self.assets.iter()
.filter(|asset| glob.matches_path_with(asset.out.as_ref(), opts))
self.hole.iter()
.filter(|item| glob.matches_path_with(item.path.as_ref(), opts))
.filter_map(|asset| match asset.kind {
OutputKind::Real(ref real) => Some(real),
_ => None,
})
.find_map(|asset| match asset.kind {
AssetKind::Bib(ref lib) => Some(lib),
_ => None
AssetKind::Bibtex(ref lib) => Some(lib),
_ => None,
})
}
}

View file

@ -4,7 +4,7 @@ use std::fs;
use camino::{Utf8Path, Utf8PathBuf};
use chrono::Datelike;
use gen::{Asset, AssetKind, Content, PipelineItem, Sack, StaticItemKind};
use gen::{Asset, AssetKind, Content, FileItemKind, Output, PipelineItem, Sack};
use hayagriva::Library;
use html::{Link, LinkDate, Linkable};
use hypertext::{Raw, Renderable};
@ -12,6 +12,8 @@ use once_cell::sync::Lazy;
use serde::Deserialize;
use text::md::Outline;
use crate::gen::Dynamic;
mod md;
mod html;
mod ts;
@ -177,16 +179,23 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
T: for<'de> Deserialize<'de> + Content + 'static,
{
let meta = match item {
PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Index) => meta,
PipelineItem::Skip(meta) if matches!(meta.kind, FileItemKind::Index) => meta,
_ => return item,
};
let dir = meta.dir.strip_prefix("content").unwrap();
match meta.ext.as_str() {
"md" | "mdx" | "lhs" => {
let path = dir.join("index.html");
// FIXME: clean this up
let dir = meta.path.parent().unwrap();
let ext = meta.path.extension().unwrap();
let dir = dir.strip_prefix("content").unwrap();
let dir = match meta.path.file_stem().unwrap() {
"index" => dir.to_owned(),
name => dir.join(name),
};
let path = dir.join("index.html");
let data = fs::read_to_string(&meta.src).unwrap();
match ext {
"md" | "mdx" | "lhs" => {
let data = fs::read_to_string(&meta.path).unwrap();
let (fm, md) = md::preflight::<T>(&data);
let link = T::as_link(&fm, Utf8Path::new("/").join(dir));
@ -196,11 +205,13 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
T::transform(&fm, Raw(html), outline, sack, bib).render().into()
};
gen::Asset {
kind: gen::AssetKind::Html(Box::new(call)),
out: path,
Output {
kind: Asset {
kind: gen::AssetKind::Html(Box::new(call)),
meta,
}.into(),
path,
link,
meta,
}.into()
},
_ => meta.into(),
@ -209,29 +220,33 @@ fn to_index<T>(item: PipelineItem) -> PipelineItem
fn to_bundle(item: PipelineItem) -> PipelineItem {
let meta = match item {
PipelineItem::Skip(meta) if matches!(meta.kind, StaticItemKind::Bundle) => meta,
PipelineItem::Skip(meta) if matches!(meta.kind, FileItemKind::Bundle) => meta,
_ => return item,
};
let dir = meta.dir.strip_prefix("content").unwrap();
let out = dir.join(meta.src.file_name().unwrap()).to_owned();
let dirs = meta.path.strip_prefix("content").unwrap().parent().unwrap();
let path = dirs.join(meta.path.file_name().unwrap()).to_owned();
match meta.ext.as_str() {
"jpg" | "png" | "gif" => gen::Asset {
kind: gen::AssetKind::Image,
out,
match meta.path.extension().unwrap() {
"jpg" | "png" | "gif" => Output {
kind: Asset {
kind: AssetKind::Image,
meta,
}.into(),
path,
link: None,
meta,
}.into(),
"bib" => {
let data = fs::read_to_string(&meta.src).unwrap();
let data = fs::read_to_string(&meta.path).unwrap();
let data = hayagriva::io::from_biblatex_str(&data).unwrap();
Asset {
kind: AssetKind::Bib(data),
out,
Output {
kind: Asset {
kind: AssetKind::Bibtex(data),
meta,
}.into(),
path,
link: None,
meta,
}.into()
},
_ => meta.into(),
@ -247,7 +262,7 @@ fn main() {
fs::create_dir("dist").unwrap();
let assets: Vec<Asset> = vec![
let assets: Vec<Output> = vec![
gen::gather("content/about.md", &["md"].into())
.into_iter()
.map(to_index::<md::Post> as fn(PipelineItem) -> PipelineItem),
@ -266,52 +281,60 @@ fn main() {
.map(to_bundle)
.filter_map(|item| match item {
PipelineItem::Skip(skip) => {
println!("Skipping {}", skip.src);
println!("Skipping {}", skip.path);
None
},
PipelineItem::Take(take) => Some(take),
})
.collect();
let assets: Vec<Vec<gen::Item>> = vec![
assets.into_iter()
.map(Into::into)
.collect(),
let assets: Vec<Output> = vec![
assets,
vec![
gen::Virtual::new("map/index.html", |_| html::map().render().to_owned().into()).into(),
gen::Virtual::new("search/index.html", |_| html::search().render().to_owned().into()).into(),
gen::Asset {
kind: gen::AssetKind::Html(Box::new(|_| {
let data = std::fs::read_to_string("content/index.md").unwrap();
let (_, html, bib) = text::md::parse(&data, None);
html::home(Raw(html)).render().to_owned().into()
})),
out: "index.html".into(),
Output {
kind: Dynamic::new(|_| html::map().render().to_owned().into()).into(),
path: "map/index.html".into(),
link: None,
},
Output {
kind: Dynamic::new(|_| html::search().render().to_owned().into()).into(),
path: "search/index.html".into(),
link: None,
},
Output {
kind: Asset {
kind: gen::AssetKind::Html(Box::new(|_| {
let data = std::fs::read_to_string("content/index.md").unwrap();
let (_, html, bib) = text::md::parse(&data, None);
html::home(Raw(html)).render().to_owned().into()
})).into(),
meta: gen::FileItem {
kind: gen::FileItemKind::Index,
path: "content/index.md".into()
}
}.into(),
path: "index.html".into(),
link: None,
meta: gen::StaticItem {
kind: gen::StaticItemKind::Index,
ext: "md".into(),
dir: "".into(),
src: "content/index.md".into()
}
}.into(),
gen::Virtual("posts/index.html".into(), Box::new(|all|
to_list(all.get_links("posts/**/*.html"))
)).into(),
gen::Virtual("slides/index.html".into(), Box::new(|all|
to_list(all.get_links("slides/**/*.html"))
)).into(),
Output {
kind: Dynamic::new(|sack| to_list(sack.get_links("posts/**/*.html"))).into(),
path: "posts/index.html".into(),
link: None,
},
Output {
kind: Dynamic::new(|sack| to_list(sack.get_links("slides/**/*.html"))).into(),
path: "slides/index.html".into(),
link: None,
},
],
];
let all: Vec<gen::Item> = assets
]
.into_iter()
.flatten()
.collect();
{
let now = std::time::Instant::now();
gen::render(&all);
gen::render_all(&assets);
println!("Elapsed: {:.2?}", now.elapsed());
}