From de896baff7e97fac4dde79078c9a2fa1c652576b Mon Sep 17 00:00:00 2001 From: Minijackson Date: Wed, 18 Dec 2019 20:56:53 +0100 Subject: Big refactoring - entities should be more coherent when parsing multiple files - well defined, language agnostic entity tree - each module has its own configuration - less dead code --- src/generator/config.rs | 53 ++++++++++++ src/generator/mod.rs | 216 ++++++++++++++++++++++++++++++++++++++++++++++++ src/generator/pandoc.rs | 166 +++++++++++++++++++++++++++++++++++++ 3 files changed, 435 insertions(+) create mode 100644 src/generator/config.rs create mode 100644 src/generator/mod.rs create mode 100644 src/generator/pandoc.rs (limited to 'src/generator') diff --git a/src/generator/config.rs b/src/generator/config.rs new file mode 100644 index 0000000..43fee60 --- /dev/null +++ b/src/generator/config.rs @@ -0,0 +1,53 @@ +use serde_derive::{Deserialize, Serialize}; +use structopt::StructOpt; + +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Config { + /// Tells us in which language, which entity types should inline which children entity type in + /// their documentation. + /// + /// e.g. in C++, classes should inline their children methods documentation. + pub(crate) inlines: HashMap>>, +} + +impl Config { + pub(crate) fn from_merge(_cli: ProvidedConfig, conf: ProvidedConfig) -> Self { + Self { + inlines: conf.inlines.unwrap_or_else(default_inlines), + } + } +} + +fn default_inlines() -> HashMap>> { + let mut clang = HashMap::new(); + let mut clang_inline_children = HashSet::new(); + // TODO: this is not great: no differences between variable/field for children, but differences + // as a parent... + clang_inline_children.insert(String::from("variable")); + //classes.insert(String::from("field")); + clang_inline_children.insert(String::from("function")); + //classes.insert(String::from("method")); + clang.insert(String::from("struct"), clang_inline_children.clone()); + clang.insert(String::from("class"), clang_inline_children.clone()); + clang.insert(String::from("namespace"), clang_inline_children); + + let mut inlines = HashMap::new(); + inlines.insert(String::from("clang"), clang); + + inlines +} + +impl Default for Config { + fn default() -> Self { + Config::from_merge(ProvidedConfig::default(), ProvidedConfig::default()) + } +} + +#[derive(Debug, Clone, Default, StructOpt, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct ProvidedConfig { + #[structopt(skip)] + pub(crate) inlines: Option>>>, +} diff --git a/src/generator/mod.rs b/src/generator/mod.rs new file mode 100644 index 0000000..b4e1c94 --- /dev/null +++ b/src/generator/mod.rs @@ -0,0 +1,216 @@ +pub(crate) mod config; +mod pandoc; + +use self::config::Config; +//use crate::entities::{ +// Description, DynEntity, EntitiesManager, EntitiesManagerComponents, Usr, +//}; +use self::pandoc::into_pandoc; +use crate::types::Entity; + +use anyhow::{ensure, Context, Result}; +use rayon::Scope; +use thiserror::Error; + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; + +const DEFAULT_CSS: &[u8] = include_bytes!("../../res/style.css"); + +pub(crate) fn generate(base_dir: &Path, entities: BTreeMap, config: &Config) -> Result<()> { + let md_output_dir = base_dir.join("markdown"); + let html_output_dir = base_dir.join("html"); + + std::fs::create_dir_all(&md_output_dir) + .context("Failed to create the markdown output directory")?; + std::fs::create_dir_all(&html_output_dir) + .context("Failed to create the HTML output directory")?; + + let mut css_tempfile = tempfile::Builder::new() + .prefix("style") + .suffix(".css") + .tempfile()?; + css_tempfile.write_all(DEFAULT_CSS)?; + + let css_path = css_tempfile.path(); + debug!("Generated temporary file with CSS at: {:?}", css_path); + + rayon::scope(|scope| { + for (id, entity) in entities { + generate_recursively( + id, + entity, + scope, + &md_output_dir, + &html_output_dir, + css_path, + config, + ); + } + }); + + Ok(()) +} + +fn generate_recursively<'a>( + id: String, + entity: Entity, + pool: &Scope<'a>, + md_output_dir: &'a Path, + html_output_dir: &'a Path, + css_path: &'a Path, + config: &'a Config, +) { + pool.spawn(move |pool| { + trace!("Trying to generate {}", id); + + let leftovers = generate_single( + &id, + entity, + &md_output_dir, + &html_output_dir, + &css_path, + config, + ) + .unwrap(); + + for (id, entity) in leftovers { + generate_recursively( + id, + entity, + pool, + md_output_dir, + html_output_dir, + css_path, + config, + ); + } + }); +} + +fn generate_single( + id: &str, + entity: Entity, + md_output_dir: impl AsRef, + html_output_dir: impl AsRef, + css_path: impl AsRef, + config: &Config +) -> Result> { + use std::io::prelude::*; + use std::process::{Command, Stdio}; + + let md_output_file = md_output_dir.as_ref().join(id); + let html_output_file = html_output_dir.as_ref().join(id); + + let mut command = Command::new("pandoc"); + + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(&[ + "--from=json", + "--to=markdown", + "--output", + md_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + ]); + + debug!("Launching command: {:?}", command); + + let mut pandoc = command + .spawn() + .context("Failed to execute Pandoc command")?; + + let (pandoc_ast, leftovers) = into_pandoc(entity, config); + + if log_enabled!(log::Level::Trace) { + let json = + serde_json::to_string(&pandoc_ast).context("Failed to convert Pandoc AST to JSON")?; + trace!("Sending json: {}", json); + write!( + pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), + "{}", + &json, + ) + .context("Failed to convert Pandoc AST to JSON")?; + } else { + serde_json::to_writer( + pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), + &pandoc_ast, + ) + .context("Failed to convert Pandoc AST to JSON")?; + } + + let command_str = format!("{:?}", pandoc); + + let output = pandoc + .wait_with_output() + .expect("Pandoc command wasn't running"); + + ensure!( + output.status.success(), + CommandError { + command: format!("{:?}", command_str), + status: output.status, + stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), + } + ); + + let mut command = Command::new("pandoc"); + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(&[ + "--from=markdown-raw_tex", + "--to=html", + "--css", + css_path + .as_ref() + .to_str() + .context("CSS path is not valid UTF-8")?, + "--standalone", + "--self-contained", + md_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + "--output", + html_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + ]); + + let command_str = format!("{:?}", command); + debug!("Launching command: {}", command_str); + + let output = command + .output() + .context("Failed to execute Pandoc command")?; + + ensure!( + output.status.success(), + CommandError { + command: command_str, + status: output.status, + stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), + } + ); + + Ok(leftovers) +} + +#[derive(Debug, Clone, Error)] +#[error( + "Command {} returned status {:?} and stderr {:?}", + command, + status, + stderr +)] +struct CommandError { + command: String, + status: std::process::ExitStatus, + stderr: String, +} diff --git a/src/generator/pandoc.rs b/src/generator/pandoc.rs new file mode 100644 index 0000000..1753d34 --- /dev/null +++ b/src/generator/pandoc.rs @@ -0,0 +1,166 @@ +use super::config::Config; +use crate::types::Entity; + +use pandoc_types::definition::{Attr, Block, Inline, Meta, MetaValue, Pandoc}; + +use std::collections::BTreeMap; + +pub(crate) fn into_pandoc( + entity: Entity, + config: &Config, +) -> (Pandoc, BTreeMap) { + let mut meta = Meta::null(); + + let title = vec![Inline::Code(Attr::null(), entity.name.clone())]; + + meta.0 + .insert("title".to_string(), MetaValue::MetaString(entity.name)); + + let mut content = Vec::new(); + + content.push(Block::Header(1, Attr::null(), title)); + + if !entity.documentation.is_empty() { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(String::from("Description"))], + )); + + content.push(Block::Div( + Attr(String::new(), vec![String::from("doc")], vec![]), + vec![raw_markdown(entity.documentation.clone())], + )); + } + + let mut inline_children = BTreeMap::new(); + let mut separate_children = BTreeMap::new(); + + let entity_kind = entity.kind; + for (children_kind, children) in entity.children { + if config + .inlines + .get(&entity.language) + .and_then(|lang_inlines| lang_inlines.get(&entity_kind)) + .map(|children_inlines| children_inlines.contains(&children_kind)) + == Some(true) + { + inline_children.insert(children_kind, children); + } else { + // By default, do not inline + separate_children.insert(children_kind, children); + } + } + + for (section_name, children) in &separate_children { + if let Some(members_list) = member_list(children) { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(String::from(section_name))], + )); + + content.push(members_list); + } + } + + let mut embedded_documentation = Vec::new(); + + for (section_name, children) in inline_children { + if let Some(members_list) = member_list(&children) { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(section_name.clone())], + )); + + content.push(members_list); + + embedded_documentation.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(section_name + " Documentation")], + )); + + for (_id, child) in children { + embedded_documentation.push(Block::Header( + 3, + Attr::null(), + vec![Inline::Code(Attr::null(), child.name)], + )); + + embedded_documentation.push(Block::Div( + Attr(String::new(), vec![String::from("doc")], vec![]), + vec![raw_markdown(child.documentation)], + )); + } + } + } + + content.append(&mut embedded_documentation); + + let leftovers = separate_children + .into_iter() + .map(|(_section_name, children)| children) + .flatten() + .collect(); + + (Pandoc(meta, content), leftovers) +} + +fn str_block(content: String) -> Block { + Block::Plain(vec![Inline::Str(content)]) +} + +fn entity_link(id: &str, name: String) -> Inline { + use pandoc_types::definition::Target; + use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; + + use std::iter::once; + + // https://url.spec.whatwg.org/#fragment-percent-encode-set + const FRAGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'<') + .add(b'>') + .add(b'`') + .add(b'#') + .add(b'?') + .add(b'{') + .add(b'}'); + + Inline::Link( + Attr::null(), + vec![Inline::Code(Attr::null(), name)], + Target( + once("./") + .chain(utf8_percent_encode(id, FRAGMENT)) + .collect(), + String::new(), + ), + ) +} + +fn raw_markdown(text: String) -> Block { + use pandoc_types::definition::Format; + Block::RawBlock(Format(String::from("markdown")), text) +} + +fn member_list<'a>(members: impl IntoIterator) -> Option { + let definitions: Vec<(Vec, Vec>)> = members + .into_iter() + .map(|(id, entity)| { + ( + vec![entity_link(id, entity.name.clone())], + vec![vec![str_block(entity.brief_description.clone())]], + ) + }) + .collect(); + + if definitions.is_empty() { + None + } else { + Some(Block::DefinitionList(definitions)) + } +} -- cgit v1.2.3