Big refactoring

- entities should be more coherent when parsing multiple files
- well defined, language agnostic entity tree
- each module has its own configuration
- less dead code

3 files changed, 435 insertions, 0 deletions

diff --git a/src/generator/config.rs b/src/generator/config.rs
new file mode 100644
index 0000000..43fee60
--- /dev/null
+++ b/src/generator/config.rs
@@ -0,0 +1,53 @@
+use serde_derive::{Deserialize, Serialize};
+use structopt::StructOpt;
+
+use std::collections::{HashMap, HashSet};
+
+#[derive(Debug, Clone, Serialize)]
+pub(crate) struct Config {
+    /// Tells us in which language, which entity types should inline which children entity type in
+    /// their documentation.
+    ///
+    /// e.g. in C++, classes should inline their children methods documentation.
+    pub(crate) inlines: HashMap<String, HashMap<String, HashSet<String>>>,
+}
+
+impl Config {
+    pub(crate) fn from_merge(_cli: ProvidedConfig, conf: ProvidedConfig) -> Self {
+        Self {
+            inlines: conf.inlines.unwrap_or_else(default_inlines),
+        }
+    }
+}
+
+fn default_inlines() -> HashMap<String, HashMap<String, HashSet<String>>> {
+    let mut clang = HashMap::new();
+    let mut clang_inline_children = HashSet::new();
+    // TODO: this is not great: no differences between variable/field for children, but differences
+    // as a parent...
+    clang_inline_children.insert(String::from("variable"));
+    //classes.insert(String::from("field"));
+    clang_inline_children.insert(String::from("function"));
+    //classes.insert(String::from("method"));
+    clang.insert(String::from("struct"), clang_inline_children.clone());
+    clang.insert(String::from("class"), clang_inline_children.clone());
+    clang.insert(String::from("namespace"), clang_inline_children);
+
+    let mut inlines = HashMap::new();
+    inlines.insert(String::from("clang"), clang);
+
+    inlines
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Config::from_merge(ProvidedConfig::default(), ProvidedConfig::default())
+    }
+}
+
+#[derive(Debug, Clone, Default, StructOpt, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct ProvidedConfig {
+    #[structopt(skip)]
+    pub(crate) inlines: Option<HashMap<String, HashMap<String, HashSet<String>>>>,
+}
diff --git a/src/generator/mod.rs b/src/generator/mod.rs
new file mode 100644
index 0000000..b4e1c94
--- /dev/null
+++ b/src/generator/mod.rs
@@ -0,0 +1,216 @@
+pub(crate) mod config;
+mod pandoc;
+
+use self::config::Config;
+//use crate::entities::{
+//    Description, DynEntity, EntitiesManager, EntitiesManagerComponents, Usr,
+//};
+use self::pandoc::into_pandoc;
+use crate::types::Entity;
+
+use anyhow::{ensure, Context, Result};
+use rayon::Scope;
+use thiserror::Error;
+
+use std::collections::BTreeMap;
+use std::io::Write;
+use std::path::Path;
+
+const DEFAULT_CSS: &[u8] = include_bytes!("../../res/style.css");
+
+pub(crate) fn generate(base_dir: &Path, entities: BTreeMap<String, Entity>, config: &Config) -> Result<()> {
+    let md_output_dir = base_dir.join("markdown");
+    let html_output_dir = base_dir.join("html");
+
+    std::fs::create_dir_all(&md_output_dir)
+        .context("Failed to create the markdown output directory")?;
+    std::fs::create_dir_all(&html_output_dir)
+        .context("Failed to create the HTML output directory")?;
+
+    let mut css_tempfile = tempfile::Builder::new()
+        .prefix("style")
+        .suffix(".css")
+        .tempfile()?;
+    css_tempfile.write_all(DEFAULT_CSS)?;
+
+    let css_path = css_tempfile.path();
+    debug!("Generated temporary file with CSS at: {:?}", css_path);
+
+    rayon::scope(|scope| {
+        for (id, entity) in entities {
+            generate_recursively(
+                id,
+                entity,
+                scope,
+                &md_output_dir,
+                &html_output_dir,
+                css_path,
+                config,
+            );
+        }
+    });
+
+    Ok(())
+}
+
+fn generate_recursively<'a>(
+    id: String,
+    entity: Entity,
+    pool: &Scope<'a>,
+    md_output_dir: &'a Path,
+    html_output_dir: &'a Path,
+    css_path: &'a Path,
+    config: &'a Config,
+) {
+    pool.spawn(move |pool| {
+        trace!("Trying to generate {}", id);
+
+        let leftovers = generate_single(
+            &id,
+            entity,
+            &md_output_dir,
+            &html_output_dir,
+            &css_path,
+            config,
+        )
+        .unwrap();
+
+        for (id, entity) in leftovers {
+            generate_recursively(
+                id,
+                entity,
+                pool,
+                md_output_dir,
+                html_output_dir,
+                css_path,
+                config,
+            );
+        }
+    });
+}
+
+fn generate_single(
+    id: &str,
+    entity: Entity,
+    md_output_dir: impl AsRef<Path>,
+    html_output_dir: impl AsRef<Path>,
+    css_path: impl AsRef<Path>,
+    config: &Config
+) -> Result<BTreeMap<String, Entity>> {
+    use std::io::prelude::*;
+    use std::process::{Command, Stdio};
+
+    let md_output_file = md_output_dir.as_ref().join(id);
+    let html_output_file = html_output_dir.as_ref().join(id);
+
+    let mut command = Command::new("pandoc");
+
+    command
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .args(&[
+            "--from=json",
+            "--to=markdown",
+            "--output",
+            md_output_file
+                .to_str()
+                .context("Entity name is not valid UTF-8")?,
+        ]);
+
+    debug!("Launching command: {:?}", command);
+
+    let mut pandoc = command
+        .spawn()
+        .context("Failed to execute Pandoc command")?;
+
+    let (pandoc_ast, leftovers) = into_pandoc(entity, config);
+
+    if log_enabled!(log::Level::Trace) {
+        let json =
+            serde_json::to_string(&pandoc_ast).context("Failed to convert Pandoc AST to JSON")?;
+        trace!("Sending json: {}", json);
+        write!(
+            pandoc.stdin.as_mut().expect("Pandoc stdin not captured"),
+            "{}",
+            &json,
+        )
+        .context("Failed to convert Pandoc AST to JSON")?;
+    } else {
+        serde_json::to_writer(
+            pandoc.stdin.as_mut().expect("Pandoc stdin not captured"),
+            &pandoc_ast,
+        )
+        .context("Failed to convert Pandoc AST to JSON")?;
+    }
+
+    let command_str = format!("{:?}", pandoc);
+
+    let output = pandoc
+        .wait_with_output()
+        .expect("Pandoc command wasn't running");
+
+    ensure!(
+        output.status.success(),
+        CommandError {
+            command: format!("{:?}", command_str),
+            status: output.status,
+            stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"),
+        }
+    );
+
+    let mut command = Command::new("pandoc");
+    command
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .args(&[
+            "--from=markdown-raw_tex",
+            "--to=html",
+            "--css",
+            css_path
+                .as_ref()
+                .to_str()
+                .context("CSS path is not valid UTF-8")?,
+            "--standalone",
+            "--self-contained",
+            md_output_file
+                .to_str()
+                .context("Entity name is not valid UTF-8")?,
+            "--output",
+            html_output_file
+                .to_str()
+                .context("Entity name is not valid UTF-8")?,
+        ]);
+
+    let command_str = format!("{:?}", command);
+    debug!("Launching command: {}", command_str);
+
+    let output = command
+        .output()
+        .context("Failed to execute Pandoc command")?;
+
+    ensure!(
+        output.status.success(),
+        CommandError {
+            command: command_str,
+            status: output.status,
+            stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"),
+        }
+    );
+
+    Ok(leftovers)
+}
+
+#[derive(Debug, Clone, Error)]
+#[error(
+    "Command {} returned status {:?} and stderr {:?}",
+    command,
+    status,
+    stderr
+)]
+struct CommandError {
+    command: String,
+    status: std::process::ExitStatus,
+    stderr: String,
+}
diff --git a/src/generator/pandoc.rs b/src/generator/pandoc.rs
new file mode 100644
index 0000000..1753d34
--- /dev/null
+++ b/src/generator/pandoc.rs
@@ -0,0 +1,166 @@
+use super::config::Config;
+use crate::types::Entity;
+
+use pandoc_types::definition::{Attr, Block, Inline, Meta, MetaValue, Pandoc};
+
+use std::collections::BTreeMap;
+
+pub(crate) fn into_pandoc(
+    entity: Entity,
+    config: &Config,
+) -> (Pandoc, BTreeMap<String, Entity>) {
+    let mut meta = Meta::null();
+
+    let title = vec![Inline::Code(Attr::null(), entity.name.clone())];
+
+    meta.0
+        .insert("title".to_string(), MetaValue::MetaString(entity.name));
+
+    let mut content = Vec::new();
+
+    content.push(Block::Header(1, Attr::null(), title));
+
+    if !entity.documentation.is_empty() {
+        content.push(Block::Header(
+            2,
+            Attr::null(),
+            vec![Inline::Str(String::from("Description"))],
+        ));
+
+        content.push(Block::Div(
+            Attr(String::new(), vec![String::from("doc")], vec![]),
+            vec![raw_markdown(entity.documentation.clone())],
+        ));
+    }
+
+    let mut inline_children = BTreeMap::new();
+    let mut separate_children = BTreeMap::new();
+
+    let entity_kind = entity.kind;
+    for (children_kind, children) in entity.children {
+        if config
+            .inlines
+            .get(&entity.language)
+            .and_then(|lang_inlines| lang_inlines.get(&entity_kind))
+            .map(|children_inlines| children_inlines.contains(&children_kind))
+            == Some(true)
+        {
+            inline_children.insert(children_kind, children);
+        } else {
+            // By default, do not inline
+            separate_children.insert(children_kind, children);
+        }
+    }
+
+    for (section_name, children) in &separate_children {
+        if let Some(members_list) = member_list(children) {
+            content.push(Block::Header(
+                2,
+                Attr::null(),
+                vec![Inline::Str(String::from(section_name))],
+            ));
+
+            content.push(members_list);
+        }
+    }
+
+    let mut embedded_documentation = Vec::new();
+
+    for (section_name, children) in inline_children {
+        if let Some(members_list) = member_list(&children) {
+            content.push(Block::Header(
+                2,
+                Attr::null(),
+                vec![Inline::Str(section_name.clone())],
+            ));
+
+            content.push(members_list);
+
+            embedded_documentation.push(Block::Header(
+                2,
+                Attr::null(),
+                vec![Inline::Str(section_name + " Documentation")],
+            ));
+
+            for (_id, child) in children {
+                embedded_documentation.push(Block::Header(
+                    3,
+                    Attr::null(),
+                    vec![Inline::Code(Attr::null(), child.name)],
+                ));
+
+                embedded_documentation.push(Block::Div(
+                    Attr(String::new(), vec![String::from("doc")], vec![]),
+                    vec![raw_markdown(child.documentation)],
+                ));
+            }
+        }
+    }
+
+    content.append(&mut embedded_documentation);
+
+    let leftovers = separate_children
+        .into_iter()
+        .map(|(_section_name, children)| children)
+        .flatten()
+        .collect();
+
+    (Pandoc(meta, content), leftovers)
+}
+
+fn str_block(content: String) -> Block {
+    Block::Plain(vec![Inline::Str(content)])
+}
+
+fn entity_link(id: &str, name: String) -> Inline {
+    use pandoc_types::definition::Target;
+    use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
+
+    use std::iter::once;
+
+    // https://url.spec.whatwg.org/#fragment-percent-encode-set
+    const FRAGMENT: &AsciiSet = &CONTROLS
+        .add(b' ')
+        .add(b'"')
+        .add(b'<')
+        .add(b'>')
+        .add(b'`')
+        .add(b'#')
+        .add(b'?')
+        .add(b'{')
+        .add(b'}');
+
+    Inline::Link(
+        Attr::null(),
+        vec![Inline::Code(Attr::null(), name)],
+        Target(
+            once("./")
+                .chain(utf8_percent_encode(id, FRAGMENT))
+                .collect(),
+            String::new(),
+        ),
+    )
+}
+
+fn raw_markdown(text: String) -> Block {
+    use pandoc_types::definition::Format;
+    Block::RawBlock(Format(String::from("markdown")), text)
+}
+
+fn member_list<'a>(members: impl IntoIterator<Item = (&'a String, &'a Entity)>) -> Option<Block> {
+    let definitions: Vec<(Vec<Inline>, Vec<Vec<Block>>)> = members
+        .into_iter()
+        .map(|(id, entity)| {
+            (
+                vec![entity_link(id, entity.name.clone())],
+                vec![vec![str_block(entity.brief_description.clone())]],
+            )
+        })
+        .collect();
+
+    if definitions.is_empty() {
+        None
+    } else {
+        Some(Block::DefinitionList(definitions))
+    }
+}