From de896baff7e97fac4dde79078c9a2fa1c652576b Mon Sep 17 00:00:00 2001 From: Minijackson Date: Wed, 18 Dec 2019 20:56:53 +0100 Subject: Big refactoring - entities should be more coherent when parsing multiple files - well defined, language agnostic entity tree - each module has its own configuration - less dead code --- Cargo.toml | 2 +- shell.nix | 6 +- src/cli.rs | 7 +- src/config.rs | 89 +++-- src/doxygen.rs | 454 ------------------------- src/doxygen/builders.rs | 109 ------ src/entities.rs | 216 ------------ src/generator.rs | 219 ------------- src/generator/config.rs | 53 +++ src/generator/mod.rs | 216 ++++++++++++ src/generator/pandoc.rs | 166 ++++++++++ src/main.rs | 19 +- src/pandoc.rs | 166 ---------- src/parsing.rs | 482 --------------------------- src/parsing/clang/config.rs | 53 +++ src/parsing/clang/entities.rs | 369 +++++++++++++++++++++ src/parsing/clang/mod.rs | 5 + src/parsing/clang/parsing.rs | 748 ++++++++++++++++++++++++++++++++++++++++++ src/parsing/mod.rs | 1 + src/types.rs | 22 ++ 20 files changed, 1687 insertions(+), 1715 deletions(-) delete mode 100644 src/doxygen.rs delete mode 100644 src/doxygen/builders.rs delete mode 100644 src/entities.rs delete mode 100644 src/generator.rs create mode 100644 src/generator/config.rs create mode 100644 src/generator/mod.rs create mode 100644 src/generator/pandoc.rs delete mode 100644 src/pandoc.rs delete mode 100644 src/parsing.rs create mode 100644 src/parsing/clang/config.rs create mode 100644 src/parsing/clang/entities.rs create mode 100644 src/parsing/clang/mod.rs create mode 100644 src/parsing/clang/parsing.rs create mode 100644 src/parsing/mod.rs create mode 100644 src/types.rs diff --git a/Cargo.toml b/Cargo.toml index 9572fa9..b01238f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] anyhow = "1" -clang = "0.23" +clang = { version = "0.23", features = ["clang_8_0"] } codemap = "0.1" codemap-diagnostic = "0.1" log = "0.4" diff --git a/shell.nix b/shell.nix index 5368dcd..5e370ca 100644 --- a/shell.nix +++ b/shell.nix @@ -4,9 +4,9 @@ let clang = pkgs.llvmPackages_latest.clang; mkWrapper = name: mode: pkgs.writeShellScriptBin name '' ./target/${mode}/poseidoc \ - '--extra-arg=-idirafter "${clang.libc_dev}/include"' \ - --extra-arg="$(echo -n "${clang.default_cxx_stdlib_compile}")" \ - '--extra-arg="-resource-dir=${clang}/resource-root"' \ + '--clang-extra-args=-idirafter "${clang.libc_dev}/include"' \ + --clang-extra-args="$(echo -n "${clang.default_cxx_stdlib_compile}")" \ + '--clang-extra-args="-resource-dir=${clang}/resource-root"' \ "$@" ''; diff --git a/src/cli.rs b/src/cli.rs index 0a883af..ff25792 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,4 +1,4 @@ -use crate::config::Config; +use crate::config::ProvidedConfig; use structopt::StructOpt; @@ -9,9 +9,6 @@ pub(crate) struct Cli { #[structopt(long, short, parse(from_occurrences))] pub(crate) verbosity: u8, - #[structopt(long, number_of_values = 1, parse(try_from_str = shell_words::split))] - pub(crate) extra_arg: Vec>, - #[structopt(long, short = "C", default_value = ".")] pub(crate) directory: PathBuf, @@ -19,7 +16,7 @@ pub(crate) struct Cli { pub(crate) command: Command, #[structopt(flatten)] - pub(crate) common_options: Config, + pub(crate) common_options: ProvidedConfig, } #[derive(Debug, Clone, StructOpt)] diff --git a/src/config.rs b/src/config.rs index b448469..72a9ff6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,8 @@ use crate::cli::Cli; +use crate::generator::config::{ + Config as GeneratorConfig, ProvidedConfig as ProvidedGeneratorConfig, +}; +use crate::parsing::clang::config::{Config as ClangConfig, ProvidedConfig as ProvidedClangConfig}; use anyhow::{anyhow, Context, Result}; use codemap::CodeMap; @@ -12,71 +16,56 @@ use std::sync::Arc; pub(super) const DEFAULT_PROJECT_CONFIGURATION_FILE_NAME: &str = "poseidoc.toml"; -#[derive(Debug, Clone, StructOpt, Deserialize, Serialize)] -#[structopt(rename_all = "kebab-case")] -#[serde(rename_all = "kebab-case", default)] +#[derive(Debug, Clone, Serialize)] pub(crate) struct Config { - #[structopt(skip)] - pub(crate) name: Option, - #[structopt(long)] - pub(crate) compile_commands_location: Option, - #[structopt(skip = vec![])] - pub(crate) extra_clang_args: Vec, - #[structopt(flatten)] - pub(crate) class: ClassConfig, -} - -impl Config { - pub(crate) fn merge_cli(self, cli: Cli) -> Self { - Config { - name: cli.common_options.name.or(self.name), - compile_commands_location: cli - .common_options - .compile_commands_location - .or(self.compile_commands_location), - extra_clang_args: cli - .extra_arg - .into_iter() - .flatten() - .chain(self.extra_clang_args) - .collect(), - class: self.class.merge_cli(cli.common_options.class), - } - } + pub(crate) name: String, + pub(crate) clang: ClangConfig, + pub(crate) generator: GeneratorConfig, } impl Default for Config { fn default() -> Self { Config { - name: Some(String::from("My Project")), - compile_commands_location: Some(PathBuf::from(r".")), - extra_clang_args: Vec::new(), - class: ClassConfig::default(), + name: String::from("My Project"), + clang: ClangConfig::default(), + generator: GeneratorConfig::default(), } } } -#[derive(Debug, Clone, StructOpt, Deserialize, Serialize)] -#[structopt(rename_all = "kebab-case")] -#[serde(rename_all = "kebab-case", default)] -pub(crate) struct ClassConfig { -} - -impl ClassConfig { - fn merge_cli(self, _cli: ClassConfig) -> Self { - ClassConfig { - } +impl Config { + pub(crate) fn from_merge(cli: Cli, conf: ProvidedConfig) -> Result { + Ok(Config { + name: cli + .common_options + .name + .or(conf.name) + .unwrap_or_else(|| String::from("My Project")), + clang: ClangConfig::from_merge(cli.common_options.clang, conf.clang)?, + generator: GeneratorConfig::from_merge(cli.common_options.generator, conf.generator), + }) } } -impl Default for ClassConfig { - fn default() -> Self { - ClassConfig { - } - } +/// Same as ProvidedConfig but with options for when not user-provided +#[derive(Debug, Clone, Default, StructOpt, Deserialize)] +#[structopt(rename_all = "kebab-case")] +#[serde(rename_all = "kebab-case")] +pub(crate) struct ProvidedConfig { + #[structopt(skip)] + pub(crate) name: Option, + #[structopt(flatten)] + #[serde(default)] + pub(crate) clang: ProvidedClangConfig, + #[structopt(skip)] + #[serde(default)] + pub(crate) generator: ProvidedGeneratorConfig, } -pub(super) fn load_config(location: impl AsRef, codemap: &mut CodeMap) -> Result { +pub(super) fn load_config( + location: impl AsRef, + codemap: &mut CodeMap, +) -> Result { let location = location.as_ref(); let final_path = if location.is_dir() { diff --git a/src/doxygen.rs b/src/doxygen.rs deleted file mode 100644 index 8d2267d..0000000 --- a/src/doxygen.rs +++ /dev/null @@ -1,454 +0,0 @@ -/// A 'recursive descent' parser for Doxygen XML files -mod builders; - -use xml::{ - attribute::OwnedAttribute, - reader::{EventReader, ParserConfig, XmlEvent}, -}; - -use std::collections::HashMap; -use std::io::Read; - -// === Types === - -#[derive(Debug, Clone)] -pub(crate) enum DoxygenType { - NameSpace(NameSpace), - Class(Class), - Unhandled, -} - -#[derive(Debug, Clone)] -pub(crate) struct NameSpace { - pub(crate) name: String, - pub(crate) members: Vec, - pub(crate) brief_description: Description, - pub(crate) detailed_description: Description, -} - -#[derive(Debug, Clone)] -pub(crate) struct FunctionArgument { - pub(crate) name: String, - pub(crate) r#type: String, - pub(crate) default_value: Option, -} - -#[derive(Debug, Clone)] -pub(crate) enum Member { - Variable { - name: String, - r#type: String, - }, - // What - /* - Enum { - - } - */ - Function { - name: String, - args: Vec, - return_type: String, - }, - Unhandled, -} - -#[derive(Debug, Clone)] -pub(crate) struct Class { - pub(crate) name: String, - pub(crate) inners: Vec<(String, String)>, - pub(crate) sections: HashMap>, - pub(crate) brief_description: Description, - pub(crate) detailed_description: Description, -} - -// === Description === - -#[derive(Debug, Clone)] -pub(crate) struct Description { - pub(crate) inner: Vec, -} - -impl Description { - fn new() -> Self { - Self { inner: Vec::new() } - } -} - -#[derive(Debug, Clone)] -pub(crate) enum DescriptionNode { - Text(String), - Title(String), - Para(Paragraph), - Sect1(String), - Internal(String), -} - -#[derive(Debug, Clone)] -pub(crate) struct Paragraph { - pub(crate) inner: Vec, -} - -#[derive(Debug, Clone)] -pub(crate) enum ParagraphNode { - Text(String), - Ref(Ref), -} - -#[derive(Debug, Clone)] -pub(crate) struct Ref { - pub(crate) id: String, - pub(crate) kind: RefKind, - pub(crate) text: String, -} - -#[derive(Debug, Clone)] -pub(crate) enum RefKind { - Compound, - Member, -} - -#[derive(Debug, Clone)] -enum DoxygenTypeKind { - NameSpace, - Unhandled, -} - -trait TypeBuilder { - //const KIND: DoxygenTypeKind; - - fn build(self: Box) -> DoxygenType; - - fn name(&mut self, value: String); - fn brief_description(&mut self, description: Description); - fn detailed_description(&mut self, description: Description); - - fn add_inner(&mut self, name: String, kind: String); - fn add_member(&mut self, section: String, member: Member); -} - -trait FromXML { - fn from_xml(attributes: Vec, event_reader: &mut EventReader) - -> Self; -} - -impl FromXML for Description { - fn from_xml( - _attributes: Vec, - mut event_reader: &mut EventReader, - ) -> Self { - let mut inner = Vec::new(); - - while let Ok(event) = event_reader.next() { - match event { - XmlEvent::Characters(text) => inner.push(DescriptionNode::Text(text)), - XmlEvent::StartElement { - name, attributes, .. - } => inner.push(match name.local_name.as_str() { - "para" => { - DescriptionNode::Para(Paragraph::from_xml(attributes, &mut event_reader)) - } - "title" => DescriptionNode::Title(event_simple(&mut event_reader)), - "sect1" => DescriptionNode::Sect1(event_simple(&mut event_reader)), - "internal" => DescriptionNode::Internal(event_simple(&mut event_reader)), - other => { - warn!("Description element not supported: {}", other); - continue; - } - }), - XmlEvent::EndElement { .. } => break, - other => { - warn!("Description event not supported: {:?}", other); - } - } - } - - Self { inner } - } -} - -impl FromXML for Paragraph { - fn from_xml( - _attributes: Vec, - mut event_reader: &mut EventReader, - ) -> Self { - let mut inner = Vec::new(); - - while let Ok(event) = event_reader.next() { - match event { - XmlEvent::Characters(text) => inner.push(ParagraphNode::Text(text)), - XmlEvent::StartElement { - name, attributes, .. - } => inner.push(match name.local_name.as_str() { - "ref" => ParagraphNode::Ref(Ref::from_xml(attributes, &mut event_reader)), - other => { - warn!("Paragraph element not supported: {}", other); - continue; - } - }), - XmlEvent::EndElement { .. } => break, - other => { - warn!("Paragraph event not supported: {:?}", other); - } - } - } - - Self { inner } - } -} - -impl FromXML for Ref { - fn from_xml( - attributes: Vec, - mut event_reader: &mut EventReader, - ) -> Self { - let mut id = None; - let mut kind = None; - - for attr in attributes.into_iter() { - match attr.name.local_name.as_str() { - "refid" => id = Some(attr.value), - "kindref" => { - kind = Some(match attr.value.as_str() { - "compound" => RefKind::Compound, - "member" => RefKind::Member, - other => { - warn!("Ref kind not supported: {}", other); - RefKind::Compound - } - }); - } - other => { - warn!("Ref element not supported: {}", other); - } - } - } - - let text = event_simple(&mut event_reader); - - Ref { - id: id.unwrap(), - kind: kind.unwrap(), - text, - } - } -} - -/// Parse a type from a XML Doxygen file -pub(crate) fn parse_type(reader: impl Read) -> Vec { - let mut event_reader = EventReader::new(reader); - - match event_reader.next().unwrap() { - XmlEvent::StartDocument { .. } => (), - _ => panic!("XML file does not begin with document"), - } - - match event_reader.next().unwrap() { - XmlEvent::StartElement { name, .. } => { - if name.local_name != "doxygen" { - panic!("XML file does not start with a 'doxygen' element"); - } - } - _ => panic!("XML file does not start with a 'doxygen' element"), - } - - let mut result = Vec::new(); - - while let Ok(event) = event_reader.next() { - match dbg!(event) { - XmlEvent::StartElement { - name, attributes, .. - } => { - if name.local_name != "compounddef" { - panic!("'doxygen' is not a sequence of 'compounddef' elements"); - } - - result.push(event_compounddef(attributes, &mut event_reader)); - } - XmlEvent::Whitespace(_) => (), - XmlEvent::EndElement { .. } => break, - _ => panic!("'doxygen' is not a sequence of 'compounddef' elements"), - } - } - - // Unnecessarily strict? - match event_reader.next().unwrap() { - XmlEvent::EndDocument => (), - _ => panic!("XML file does not end after 'doxygen' element"), - } - - dbg!(result) -} - -fn event_compounddef( - attributes: Vec, - mut event_reader: &mut EventReader, -) -> DoxygenType { - let kind = attributes - .into_iter() - .find_map(|attr| { - if attr.name.local_name == "kind" { - Some(attr.value) - } else { - None - } - }) - .unwrap(); - - let mut builder = builders::builder_for(kind); - - while let Ok(event) = event_reader.next() { - match dbg!(event) { - XmlEvent::StartElement { - name, attributes, .. - } => match name.local_name.as_str() { - "compoundname" => { - let name = event_simple(&mut event_reader); - builder.name(name); - } - "briefdescription" => { - let brief_description = Description::from_xml(attributes, &mut event_reader); - builder.brief_description(brief_description); - } - "detaileddescription" => { - let detailed_description = Description::from_xml(attributes, &mut event_reader); - builder.detailed_description(detailed_description); - } - "location" => { - event_simple(&mut event_reader); - debug!("Do something?"); - } - "sectiondef" => { - event_section(&mut builder, &mut event_reader); - } - other_name if is_inner_type(other_name) => { - let inner = event_simple(&mut event_reader); - builder.add_inner(inner, other_name.to_string()); - } - _other_name => { - event_ignore(&mut event_reader); - } - }, - XmlEvent::Whitespace(_) => (), - XmlEvent::EndElement { .. } => break, - _ => panic!("Unhandled XML event"), - } - } - - builder.build() -} - -/// Returns true if the given XML Element is a reference to another inner type. -/// -/// Corresponds to the "refType" XSD type in `compound.xsd` -fn is_inner_type(element_name: &str) -> bool { - match element_name { - "innerdir" | "innerfile" | "innerclass" | "innernamespace" | "innerpage" | "innergroup" => { - true - } - _ => false, - } -} - -/// Get the text inside a simple, non recursive XML tag -fn event_simple(event_reader: &mut EventReader) -> String { - let result; - - match dbg!(event_reader.next().unwrap()) { - XmlEvent::Characters(tmp_result) => { - result = tmp_result; - } - XmlEvent::EndElement { .. } => return "".to_string(), - _ => panic!("Simple XML event is not so simple"), - } - - match dbg!(event_reader.next().unwrap()) { - XmlEvent::EndElement { .. } => (), - _ => panic!("Simple XML event is not so simple"), - } - - result -} - -fn event_ignore(mut event_reader: &mut EventReader) { - loop { - let event = event_reader.next().unwrap(); - match dbg!(event) { - XmlEvent::StartElement { .. } => event_ignore(&mut event_reader), - XmlEvent::EndElement { .. } => break, - _ => (), - } - } -} - -fn event_section( - mut builder: &mut Box, - mut event_reader: &mut EventReader, -) { - loop { - let event = event_reader.next().unwrap(); - match dbg!(event) { - XmlEvent::StartElement { - name, attributes, .. - } => match name.local_name.as_str() { - "memberdef" => { - let member = event_member(attributes, &mut event_reader); - builder.add_member("bla".to_owned(), member); - } - _ => panic!("Unhandled thing"), - }, - - XmlEvent::Whitespace(_) => (), - XmlEvent::EndElement { .. } => break, - _ => panic!("Unknown section XML event"), - } - } -} - -fn event_member( - attributes: Vec, - mut event_reader: &mut EventReader, -) -> Member { - let kind = attributes - .into_iter() - .find_map(|attr| { - if attr.name.local_name == "kind" { - Some(attr.value) - } else { - None - } - }) - .unwrap(); - - match kind.as_str() { - "variable" => event_member_variable(&mut event_reader), - _ => { - event_ignore(&mut event_reader); - Member::Unhandled - } - } -} - -fn event_member_variable(mut event_reader: &mut EventReader) -> Member { - let mut member_name = None; - let mut r#type = None; - - loop { - let event = event_reader.next().unwrap(); - match dbg!(event) { - XmlEvent::StartElement { name, .. } => match name.local_name.as_str() { - "name" => member_name = Some(event_simple(&mut event_reader)), - "definition" => r#type = Some(event_simple(&mut event_reader)), - _ => event_ignore(&mut event_reader), - }, - XmlEvent::Whitespace(_) => (), - XmlEvent::EndElement { .. } => break, - _ => panic!("Unknown member XML event"), - } - } - - Member::Variable { - name: member_name.unwrap(), - r#type: r#type.unwrap(), - } -} diff --git a/src/doxygen/builders.rs b/src/doxygen/builders.rs deleted file mode 100644 index 9b60325..0000000 --- a/src/doxygen/builders.rs +++ /dev/null @@ -1,109 +0,0 @@ -use super::*; - -pub(super) fn builder_for(doxygen_kind: String) -> Box { - match doxygen_kind.as_str() { - "namespace" => Box::new(NameSpaceBuilder::default()), - "class" | "struct" => Box::new(ClassBuilder::default()), - _ => Box::new(UnhandledBuilder), - } -} - -#[derive(Debug, Default)] -struct NameSpaceBuilder { - name: Option, - members: Vec, - brief_description: Option, - detailed_description: Option, -} - -impl TypeBuilder for NameSpaceBuilder { - fn build(self: Box) -> DoxygenType { - let name = self.name.unwrap(); - - DoxygenType::NameSpace(NameSpace { - name, - members: self.members, - brief_description: self.brief_description.unwrap(), - detailed_description: self.detailed_description.unwrap(), - }) - } - - fn name(&mut self, value: String) { - self.name = Some(value); - } - - fn add_inner(&mut self, name: String, kind: String) { - self.members.push(name); - } - - fn add_member(&mut self, _section: String, _member: Member) { - panic!("Adding member in namespace?"); - } - - fn brief_description(&mut self, description: Description) { - self.brief_description = Some(description); - } - - fn detailed_description(&mut self, description: Description) { - self.brief_description = Some(description); - } -} - -#[derive(Debug, Default)] -struct UnhandledBuilder; - -impl TypeBuilder for UnhandledBuilder { - fn build(self: Box) -> DoxygenType { - DoxygenType::Unhandled - } - - fn name(&mut self, _value: String) {} - - fn add_inner(&mut self, _name: String, _kind: String) {} - fn add_member(&mut self, _section: String, _member: Member) {} - fn brief_description(&mut self, _description: Description) {} - fn detailed_description(&mut self, _description: Description) {} -} - -#[derive(Debug, Default)] -struct ClassBuilder { - name: Option, - inners: Vec<(String, String)>, - sections: HashMap>, - brief_description: Option, - detailed_description: Option, -} - -impl TypeBuilder for ClassBuilder { - fn build(self: Box) -> DoxygenType { - let name = self.name.unwrap(); - - DoxygenType::Class(Class { - name, - inners: self.inners, - sections: self.sections, - brief_description: self.brief_description.unwrap_or(Description::new()), - detailed_description: self.detailed_description.unwrap_or(Description::new()), - }) - } - - fn name(&mut self, value: String) { - self.name = Some(value); - } - - fn add_inner(&mut self, name: String, kind: String) { - self.inners.push((name, kind)); - } - - fn add_member(&mut self, section: String, member: Member) { - self.sections.entry(section).or_default().push(member); - } - - fn brief_description(&mut self, description: Description) { - self.brief_description = Some(description); - } - - fn detailed_description(&mut self, description: Description) { - self.brief_description = Some(description); - } -} diff --git a/src/entities.rs b/src/entities.rs deleted file mode 100644 index 89905c6..0000000 --- a/src/entities.rs +++ /dev/null @@ -1,216 +0,0 @@ -use std::collections::HashMap; - -pub(crate) type DynEntity = dyn Entity + Sync + Send; - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub(crate) struct Usr(pub(crate) String); - -pub(crate) struct EntitiesManager { - toplevel_entities: HashMap>, - descriptions: HashMap, -} - -pub(crate) struct EntitiesManagerComponents { - pub(crate) toplevel_entities: HashMap>, - pub(crate) descriptions: HashMap, -} - -impl EntitiesManager { - pub fn new() -> Self { - EntitiesManager { - toplevel_entities: HashMap::new(), - descriptions: HashMap::new(), - } - } - - pub fn insert(&mut self, usr: Usr, description: Description) { - info!("Found entity {:?}", description.name); - self.descriptions.insert(usr, description); - } - - pub fn insert_toplevel(&mut self, usr: Usr, entity: Box) { - self.toplevel_entities.insert(usr, entity); - } - - pub fn decompose(self) -> EntitiesManagerComponents { - EntitiesManagerComponents { - toplevel_entities: self.toplevel_entities, - descriptions: self.descriptions, - } - } -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct Description { - pub(crate) name: String, - pub(crate) brief: String, - pub(crate) detailed: String, -} - -impl Description { - pub(crate) fn with_name(name: String) -> Self { - Description { - name, - ..Default::default() - } - } -} - -#[derive(Debug, Clone)] -pub(crate) struct Described { - pub(crate) description: Description, - pub(crate) entity: T, -} - -pub(crate) struct ChildGroup<'a> { - pub(crate) name: &'static str, - pub(crate) children: Vec<(&'a Usr, &'a DynEntity)>, -} - -pub(crate) trait Entity { - fn kind_name(&self) -> &'static str; - fn embeddable_children(&self) -> Vec { - Vec::new() - } - fn separate_children(&self) -> Vec { - Vec::new() - } -} - -#[derive(Debug, Clone)] -pub(crate) struct NameSpace { - // TODO: replace with Vec to keep order - pub(crate) members: HashMap, -} - -#[derive(Debug, Clone)] -pub(crate) enum NameSpaceChild { - NameSpace(NameSpace), - Class(Class), - Variable(Variable), - Function(Function), -} - -impl Entity for NameSpace { - fn kind_name(&self) -> &'static str { - "namespace" - } - - fn embeddable_children(&self) -> Vec { - let mut variables = Vec::new(); - let mut functions = Vec::new(); - for (usr, member) in &self.members { - match member { - NameSpaceChild::NameSpace(_) => {} - NameSpaceChild::Class(_) => {} - NameSpaceChild::Variable(variable) => variables.push((usr, variable as &DynEntity)), - NameSpaceChild::Function(function) => functions.push((usr, function as &DynEntity)), - } - } - - vec![ - ChildGroup { - name: "Variables", - children: variables, - }, - ChildGroup { - name: "Functions", - children: functions, - }, - ] - } - - fn separate_children(&self) -> Vec { - let mut namespaces = Vec::new(); - let mut classes = Vec::new(); - for (usr, member) in &self.members { - match member { - NameSpaceChild::NameSpace(namespace) => { - namespaces.push((usr, namespace as &DynEntity)) - } - NameSpaceChild::Class(class) => classes.push((usr, class as &DynEntity)), - NameSpaceChild::Variable(_) => {} - NameSpaceChild::Function(_) => {} - } - } - - vec![ - ChildGroup { - name: "Namespaces", - children: namespaces, - }, - ChildGroup { - name: "Classes", - children: classes, - }, - ] - } -} - -#[derive(Debug, Clone)] -pub(crate) struct Variable { - pub(crate) r#type: String, -} - -impl Entity for Variable { - fn kind_name(&self) -> &'static str { - "variable" - } -} - -#[derive(Debug, Clone)] -pub(crate) struct Function { - pub(crate) arguments: Vec, - pub(crate) return_type: String, -} - -impl Entity for Function { - fn kind_name(&self) -> &'static str { - "function" - } -} - -#[derive(Debug, Clone)] -pub(crate) struct FunctionArgument { - pub(crate) name: String, - pub(crate) r#type: String, -} - -#[derive(Debug, Clone)] -pub(crate) struct Class { - //pub(crate) member_types: Vec, - // TODO: replace with Vec to keep order - pub(crate) member_functions: HashMap, - pub(crate) member_variables: HashMap, -} - -impl Entity for Class { - fn kind_name(&self) -> &'static str { - "class" - } - - fn embeddable_children(&self) -> Vec { - vec![ - ChildGroup { - name: "Functions", - children: self - .member_functions - .iter() - .map(|(usr, func)| (usr, func as &DynEntity)) - .collect(), - }, - ChildGroup { - name: "Variables", - children: self - .member_variables - .iter() - .map(|(usr, var)| (usr, var as &DynEntity)) - .collect(), - }, - ] - } - - fn separate_children(&self) -> Vec { - Vec::new() - } -} diff --git a/src/generator.rs b/src/generator.rs deleted file mode 100644 index 0fa5a94..0000000 --- a/src/generator.rs +++ /dev/null @@ -1,219 +0,0 @@ -use crate::config::Config; -use crate::entities::{ - Description, DynEntity, EntitiesManager, EntitiesManagerComponents, Usr, -}; -use crate::pandoc::into_pandoc; - -use anyhow::{ensure, Context, Result}; -use rayon::Scope; -use thiserror::Error; - -use std::collections::HashMap; -use std::io::Write; -use std::path::Path; - -const DEFAULT_CSS: &[u8] = include_bytes!("../res/style.css"); - -pub(crate) fn generate(base_dir: &Path, manager: EntitiesManager, config: &Config) -> Result<()> { - let EntitiesManagerComponents { - toplevel_entities, - descriptions, - } = manager.decompose(); - - let md_output_dir = base_dir.join("markdown"); - let html_output_dir = base_dir.join("html"); - - std::fs::create_dir_all(&md_output_dir) - .context("Failed to create the markdown output directory")?; - std::fs::create_dir_all(&html_output_dir) - .context("Failed to create the HTML output directory")?; - - let mut css_tempfile = tempfile::Builder::new() - .prefix("style") - .suffix(".css") - .tempfile()?; - css_tempfile.write_all(DEFAULT_CSS)?; - - let css_path = css_tempfile.path(); - debug!("Generated temporary file with CSS at: {:?}", css_path); - - rayon::scope(|scope| { - for (usr, entity) in &toplevel_entities { - generate_recursively( - &usr, - entity.as_ref(), - scope, - &descriptions, - &md_output_dir, - &html_output_dir, - css_path, - ); - } - }); - - Ok(()) -} - -fn generate_recursively<'a>( - usr: &'a Usr, - entity: &'a DynEntity, - pool: &Scope<'a>, - descriptions: &'a HashMap, - md_output_dir: &'a Path, - html_output_dir: &'a Path, - css_path: &'a Path, -) { - pool.spawn(move |pool| { - trace!("Trying to generate {}", usr.0); - - let leftovers = generate_single( - &usr, - entity, - descriptions.get(&usr).unwrap(), - &descriptions, - &md_output_dir, - &html_output_dir, - &css_path, - ) - .unwrap(); - - for (usr, entity) in leftovers { - generate_recursively( - usr, - entity, - pool, - descriptions, - md_output_dir, - html_output_dir, - css_path, - ); - } - }); -} - -fn generate_single<'e>( - usr: &Usr, - entity: &'e DynEntity, - description: &Description, - descriptions: &HashMap, - md_output_dir: impl AsRef, - html_output_dir: impl AsRef, - css_path: impl AsRef, -) -> Result> { - use std::io::prelude::*; - use std::process::{Command, Stdio}; - - let md_output_file = md_output_dir.as_ref().join(&usr.0); - let html_output_file = html_output_dir.as_ref().join(&usr.0); - - let mut command = Command::new("pandoc"); - - command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .args(&[ - "--from=json", - "--to=markdown", - "--output", - md_output_file - .to_str() - .context("Entity name is not valid UTF-8")?, - ]); - - debug!("Launching command: {:?}", command); - - let mut pandoc = command - .spawn() - .context("Failed to execute Pandoc command")?; - - let (pandoc_ast, leftovers) = into_pandoc(entity, description, descriptions); - - if log_enabled!(log::Level::Trace) { - let json = - serde_json::to_string(&pandoc_ast).context("Failed to convert Pandoc AST to JSON")?; - trace!("Sending json: {}", json); - write!( - pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), - "{}", - &json, - ) - .context("Failed to convert Pandoc AST to JSON")?; - } else { - serde_json::to_writer( - pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), - &pandoc_ast, - ) - .context("Failed to convert Pandoc AST to JSON")?; - } - - let command_str = format!("{:?}", pandoc); - - let output = pandoc - .wait_with_output() - .expect("Pandoc command wasn't running"); - - ensure!( - output.status.success(), - CommandError { - command: format!("{:?}", command_str), - status: output.status, - stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), - } - ); - - let mut command = Command::new("pandoc"); - command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .args(&[ - "--from=markdown", - "--to=html", - "--css", - css_path - .as_ref() - .to_str() - .context("CSS path is not valid UTF-8")?, - "--standalone", - "--self-contained", - md_output_file - .to_str() - .context("Entity name is not valid UTF-8")?, - "--output", - html_output_file - .to_str() - .context("Entity name is not valid UTF-8")?, - ]); - - let command_str = format!("{:?}", command); - debug!("Launching command: {}", command_str); - - let output = command - .output() - .context("Failed to execute Pandoc command")?; - - ensure!( - output.status.success(), - CommandError { - command: command_str, - status: output.status, - stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), - } - ); - - Ok(leftovers) -} - -#[derive(Debug, Clone, Error)] -#[error( - "Command {} returned status {:?} and stderr {:?}", - command, - status, - stderr -)] -struct CommandError { - command: String, - status: std::process::ExitStatus, - stderr: String, -} diff --git a/src/generator/config.rs b/src/generator/config.rs new file mode 100644 index 0000000..43fee60 --- /dev/null +++ b/src/generator/config.rs @@ -0,0 +1,53 @@ +use serde_derive::{Deserialize, Serialize}; +use structopt::StructOpt; + +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Config { + /// Tells us in which language, which entity types should inline which children entity type in + /// their documentation. + /// + /// e.g. in C++, classes should inline their children methods documentation. + pub(crate) inlines: HashMap>>, +} + +impl Config { + pub(crate) fn from_merge(_cli: ProvidedConfig, conf: ProvidedConfig) -> Self { + Self { + inlines: conf.inlines.unwrap_or_else(default_inlines), + } + } +} + +fn default_inlines() -> HashMap>> { + let mut clang = HashMap::new(); + let mut clang_inline_children = HashSet::new(); + // TODO: this is not great: no differences between variable/field for children, but differences + // as a parent... + clang_inline_children.insert(String::from("variable")); + //classes.insert(String::from("field")); + clang_inline_children.insert(String::from("function")); + //classes.insert(String::from("method")); + clang.insert(String::from("struct"), clang_inline_children.clone()); + clang.insert(String::from("class"), clang_inline_children.clone()); + clang.insert(String::from("namespace"), clang_inline_children); + + let mut inlines = HashMap::new(); + inlines.insert(String::from("clang"), clang); + + inlines +} + +impl Default for Config { + fn default() -> Self { + Config::from_merge(ProvidedConfig::default(), ProvidedConfig::default()) + } +} + +#[derive(Debug, Clone, Default, StructOpt, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct ProvidedConfig { + #[structopt(skip)] + pub(crate) inlines: Option>>>, +} diff --git a/src/generator/mod.rs b/src/generator/mod.rs new file mode 100644 index 0000000..b4e1c94 --- /dev/null +++ b/src/generator/mod.rs @@ -0,0 +1,216 @@ +pub(crate) mod config; +mod pandoc; + +use self::config::Config; +//use crate::entities::{ +// Description, DynEntity, EntitiesManager, EntitiesManagerComponents, Usr, +//}; +use self::pandoc::into_pandoc; +use crate::types::Entity; + +use anyhow::{ensure, Context, Result}; +use rayon::Scope; +use thiserror::Error; + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::Path; + +const DEFAULT_CSS: &[u8] = include_bytes!("../../res/style.css"); + +pub(crate) fn generate(base_dir: &Path, entities: BTreeMap, config: &Config) -> Result<()> { + let md_output_dir = base_dir.join("markdown"); + let html_output_dir = base_dir.join("html"); + + std::fs::create_dir_all(&md_output_dir) + .context("Failed to create the markdown output directory")?; + std::fs::create_dir_all(&html_output_dir) + .context("Failed to create the HTML output directory")?; + + let mut css_tempfile = tempfile::Builder::new() + .prefix("style") + .suffix(".css") + .tempfile()?; + css_tempfile.write_all(DEFAULT_CSS)?; + + let css_path = css_tempfile.path(); + debug!("Generated temporary file with CSS at: {:?}", css_path); + + rayon::scope(|scope| { + for (id, entity) in entities { + generate_recursively( + id, + entity, + scope, + &md_output_dir, + &html_output_dir, + css_path, + config, + ); + } + }); + + Ok(()) +} + +fn generate_recursively<'a>( + id: String, + entity: Entity, + pool: &Scope<'a>, + md_output_dir: &'a Path, + html_output_dir: &'a Path, + css_path: &'a Path, + config: &'a Config, +) { + pool.spawn(move |pool| { + trace!("Trying to generate {}", id); + + let leftovers = generate_single( + &id, + entity, + &md_output_dir, + &html_output_dir, + &css_path, + config, + ) + .unwrap(); + + for (id, entity) in leftovers { + generate_recursively( + id, + entity, + pool, + md_output_dir, + html_output_dir, + css_path, + config, + ); + } + }); +} + +fn generate_single( + id: &str, + entity: Entity, + md_output_dir: impl AsRef, + html_output_dir: impl AsRef, + css_path: impl AsRef, + config: &Config +) -> Result> { + use std::io::prelude::*; + use std::process::{Command, Stdio}; + + let md_output_file = md_output_dir.as_ref().join(id); + let html_output_file = html_output_dir.as_ref().join(id); + + let mut command = Command::new("pandoc"); + + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(&[ + "--from=json", + "--to=markdown", + "--output", + md_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + ]); + + debug!("Launching command: {:?}", command); + + let mut pandoc = command + .spawn() + .context("Failed to execute Pandoc command")?; + + let (pandoc_ast, leftovers) = into_pandoc(entity, config); + + if log_enabled!(log::Level::Trace) { + let json = + serde_json::to_string(&pandoc_ast).context("Failed to convert Pandoc AST to JSON")?; + trace!("Sending json: {}", json); + write!( + pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), + "{}", + &json, + ) + .context("Failed to convert Pandoc AST to JSON")?; + } else { + serde_json::to_writer( + pandoc.stdin.as_mut().expect("Pandoc stdin not captured"), + &pandoc_ast, + ) + .context("Failed to convert Pandoc AST to JSON")?; + } + + let command_str = format!("{:?}", pandoc); + + let output = pandoc + .wait_with_output() + .expect("Pandoc command wasn't running"); + + ensure!( + output.status.success(), + CommandError { + command: format!("{:?}", command_str), + status: output.status, + stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), + } + ); + + let mut command = Command::new("pandoc"); + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .args(&[ + "--from=markdown-raw_tex", + "--to=html", + "--css", + css_path + .as_ref() + .to_str() + .context("CSS path is not valid UTF-8")?, + "--standalone", + "--self-contained", + md_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + "--output", + html_output_file + .to_str() + .context("Entity name is not valid UTF-8")?, + ]); + + let command_str = format!("{:?}", command); + debug!("Launching command: {}", command_str); + + let output = command + .output() + .context("Failed to execute Pandoc command")?; + + ensure!( + output.status.success(), + CommandError { + command: command_str, + status: output.status, + stderr: String::from_utf8(output.stderr).expect("Pandoc outputted invalid UTF-8"), + } + ); + + Ok(leftovers) +} + +#[derive(Debug, Clone, Error)] +#[error( + "Command {} returned status {:?} and stderr {:?}", + command, + status, + stderr +)] +struct CommandError { + command: String, + status: std::process::ExitStatus, + stderr: String, +} diff --git a/src/generator/pandoc.rs b/src/generator/pandoc.rs new file mode 100644 index 0000000..1753d34 --- /dev/null +++ b/src/generator/pandoc.rs @@ -0,0 +1,166 @@ +use super::config::Config; +use crate::types::Entity; + +use pandoc_types::definition::{Attr, Block, Inline, Meta, MetaValue, Pandoc}; + +use std::collections::BTreeMap; + +pub(crate) fn into_pandoc( + entity: Entity, + config: &Config, +) -> (Pandoc, BTreeMap) { + let mut meta = Meta::null(); + + let title = vec![Inline::Code(Attr::null(), entity.name.clone())]; + + meta.0 + .insert("title".to_string(), MetaValue::MetaString(entity.name)); + + let mut content = Vec::new(); + + content.push(Block::Header(1, Attr::null(), title)); + + if !entity.documentation.is_empty() { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(String::from("Description"))], + )); + + content.push(Block::Div( + Attr(String::new(), vec![String::from("doc")], vec![]), + vec![raw_markdown(entity.documentation.clone())], + )); + } + + let mut inline_children = BTreeMap::new(); + let mut separate_children = BTreeMap::new(); + + let entity_kind = entity.kind; + for (children_kind, children) in entity.children { + if config + .inlines + .get(&entity.language) + .and_then(|lang_inlines| lang_inlines.get(&entity_kind)) + .map(|children_inlines| children_inlines.contains(&children_kind)) + == Some(true) + { + inline_children.insert(children_kind, children); + } else { + // By default, do not inline + separate_children.insert(children_kind, children); + } + } + + for (section_name, children) in &separate_children { + if let Some(members_list) = member_list(children) { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(String::from(section_name))], + )); + + content.push(members_list); + } + } + + let mut embedded_documentation = Vec::new(); + + for (section_name, children) in inline_children { + if let Some(members_list) = member_list(&children) { + content.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(section_name.clone())], + )); + + content.push(members_list); + + embedded_documentation.push(Block::Header( + 2, + Attr::null(), + vec![Inline::Str(section_name + " Documentation")], + )); + + for (_id, child) in children { + embedded_documentation.push(Block::Header( + 3, + Attr::null(), + vec![Inline::Code(Attr::null(), child.name)], + )); + + embedded_documentation.push(Block::Div( + Attr(String::new(), vec![String::from("doc")], vec![]), + vec![raw_markdown(child.documentation)], + )); + } + } + } + + content.append(&mut embedded_documentation); + + let leftovers = separate_children + .into_iter() + .map(|(_section_name, children)| children) + .flatten() + .collect(); + + (Pandoc(meta, content), leftovers) +} + +fn str_block(content: String) -> Block { + Block::Plain(vec![Inline::Str(content)]) +} + +fn entity_link(id: &str, name: String) -> Inline { + use pandoc_types::definition::Target; + use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; + + use std::iter::once; + + // https://url.spec.whatwg.org/#fragment-percent-encode-set + const FRAGMENT: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'<') + .add(b'>') + .add(b'`') + .add(b'#') + .add(b'?') + .add(b'{') + .add(b'}'); + + Inline::Link( + Attr::null(), + vec![Inline::Code(Attr::null(), name)], + Target( + once("./") + .chain(utf8_percent_encode(id, FRAGMENT)) + .collect(), + String::new(), + ), + ) +} + +fn raw_markdown(text: String) -> Block { + use pandoc_types::definition::Format; + Block::RawBlock(Format(String::from("markdown")), text) +} + +fn member_list<'a>(members: impl IntoIterator) -> Option { + let definitions: Vec<(Vec, Vec>)> = members + .into_iter() + .map(|(id, entity)| { + ( + vec![entity_link(id, entity.name.clone())], + vec![vec![str_block(entity.brief_description.clone())]], + ) + }) + .collect(); + + if definitions.is_empty() { + None + } else { + Some(Block::DefinitionList(definitions)) + } +} diff --git a/src/main.rs b/src/main.rs index e6e36d9..b42d73c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,16 @@ #![warn(clippy::all)] -//mod doxygen; mod cli; mod config; -mod entities; mod generator; -mod pandoc; mod parsing; +mod types; #[macro_use] extern crate log; use cli::{Command, ConfigCommand}; use generator::generate; -use parsing::parse_file; use anyhow::{Context, Result}; use codemap::CodeMap; @@ -86,14 +83,15 @@ fn start(codemap: &mut CodeMap) -> Result<()> { .with_context(|| format!("Cannot change current directory to: {:?}", cli.directory))?; match &cli.command { - Command::Generate { file } => { - let file = file.clone(); + Command::Generate { file: _ } => { + //let file = file.clone(); let config = load_effective_config(cli, codemap)?; - let manager = parse_file(file, &config.extra_clang_args); + let entities = parsing::clang::parse_compile_commands(&config.clang, codemap)?; + //let manager = parse_file(file, &config.extra_clang_args); let base_output_dir = std::path::Path::new("doc"); - generate(&base_output_dir, manager, &config)?; + generate(&base_output_dir, entities, &config.generator)?; } Command::Config { command: ConfigCommand::Default, @@ -105,7 +103,7 @@ fn start(codemap: &mut CodeMap) -> Result<()> { } => { print!( "{}", - toml::to_string_pretty(&config::load_config(".", codemap)?)? + toml::to_string_pretty(&load_effective_config(cli, codemap)?)? ); } } @@ -114,5 +112,6 @@ fn start(codemap: &mut CodeMap) -> Result<()> { } fn load_effective_config(cli: cli::Cli, codemap: &mut CodeMap) -> Result { - Ok(config::load_config(".", codemap)?.merge_cli(cli)) + let conf_file_conf = config::load_config(".", codemap)?; + Ok(config::Config::from_merge(cli, conf_file_conf)?) } diff --git a/src/pandoc.rs b/src/pandoc.rs deleted file mode 100644 index 8477f28..0000000 --- a/src/pandoc.rs +++ /dev/null @@ -1,166 +0,0 @@ -use crate::entities::*; - -use pandoc_types::definition::{Attr, Block, Inline, Meta, MetaValue, Pandoc}; - -use std::collections::HashMap; - -pub(crate) fn into_pandoc<'e>( - entity: &'e dyn Entity, - description: &Description, - descriptions: &HashMap, -) -> (Pandoc, HashMap<&'e Usr, &'e DynEntity>) { - let mut meta = Meta::null(); - - let title = vec![Inline::Code(Attr::null(), description.name.clone())]; - - meta.0.insert( - "title".to_string(), - MetaValue::MetaString(description.name.clone()), - ); - - let mut content = Vec::new(); - - content.push(Block::Header(1, Attr::null(), title)); - - if !description.detailed.is_empty() { - content.push(Block::Header( - 2, - Attr::null(), - vec![Inline::Str(String::from("Description"))], - )); - - content.push(Block::Div( - Attr(String::new(), vec![String::from("doc")], vec![]), - vec![raw_markdown(description.detailed.clone())], - )); - } - - let separate_children = entity.separate_children(); - let embeddable_children = entity.embeddable_children(); - - for section in &separate_children { - if let Some(members_list) = member_list( - section.children.iter().map(|&(usr, _child)| usr), - descriptions, - ) { - content.push(Block::Header( - 2, - Attr::null(), - vec![Inline::Str(String::from(section.name))], - )); - - content.push(members_list); - } - } - - let mut embedded_documentation = Vec::new(); - - for section in &embeddable_children { - if let Some(members_list) = member_list( - section.children.iter().map(|&(usr, _child)| usr), - descriptions, - ) { - content.push(Block::Header( - 2, - Attr::null(), - vec![Inline::Str(String::from(section.name))], - )); - - content.push(members_list); - - embedded_documentation.push(Block::Header( - 2, - Attr::null(), - vec![Inline::Str(String::from(section.name) + " Documentation")], - )); - - for (usr, _child) in §ion.children { - let child_doc = descriptions.get(usr).unwrap(); - - embedded_documentation.push(Block::Header( - 3, - Attr::null(), - vec![Inline::Code(Attr::null(), String::from(&child_doc.name))], - )); - - embedded_documentation.push(Block::Div( - Attr(String::new(), vec![String::from("doc")], vec![]), - vec![raw_markdown(child_doc.detailed.clone())], - )); - } - } - } - - content.append(&mut embedded_documentation); - - let leftovers = separate_children - .iter() - .map(|section| section.children.clone()) - .flatten() - .collect(); - - (Pandoc(meta, content), leftovers) -} - -fn str_block(content: String) -> Block { - Block::Plain(vec![Inline::Str(content)]) -} - -fn entity_link(usr: &Usr, name: String) -> Inline { - use pandoc_types::definition::Target; - use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; - - use std::iter::once; - - // https://url.spec.whatwg.org/#fragment-percent-encode-set - const FRAGMENT: &AsciiSet = &CONTROLS - .add(b' ') - .add(b'"') - .add(b'<') - .add(b'>') - .add(b'`') - .add(b'#') - .add(b'?') - .add(b'{') - .add(b'}'); - - Inline::Link( - Attr::null(), - vec![Inline::Code(Attr::null(), name)], - Target( - once("./") - .chain(utf8_percent_encode(&usr.0, FRAGMENT)) - .collect(), - String::new(), - ), - ) -} - -fn raw_markdown(text: String) -> Block { - use pandoc_types::definition::Format; - Block::RawBlock(Format(String::from("markdown")), text) -} - -fn member_list<'a>( - members: impl IntoIterator, - descriptions: &HashMap, -) -> Option { - let definitions: Vec<(Vec, Vec>)> = members - .into_iter() - .filter_map(|usr| { - let name = &descriptions.get(usr)?.name; - Some(( - vec![entity_link(usr, name.clone())], - vec![vec![str_block( - descriptions.get(usr).unwrap().brief.clone(), - )]], - )) - }) - .collect(); - - if definitions.is_empty() { - None - } else { - Some(Block::DefinitionList(definitions)) - } -} diff --git a/src/parsing.rs b/src/parsing.rs deleted file mode 100644 index d7aaa49..0000000 --- a/src/parsing.rs +++ /dev/null @@ -1,482 +0,0 @@ -use crate::entities::*; - -use clang::{Clang, Index}; -use codemap::CodeMap; - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -pub(crate) fn parse_file(path: T, extra_args: &[S]) -> EntitiesManager -where - T: Into, - T: AsRef, - T: ToString, - S: AsRef, - S: std::fmt::Debug, -{ - let mut codemap = CodeMap::new(); - let file_map = codemap.add_file(path.to_string(), std::fs::read_to_string(&path).unwrap()); - let file_span = file_map.span; - - let clang = Clang::new().unwrap(); - let index = Index::new(&clang, true, false); - let mut parser = index.parser(path); - parser.skip_function_bodies(true); - - parser.arguments(&extra_args); - - if log_enabled!(log::Level::Debug) { - for extra_arg in extra_args { - debug!("Extra LibClang argument: {:?}", extra_arg); - } - } - - let trans_unit = parser.parse().unwrap(); - - let mut manager = EntitiesManager::new(); - - trans_unit.get_entity().visit_children(|entity, _parent| { - if entity.is_in_system_header() { - trace!( - "Entity is in system header, skipping: USR = {:?}", - entity.get_display_name() - ); - return clang::EntityVisitResult::Continue; - } - - // TODO: wrap this callback in another function so that we can use the - // "?" operator instead of all these `match`es - let usr = match entity.get_usr() { - Some(usr) => usr, - None => return clang::EntityVisitResult::Continue, - }; - trace!("Entity with USR = {:?}", usr); - - let name = match entity.get_name() { - Some(name) => name, - None => return clang::EntityVisitResult::Continue, - }; - - debug!("Parsing toplevel entity with name = {:?}", name); - - let entity = entity.into_poseidoc_entity(&mut manager); - - manager.insert_toplevel(usr.into(), entity); - - clang::EntityVisitResult::Continue - }); - - use codemap_diagnostic::{ColorConfig, Emitter}; - - let mut emitter = Emitter::stderr(ColorConfig::Auto, Some(&codemap)); - - for diagnostic in trans_unit.get_diagnostics().iter() { - let main_diag = match clang_diag_to_codemap_diag(&diagnostic, file_span) { - Some(diag) => diag, - None => continue, - }; - - let sub_diags = diagnostic - .get_children() - .into_iter() - .filter_map(|diagnostic| clang_diag_to_codemap_diag(&diagnostic, file_span)); - - let fix_it_diags = diagnostic - .get_fix_its() - .into_iter() - .map(|fix_it| clang_fix_it_to_codemap_diag(&fix_it, file_span)); - - emitter.emit( - &std::iter::once(main_diag) - .chain(sub_diags) - .chain(fix_it_diags) - .collect::>(), - ); - } - - manager -} - -fn clang_diag_to_codemap_diag( - diagnostic: &clang::diagnostic::Diagnostic, - file_span: codemap::Span, -) -> Option { - use codemap_diagnostic::{Diagnostic, Level, SpanLabel, SpanStyle}; - - let ranges = diagnostic.get_ranges(); - - let (begin, end) = if ranges.is_empty() { - let location = diagnostic.get_location(); - - ( - location.get_file_location().offset as u64, - location.get_file_location().offset as u64, - ) - } else { - let range = diagnostic.get_ranges()[0]; - if !range.is_in_main_file() { - warn!("Skipping diagnostic: {:?}", diagnostic); - return None; - } - ( - range.get_start().get_file_location().offset as u64, - range.get_end().get_file_location().offset as u64, - ) - }; - - let label = SpanLabel { - span: file_span.subspan(begin, end), - label: None, - style: SpanStyle::Primary, - }; - - Some(Diagnostic { - level: match diagnostic.get_severity() { - clang::diagnostic::Severity::Ignored => return None, - clang::diagnostic::Severity::Note => Level::Note, - clang::diagnostic::Severity::Warning => Level::Warning, - clang::diagnostic::Severity::Error => Level::Error, - clang::diagnostic::Severity::Fatal => Level::Error, - }, - message: diagnostic.get_text(), - //code: Some("-Werror=documentation".to_string()), - code: None, - spans: vec![label], - }) -} - -fn clang_fix_it_to_codemap_diag( - fix_it: &clang::diagnostic::FixIt, - file_span: codemap::Span, -) -> codemap_diagnostic::Diagnostic { - use clang::diagnostic::FixIt; - use codemap_diagnostic::{Diagnostic, Level, SpanLabel, SpanStyle}; - - let label = match fix_it { - FixIt::Deletion(range) => { - let begin = range.get_start().get_file_location().offset as u64; - let end = range.get_end().get_file_location().offset as u64; - - SpanLabel { - span: file_span.subspan(begin, end), - label: Some(String::from("Try deleting this")), - style: SpanStyle::Primary, - } - } - FixIt::Insertion(range, text) => { - let location = range.get_file_location().offset as u64; - - SpanLabel { - span: file_span.subspan(location, location), - label: Some(format!("Try insterting {:?}", text)), - style: SpanStyle::Primary, - } - } - FixIt::Replacement(range, text) => { - let begin = range.get_start().get_file_location().offset as u64; - let end = range.get_end().get_file_location().offset as u64; - - SpanLabel { - span: file_span.subspan(begin, end), - label: Some(format!("Try replacing this with {:?}", text)), - style: SpanStyle::Primary, - } - } - }; - - Diagnostic { - level: Level::Help, - message: String::from("Possible fix"), - //code: Some("-Werror=documentation".to_string()), - code: None, - spans: vec![label], - } -} - -pub(crate) struct Comment(String); - -impl Comment { - // TODO: weirdness with emojis and line returns? - pub fn from_raw(raw: String) -> Self { - #[derive(Debug)] - enum CommentStyle { - // Comments of type `/**` or `/*!` - Starred, - // Comments of type `///` - SingleLine, - } - - let mut chars = raw.chars(); - let style = match &chars.as_str()[..3] { - "/*!" | "/**" => CommentStyle::Starred, - "///" => CommentStyle::SingleLine, - _ => panic!("Comment is empty or doesn't start with either `///`, `/**`, or `/*!`"), - }; - - chars.nth(2); - - let mut result = String::new(); - - 'parse_loop: loop { - let maybe_space = chars.next(); - let mut empty_line = false; - match maybe_space { - // TODO: Warn on empty comments - None => break, - Some(' ') => {} - Some('\n') => { - empty_line = true; - result.push('\n'); - } - Some(ch) => result.push(ch), - } - - if !empty_line { - let rest = chars.as_str(); - match rest.find('\n') { - None => { - result.push_str(rest); - break; - } - Some(position) => { - result.push_str(&rest[..=position]); - chars.nth(position); - } - } - } - - // Beginning of the line - let first_non_ws_ch = 'ws_loop: loop { - let maybe_whitespace = chars.next(); - match maybe_whitespace { - None => break 'parse_loop, - Some(ch) if ch.is_whitespace() => continue, - Some(ch) => break 'ws_loop ch, - } - }; - - match style { - CommentStyle::Starred if first_non_ws_ch == '*' => { - if &chars.as_str()[..1] == "/" { - break; - } - } - CommentStyle::Starred => result.push(first_non_ws_ch), - CommentStyle::SingleLine => { - assert!(first_non_ws_ch == '/'); - let rest = chars.as_str(); - if &rest[..2] == "//" { - chars.nth(1); - } else if &rest[..1] == "/" { - chars.nth(0); - } else { - panic!("Could not parse comment"); - } - } - } - } - - Self(result) - } -} - -impl From for Usr { - fn from(usr: clang::Usr) -> Self { - Self(usr.0) - } -} - -trait FromClangEntity { - /// Is responsible for inserting its documentation into the entities manager - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self; -} - -trait IntoPoseidocEntity { - /// Useful for the `clang_entity.into_poseidoc_entity(&mut manager)` syntax. Implement - /// `FromClangEntity` instead. - fn into_poseidoc_entity(self, manager: &mut EntitiesManager) -> T; -} - -impl IntoPoseidocEntity for clang::Entity<'_> -where - T: FromClangEntity, -{ - fn into_poseidoc_entity(self, manager: &mut EntitiesManager) -> T { - T::from_clang_entity(self, manager) - } -} - -fn get_description(entity: &clang::Entity) -> Description { - let name = entity.get_display_name().unwrap(); - - if let (Some(brief), Some(comment)) = (entity.get_comment_brief(), entity.get_comment()) { - Description { - name, - brief, - detailed: Comment::from_raw(comment).0, - } - } else { - Description::with_name(name) - } -} - -impl FromClangEntity for Box { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - use clang::EntityKind; - - match entity.get_kind() { - EntityKind::Namespace => Box::new(NameSpace::from_clang_entity(entity, manager)), - EntityKind::FieldDecl | EntityKind::VarDecl => { - Box::new(Variable::from_clang_entity(entity, manager)) - } - EntityKind::FunctionDecl | EntityKind::Method | EntityKind::FunctionTemplate => { - Box::new(Function::from_clang_entity(entity, manager)) - } - EntityKind::ClassDecl | EntityKind::StructDecl | EntityKind::ClassTemplate => { - Box::new(Class::from_clang_entity(entity, manager)) - } - _ => panic!("Unsupported entity: {:?}", entity), - } - } -} - -impl FromClangEntity for NameSpace { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - match entity.get_kind() { - clang::EntityKind::Namespace => {} - _ => panic!("Trying to parse a non-variable into a variable"), - } - debug!("Parsing NameSpace"); - - let members = entity - .get_children() - .into_iter() - .map(|child| { - let usr = child.get_usr().unwrap().into(); - trace!("Namespace has member: {:?}", usr); - (usr, child.into_poseidoc_entity(manager)) - }) - .collect(); - - manager.insert(entity.get_usr().unwrap().into(), get_description(&entity)); - - NameSpace { members } - } -} - -impl FromClangEntity for NameSpaceChild { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - use clang::EntityKind; - - match entity.get_kind() { - EntityKind::Namespace => { - NameSpaceChild::NameSpace(NameSpace::from_clang_entity(entity, manager)) - } - EntityKind::FieldDecl | EntityKind::VarDecl => { - NameSpaceChild::Variable(Variable::from_clang_entity(entity, manager)) - } - EntityKind::FunctionDecl | EntityKind::Method | EntityKind::FunctionTemplate => { - NameSpaceChild::Function(Function::from_clang_entity(entity, manager)) - } - EntityKind::ClassDecl | EntityKind::StructDecl | EntityKind::ClassTemplate => { - NameSpaceChild::Class(Class::from_clang_entity(entity, manager)) - } - _ => panic!("Unsupported entity: {:?}", entity), - } - } -} - -impl FromClangEntity for Variable { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - match entity.get_kind() { - clang::EntityKind::VarDecl | clang::EntityKind::FieldDecl => {} - _ => panic!("Trying to parse a non-variable into a variable"), - } - debug!("Parsing Variable"); - - let r#type = entity.get_type().unwrap().get_display_name(); - trace!("Variable has type: {:?}", r#type); - - manager.insert(entity.get_usr().unwrap().into(), get_description(&entity)); - - Variable { r#type } - } -} - -impl FromClangEntity for Function { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - match entity.get_kind() { - clang::EntityKind::Method - | clang::EntityKind::Constructor - | clang::EntityKind::FunctionDecl - | clang::EntityKind::FunctionTemplate => {} - _ => panic!("Trying to parse a non-function into a function"), - } - debug!("Parsing Function"); - - let return_type = entity.get_result_type().unwrap().get_display_name(); - trace!("Function has return type: {:?}", return_type); - let arguments = entity - .get_arguments() - .unwrap() - .into_iter() - .map(|arg| { - let name = arg.get_display_name().unwrap(); - let r#type = arg.get_type().unwrap().get_display_name(); - trace!("Function has argument {:?} of type {:?}", name, r#type); - FunctionArgument { name, r#type } - }) - .collect(); - - manager.insert(entity.get_usr().unwrap().into(), get_description(&entity)); - - Function { - arguments, - return_type, - } - } -} - -impl FromClangEntity for Class { - fn from_clang_entity(entity: clang::Entity, manager: &mut EntitiesManager) -> Self { - match entity.get_kind() { - clang::EntityKind::ClassDecl - | clang::EntityKind::StructDecl - | clang::EntityKind::ClassTemplate => {} - _ => panic!("Trying to parse a non-class into a class"), - } - debug!("Parsing Class"); - - let mut member_types = Vec::new(); - let mut member_functions = HashMap::new(); - let mut member_variables = HashMap::new(); - - for child in entity.get_children() { - trace!("Class has child: {:?}", child); - - let child_usr = child.get_usr().unwrap().into(); - - use clang::EntityKind; - match child.get_kind() { - EntityKind::ClassDecl | EntityKind::StructDecl | EntityKind::TypeAliasDecl => { - member_types.push(child_usr); - } - EntityKind::Method | EntityKind::Constructor | EntityKind::FunctionDecl => { - member_functions.insert(child_usr, child.into_poseidoc_entity(manager)); - } - EntityKind::FieldDecl => { - member_variables.insert(child_usr, child.into_poseidoc_entity(manager)); - } - _ => trace!("Skipping child"), - } - } - - manager.insert(entity.get_usr().unwrap().into(), get_description(&entity)); - - Class { - //member_types, - member_functions, - member_variables, - } - } -} diff --git a/src/parsing/clang/config.rs b/src/parsing/clang/config.rs new file mode 100644 index 0000000..cb05876 --- /dev/null +++ b/src/parsing/clang/config.rs @@ -0,0 +1,53 @@ +use anyhow::Result; +use serde_derive::{Deserialize, Serialize}; +use structopt::StructOpt; + +use std::path::PathBuf; + +#[derive(Debug, Clone, StructOpt, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Config { + pub(crate) compile_commands_location: PathBuf, + pub(crate) extra_args: Vec, +} + +#[derive(Debug, Clone, Default, StructOpt, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct ProvidedConfig { + #[structopt(long = "clang-compile-commands-location")] + pub(crate) compile_commands_location: Option, + #[structopt(long = "clang-extra-args", number_of_values = 1)] + #[serde(default)] + pub(crate) extra_args: Vec, +} + +impl Default for Config { + fn default() -> Self { + Config::from_merge( + ProvidedConfig::default(), + ProvidedConfig::default(), + ) + // Currently errors out only on CLI parse fail with clang's extra args + .unwrap() + } +} + +impl Config { + pub(crate) fn from_merge( + cli: ProvidedConfig, + mut config: ProvidedConfig, + ) -> Result { + let mut extra_args = Vec::new(); + for args in cli.extra_args { + extra_args.append(&mut ::shell_words::split(&args)?); + } + extra_args.append(&mut config.extra_args); + Ok(Self { + compile_commands_location: cli + .compile_commands_location + .or(config.compile_commands_location) + .unwrap_or_else(|| PathBuf::from(r".")), + extra_args, + }) + } +} diff --git a/src/parsing/clang/entities.rs b/src/parsing/clang/entities.rs new file mode 100644 index 0000000..a8062bb --- /dev/null +++ b/src/parsing/clang/entities.rs @@ -0,0 +1,369 @@ +use crate::types::Entity; + +use clang::{EntityKind, Usr}; +use thiserror::Error; + +use std::collections::BTreeMap; +use std::convert::TryFrom; + +// TODO: factor out hardcoded strings + +#[derive(Debug)] +pub(super) struct Children { + namespaces: Option>>, + variables: Option>>, + structs: Option>>, + functions: Option>>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum ClangEntityKind { + Namespace, + Variable(VariableKind), + Struct(StructKind), + Function(FunctionKind), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum VariableKind { + Variable, + Field, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum StructKind { + Struct, + Class, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum FunctionKind { + Function, + Method, +} + +#[derive(Debug, Error)] +#[error("Unsupported Clang entity kind: {:?}", _0)] +pub(super) struct TryFromEntityKindError(String); + +impl TryFrom for ClangEntityKind { + type Error = TryFromEntityKindError; + + fn try_from(kind: EntityKind) -> Result { + Ok(match kind { + EntityKind::Namespace => ClangEntityKind::Namespace, + EntityKind::VarDecl => ClangEntityKind::Variable(VariableKind::Variable), + EntityKind::FieldDecl => ClangEntityKind::Variable(VariableKind::Field), + EntityKind::FunctionDecl => ClangEntityKind::Function(FunctionKind::Function), + EntityKind::Method | EntityKind::Constructor => { + ClangEntityKind::Function(FunctionKind::Method) + } + EntityKind::StructDecl => ClangEntityKind::Struct(StructKind::Struct), + EntityKind::ClassDecl => ClangEntityKind::Struct(StructKind::Class), + kind => return Err(TryFromEntityKindError(format!("{:?}", kind))), + }) + } +} + +pub(super) trait ClangEntity { + fn kind(&self) -> &'static str; + + fn get_member_namespaces(&mut self) -> Option<&mut BTreeMap>> { + None + } + fn get_member_variables(&mut self) -> Option<&mut BTreeMap>> { + None + } + fn get_member_structs(&mut self) -> Option<&mut BTreeMap>> { + None + } + fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { + None + } + + fn into_children(self) -> Children + where + Self: Sized, + { + Children { + namespaces: None, + variables: None, + structs: None, + functions: None, + } + } +} + +/* +pub(super) trait FromNamespaceParent: ClangEntity + Sized { + fn from_namespace_parent<'a>( + parent: &'a mut Described, + name: &Usr, + ) -> Option<&'a mut Described>; +} +*/ + +pub(super) trait NamespaceParentManipulation { + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>>; +} + +impl NamespaceParentManipulation for U +where + U: ClangEntity, +{ + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>> { + self.get_member_namespaces() + } +} + +impl NamespaceParentManipulation for U +where + U: ClangEntity, +{ + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>> { + self.get_member_variables() + } +} + +impl NamespaceParentManipulation for U +where + U: ClangEntity, +{ + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>> { + self.get_member_functions() + } +} + +impl NamespaceParentManipulation for U +where + U: ClangEntity, +{ + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>> { + self.get_member_structs() + } +} + +impl From> for Entity { + fn from(entity: Described) -> Self { + let mut children = BTreeMap::new(); + + let kind = String::from(entity.entity.kind()); + let clang_children = entity.entity.into_children(); + + if let Some(namespaces) = clang_children.namespaces { + children.insert( + String::from("namespace"), + namespaces + .into_iter() + .map(|(usr, namespace)| (usr.0, namespace.into())) + .collect(), + ); + } + + if let Some(variables) = clang_children.variables { + children.insert( + String::from("variable"), + variables + .into_iter() + .map(|(usr, variable)| (usr.0, variable.into())) + .collect(), + ); + } + + if let Some(structs) = clang_children.structs { + children.insert( + String::from("struct"), + structs + .into_iter() + .map(|(usr, the_struct)| (usr.0, the_struct.into())) + .collect(), + ); + } + + if let Some(functions) = clang_children.functions { + children.insert( + String::from("function"), + functions + .into_iter() + .map(|(usr, function)| (usr.0, function.into())) + .collect(), + ); + } + + Entity { + name: entity.description.name, + language: String::from("clang"), + kind, + brief_description: entity.description.brief, + documentation: entity.description.detailed, + children, + } + } +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Description { + pub(super) name: String, + pub(super) brief: String, + pub(super) detailed: String, +} + +#[derive(Debug, Clone)] +pub(super) struct Described { + pub(super) description: Description, + pub(super) entity: T, +} + +#[derive(Debug, Clone)] +pub(super) struct Namespace { + pub(super) member_namespaces: BTreeMap>, + pub(super) member_variables: BTreeMap>, + pub(super) member_structs: BTreeMap>, + pub(super) member_functions: BTreeMap>, +} + +impl ClangEntity for Namespace { + fn kind(&self) -> &'static str { + "namespace" + } + + fn get_member_namespaces(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_namespaces) + } + fn get_member_variables(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_variables) + } + fn get_member_structs(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_structs) + } + fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_functions) + } + + fn into_children(self) -> Children { + Children { + namespaces: Some(self.member_namespaces), + variables: Some(self.member_variables), + structs: Some(self.member_structs), + functions: Some(self.member_functions), + } + } +} + +/* +impl FromNamespaceParent for Namespace { + fn from_namespace_parent<'a>( + parent: &'a mut Described, + name: &Usr, + ) -> Option<&'a mut Described> { + parent.entity.member_namespaces.get_mut(name) + } +} +*/ + +#[derive(Debug, Clone)] +pub(super) struct Variable { + pub(super) kind: VariableKind, + pub(super) r#type: String, +} + +impl ClangEntity for Variable { + fn kind(&self) -> &'static str { + match self.kind { + VariableKind::Variable => "variable", + VariableKind::Field => "field", + } + } +} + +/* +impl FromNamespaceParent for Variable { + fn from_namespace_parent<'a>( + parent: &'a mut Described, + name: &Usr, + ) -> Option<&'a mut Described> { + parent.entity.member_variables.get_mut(name) + } +} +*/ + +#[derive(Debug, Clone)] +pub(super) struct Struct { + //pub(super) member_types: Vec, + pub(super) kind: StructKind, + pub(super) member_variables: BTreeMap>, + pub(super) member_structs: BTreeMap>, + pub(super) member_functions: BTreeMap>, +} + +impl ClangEntity for Struct { + fn kind(&self) -> &'static str { + match self.kind { + StructKind::Struct => "struct", + StructKind::Class => "class", + } + } + + fn get_member_variables(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_variables) + } + fn get_member_structs(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_structs) + } + fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_functions) + } + + fn into_children(self) -> Children { + Children { + namespaces: None, + variables: Some(self.member_variables), + structs: Some(self.member_structs), + functions: Some(self.member_functions), + } + } +} + +/* +impl FromNamespaceParent for Struct { + fn from_namespace_parent<'a>( + parent: &'a mut Described, + name: &Usr, + ) -> Option<&'a mut Described> { + parent.entity.member_structs.get_mut(name) + } +} +*/ + +#[derive(Debug, Clone)] +pub(super) struct Function { + pub(super) kind: FunctionKind, + pub(super) arguments: Vec, + pub(super) return_type: String, +} + +impl ClangEntity for Function { + fn kind(&self) -> &'static str { + match self.kind { + FunctionKind::Function => "function", + FunctionKind::Method => "method", + } + } +} + +/* +impl FromNamespaceParent for Function { + fn from_namespace_parent<'a>( + parent: &'a mut Described, + name: &Usr, + ) -> Option<&'a mut Described> { + parent.entity.member_functions.get_mut(name) + } +} +*/ + +#[derive(Debug, Clone)] +pub(super) struct FunctionArgument { + pub(super) name: String, + pub(super) r#type: String, +} diff --git a/src/parsing/clang/mod.rs b/src/parsing/clang/mod.rs new file mode 100644 index 0000000..047c49e --- /dev/null +++ b/src/parsing/clang/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod config; +mod entities; +mod parsing; + +pub(crate) use parsing::parse_compile_commands; diff --git a/src/parsing/clang/parsing.rs b/src/parsing/clang/parsing.rs new file mode 100644 index 0000000..5359253 --- /dev/null +++ b/src/parsing/clang/parsing.rs @@ -0,0 +1,748 @@ +use super::config::Config; +use super::entities::*; +use crate::types::Entity; + +use anyhow::{anyhow, Context, Error, Result}; +use clang::{Clang, CompilationDatabase, Index, TranslationUnit, Usr}; +use codemap::CodeMap; +use thiserror::Error; + +use std::collections::BTreeMap; +use std::convert::{TryFrom, TryInto}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Default)] +struct TopLevel { + namespaces: BTreeMap>, + variables: BTreeMap>, + structs: BTreeMap>, + functions: BTreeMap>, +} + +/* +enum TopLevelEntry<'a, T> { + Vacant { + parent: &'a mut Described, + }, + /// Vacant, but no semantic parent + TopLevel, + Occupied { + entity: &'a mut Described, + }, + Error, +} +*/ + +impl TopLevel { + // Somehow has a lifetime issue I can't get my head around + /* + fn entry<'a, T>(&'a mut self, path: &::clang::Entity) -> Result> + where + T: ClangEntity + FromNamespaceParent + FromTopLevel, + { + let usr = path.get_usr().ok_or_else(|| anyhow!("no usr"))?; + if let Some(parent_path) = parent(&path) { + let parent_entry = self.entry::(&parent_path)?; + if let TopLevelEntry::Occupied { + entity: namespace_parent, + } = parent_entry + { + Ok(match T::from_namespace_parent(namespace_parent, &usr) { + None => TopLevelEntry::Vacant { + parent: namespace_parent, + }, + Some(entity) => TopLevelEntry::Occupied { entity }, + }) + } else { + panic!("Wut"); + } + } else { + Ok(match T::from_toplevel(self, &usr) { + Some(entity) => TopLevelEntry::Occupied { entity }, + None => TopLevelEntry::TopLevel, + }) + } + } + */ + + fn get_entity_mut(&mut self, path: clang::Entity) -> Option<&mut dyn ClangEntity> { + let usr = path.get_usr()?; + if let Some(parent_path) = parent(path) { + let parent = self.get_entity_mut(parent_path)?; + Some(match path.get_kind().try_into().ok()? { + ClangEntityKind::Namespace => { + &mut parent.get_member_namespaces()?.get_mut(&usr)?.entity + } + ClangEntityKind::Variable(_) => { + &mut parent.get_member_variables()?.get_mut(&usr)?.entity + } + ClangEntityKind::Function(_) => { + &mut parent.get_member_functions()?.get_mut(&usr)?.entity + } + ClangEntityKind::Struct(_) => { + &mut parent.get_member_structs()?.get_mut(&usr)?.entity + } + }) + } else { + Some(match path.get_kind().try_into().ok()? { + ClangEntityKind::Namespace => &mut self.namespaces.get_mut(&usr)?.entity, + ClangEntityKind::Variable(_) => &mut self.variables.get_mut(&usr)?.entity, + ClangEntityKind::Struct(_) => &mut self.structs.get_mut(&usr)?.entity, + ClangEntityKind::Function(_) => &mut self.functions.get_mut(&usr)?.entity, + }) + } + } + + fn get_namespace_mut(&mut self, path: clang::Entity) -> Option<&mut Described> { + let usr = path.get_usr()?; + + if let Some(parent_path) = parent(path) { + let parent = self.get_entity_mut(parent_path)?; + parent.get_member_namespaces()?.get_mut(&usr) + } else { + self.namespaces.get_mut(&usr) + } + } + + fn insert(&mut self, path: clang::Entity, entity: Described) -> Result<()> + where + T: ClangEntity + std::fmt::Debug, + Self: TopLevelManipulation, + Namespace: NamespaceParentManipulation, + { + let usr = path.get_usr().ok_or_else(|| anyhow!("no usr"))?; + if let Some(parent_path) = parent(path) { + if let Some(parent_namespace) = self.get_namespace_mut(parent_path) { + parent_namespace + .entity + .get_members_mut() + // Namespace should be able to contain every kind of entity + .unwrap() + .insert(usr, entity); + Ok(()) + } else { + Err(anyhow!("has parent but no parent in tree")) + } + } else { + self.insert_toplevel(usr, entity); + Ok(()) + } + } +} + +// Like .get_semantic_parent(), but return none if the parent is the translation unit +fn parent(libclang_entity: clang::Entity) -> Option { + match libclang_entity.get_semantic_parent() { + Some(parent) => { + if parent.get_kind() != clang::EntityKind::TranslationUnit { + Some(parent) + } else { + None + } + } + None => { + warn!("get_semantic_parent() returned None"); + None + } + } +} + +trait TopLevelManipulation { + fn insert_toplevel(&mut self, usr: Usr, entity: Described); +} + +impl TopLevelManipulation for TopLevel { + fn insert_toplevel(&mut self, usr: Usr, entity: Described) { + self.namespaces.insert(usr, entity); + } +} + +impl TopLevelManipulation for TopLevel { + fn insert_toplevel(&mut self, usr: Usr, entity: Described) { + self.variables.insert(usr, entity); + } +} + +impl TopLevelManipulation for TopLevel { + fn insert_toplevel(&mut self, usr: Usr, entity: Described) { + self.functions.insert(usr, entity); + } +} + +impl TopLevelManipulation for TopLevel { + fn insert_toplevel(&mut self, usr: Usr, entity: Described) { + self.structs.insert(usr, entity); + } +} + +/* +trait FromTopLevel: ClangEntity + Sized { + fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described>; +} + +impl FromTopLevel for Namespace { + fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described> { + toplevel.namespaces.get_mut(usr) + } +} + +impl FromTopLevel for Variable { + fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described> { + toplevel.variables.get_mut(usr) + } +} + +impl FromTopLevel for Function { + fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described> { + toplevel.functions.get_mut(usr) + } +} + +impl FromTopLevel for Struct { + fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described> { + toplevel.structs.get_mut(usr) + } +} +*/ + +pub(crate) fn parse_compile_commands( + config: &Config, + codemap: &mut CodeMap, +) -> Result> { + let clang = Clang::new().unwrap(); + let index = Index::new( + &clang, /* exclude from pch = */ false, /* print diagnostics = */ false, + ); + + debug!("Extra libclang argument: {:?}", config.extra_args); + + debug!( + "Loading compile commands from: {:?}", + config.compile_commands_location + ); + let database = + CompilationDatabase::from_directory(&config.compile_commands_location).map_err(|()| { + CompileCommandsLoadError { + path: config.compile_commands_location.clone(), + } + })?; + + let toplevel_directory = std::env::current_dir().context("Cannot read current directory")?; + + let mut entities = TopLevel::default(); + + for command in database.get_all_compile_commands().get_commands() { + let directory = command.get_directory(); + trace!("Changing directory to: {:?}", directory); + std::env::set_current_dir(&directory) + .with_context(|| format!("Cannot change current directory to: {:?}", directory))?; + + let filename = command.get_filename(); + + let file_map = codemap.add_file( + filename + .to_str() + .context("File is not valid UTF-8")? + .to_owned(), + std::fs::read_to_string(&filename) + .with_context(|| format!("Cannot readfile: {:?}", filename))?, + ); + + trace!("Parsing file: {:?}", filename); + // The file name is passed as an argument in the compile commands + let mut parser = index.parser(""); + parser.skip_function_bodies(true); + + let mut clang_arguments = command.get_arguments(); + clang_arguments.extend_from_slice(&config.extra_args); + trace!("Parsing with libclang arguments: {:?}", clang_arguments); + parser.arguments(&clang_arguments); + + parse_unit( + &parser + .parse() + .with_context(|| format!("Could not parse file: {:?}", filename))?, + &mut entities, + &toplevel_directory, + file_map.span, + &codemap, + )?; + + trace!("Changing directory to: {:?}", directory); + std::env::set_current_dir(&toplevel_directory).with_context(|| { + format!( + "Cannot change current directory to: {:?}", + toplevel_directory + ) + })?; + } + + let normalized_entities = entities + .namespaces + .into_iter() + .map(|(usr, entity)| (usr.0, entity.into())) + .chain(entities.variables.into_iter().map(|(usr, entity)| (usr.0, entity.into()))) + .chain(entities.structs.into_iter().map(|(usr, entity)| (usr.0, entity.into()))) + .chain(entities.functions.into_iter().map(|(usr, entity)| (usr.0, entity.into()))) + .collect(); + + Ok(normalized_entities) +} + +/* +pub(crate) fn parse_file(path: T, extra_args: &[S]) -> EntitiesManager +where + T: Into, + T: AsRef, + T: ToString, + S: AsRef, + S: std::fmt::Debug, +{ + let mut codemap = CodeMap::new(); + let file_map = codemap.add_file(path.to_string(), std::fs::read_to_string(&path).unwrap()); + let file_span = file_map.span; + + let clang = Clang::new().unwrap(); + let index = Index::new(&clang, true, false); + let mut parser = index.parser(path); + parser.skip_function_bodies(true); + + parser.arguments(&extra_args); + + if log_enabled!(log::Level::Debug) { + for extra_arg in extra_args { + debug!("Extra libclang argument: {:?}", extra_arg); + } + } + + let trans_unit = parser.parse().unwrap(); + let mut entities = EntitiesManager::new(); + + parse_unit( + &trans_unit, + &mut entities, + &std::env::current_dir().unwrap(), + file_span, + &codemap, + ) + .unwrap(); + + entities +} +*/ + +fn parse_unit( + trans_unit: &TranslationUnit, + entities: &mut TopLevel, + base_dir: impl AsRef, + file_span: codemap::Span, + codemap: &CodeMap, +) -> Result<()> { + trans_unit.get_entity().visit_children(|entity, _parent| { + if is_in_system_header(entity, &base_dir) { + trace!( + "Entity is in system header, skipping: USR = {:?}", + entity.get_display_name() + ); + return clang::EntityVisitResult::Continue; + } + + // TODO: wrap this callback in another function so that we can use the + // "?" operator instead of all these `match`es + let usr = match entity.get_usr() { + Some(usr) => usr, + None => return clang::EntityVisitResult::Continue, + }; + trace!("Entity with USR = {:?}", usr); + debug!("Parsing toplevel entity: {:?}", entity); + + add_entity(entity, entities, file_span, codemap) + }); + + /* + use codemap_diagnostic::{ColorConfig, Emitter}; + + let mut emitter = Emitter::stderr(ColorConfig::Auto, Some(&codemap)); + + for diagnostic in trans_unit.get_diagnostics().iter() { + let main_diag = match clang_diag_to_codemap_diag(&diagnostic, file_span) { + Some(diag) => diag, + None => continue, + }; + + let sub_diags = diagnostic + .get_children() + .into_iter() + .filter_map(|diagnostic| clang_diag_to_codemap_diag(&diagnostic, file_span)); + + let fix_it_diags = diagnostic + .get_fix_its() + .into_iter() + .map(|fix_it| clang_fix_it_to_codemap_diag(&fix_it, file_span)); + + emitter.emit( + &std::iter::once(main_diag) + .chain(sub_diags) + .chain(fix_it_diags) + .collect::>(), + ); + } + */ + + Ok(()) +} + +fn is_in_system_header(entity: clang::Entity, base_dir: impl AsRef) -> bool { + if entity.is_in_system_header() { + true + } else if let Some(location) = entity.get_location() { + if let Some(file) = location.get_file_location().file { + !file + .get_path() + .canonicalize() + .unwrap() + .starts_with(base_dir) + } else { + // Not defined in a file? probably shouldn't document + true + } + } else { + // Not defined anywhere? probably shouldn't document + true + } +} + +// Entries encountered in the toplevel lexical context +fn add_entity( + libclang_entity: clang::Entity, + toplevel: &mut TopLevel, + file_span: codemap::Span, + codemap: &CodeMap, +) -> clang::EntityVisitResult { + if libclang_entity.get_usr().is_none() { + return clang::EntityVisitResult::Continue; + }; + + let kind = match ClangEntityKind::try_from(libclang_entity.get_kind()) { + Ok(kind) => kind, + Err(err) => { + use codemap_diagnostic::{ + ColorConfig, Diagnostic, Emitter, Level, SpanLabel, SpanStyle, + }; + let spans = if let Some(range) = libclang_entity.get_range() { + // TODO: add every file parsed in this translation unit to the + // codemap, so we can properly report errors + if !range.is_in_main_file() { + vec![] + } else { + let begin = range.get_start().get_file_location().offset as u64; + let end = range.get_end().get_file_location().offset as u64; + + vec![SpanLabel { + span: file_span.subspan(begin, end), + label: None, + style: SpanStyle::Primary, + }] + } + } else { + vec![] + }; + + let diag = Diagnostic { + level: Level::Warning, + message: format!("{}", err), + code: None, + spans, + }; + + let mut emitter = Emitter::stderr(ColorConfig::Auto, Some(codemap)); + emitter.emit(&[diag]); + + return clang::EntityVisitResult::Continue; + } + }; + + if let Some(in_tree_entity) = toplevel.get_entity_mut(libclang_entity) { + // if current.has_documentation && !tree.has_documentation { + // append_documentation + // } + } else if libclang_entity.is_definition() { + // TODO: This probably means that you can't put documentation in forward declarations. + // + // This seems restrictive, but since there can be multiple declarations but only one definition, + // you should probably put your documentation on the definition anyway? + // + // Also, skipping forward declarations allows us to not have to insert, then update the tree + // when we see the definition. + + let result = match kind { + ClangEntityKind::Namespace => Described::::try_from(libclang_entity) + .and_then(|namespace| toplevel.insert(libclang_entity, namespace)), + ClangEntityKind::Variable(_) => Described::::try_from(libclang_entity) + .and_then(|variable| toplevel.insert(libclang_entity, variable)), + ClangEntityKind::Struct(_) => Described::::try_from(libclang_entity) + .and_then(|r#struct| toplevel.insert(libclang_entity, r#struct)), + ClangEntityKind::Function(_) => Described::::try_from(libclang_entity) + .and_then(|function| toplevel.insert(libclang_entity, function)), + }; + // TODO: check result + } + + if kind == ClangEntityKind::Namespace { + // Recurse here since namespace definitions are allowed to change between translation units. + ::clang::EntityVisitResult::Recurse + } else { + ::clang::EntityVisitResult::Continue + } +} + +impl<'a, T> TryFrom> for Described +where + T: TryFrom, Error = Error>, +{ + type Error = Error; + + fn try_from(entity: clang::Entity<'a>) -> Result { + Ok(Described:: { + description: get_description(entity)?, + entity: T::try_from(entity)?, + }) + } +} + +impl<'a> TryFrom> for Namespace { + type Error = Error; + + fn try_from(entity: clang::Entity) -> Result { + match entity.get_kind().try_into() { + Ok(ClangEntityKind::Namespace) => {} + _ => panic!("Trying to parse a non-variable into a variable"), + } + debug!("Parsing Namespace: {:?}", entity); + + // Do not recurse here, but recurse in the main loop, since namespace + // definitions is allowed to change between translation units + + Ok(Namespace { + member_namespaces: Default::default(), + member_variables: Default::default(), + member_structs: Default::default(), + member_functions: Default::default(), + }) + } +} + +impl<'a> TryFrom> for Variable { + type Error = Error; + + fn try_from(entity: clang::Entity) -> Result { + let variable_kind; + match entity.get_kind().try_into() { + Ok(ClangEntityKind::Variable(kind)) => { + variable_kind = kind; + } + _ => panic!("Trying to parse a non-variable into a variable"), + } + debug!("Parsing Variable: {:?}", entity); + + let r#type = entity.get_type().unwrap().get_display_name(); + trace!("Variable has type: {:?}", r#type); + + Ok(Variable { + r#type, + kind: variable_kind, + }) + } +} + +impl<'a> TryFrom> for Struct { + type Error = Error; + + fn try_from(entity: clang::Entity) -> Result { + let struct_kind; + match entity.get_kind().try_into() { + Ok(ClangEntityKind::Struct(kind)) => { + struct_kind = kind; + } + _ => panic!("Trying to parse a non-class into a class"), + } + debug!("Parsing Struct: {:?}", entity); + + let mut member_variables = BTreeMap::new(); + let mut member_structs = BTreeMap::new(); + let mut member_functions = BTreeMap::new(); + + for child in entity.get_children() { + trace!("Struct has child: {:?}", child); + + match child.get_kind().try_into() { + Ok(ClangEntityKind::Variable(_)) => { + let child_usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; + member_variables.insert(child_usr, Described::::try_from(child)?); + } + Ok(ClangEntityKind::Struct(_)) => { + let child_usr: Usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; + member_structs.insert(child_usr, Described::::try_from(child)?); + } + Ok(ClangEntityKind::Function(_)) => { + let child_usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; + member_functions.insert(child_usr, Described::::try_from(child)?); + } + _ => trace!("Skipping child"), + } + } + + Ok(Struct { + kind: struct_kind, + member_functions, + member_structs, + member_variables, + }) + } +} + +impl<'a> TryFrom> for Function { + type Error = Error; + + fn try_from(entity: clang::Entity) -> Result { + let function_kind; + match entity.get_kind().try_into() { + Ok(ClangEntityKind::Function(kind)) => { + function_kind = kind; + } + _ => panic!("Trying to parse a non-function into a function"), + } + debug!("Parsing Function: {:?}", entity); + + let return_type = entity.get_result_type().unwrap().get_display_name(); + trace!("Function has return type: {:?}", return_type); + let arguments = entity + .get_arguments() + .unwrap() + .into_iter() + .map(|arg| { + let name = arg + .get_display_name() + .unwrap_or_else(|| String::from("unnamed")); + let r#type = arg.get_type().unwrap().get_display_name(); + trace!("Function has argument {:?} of type {:?}", name, r#type); + FunctionArgument { name, r#type } + }) + .collect(); + + Ok(Function { + kind: function_kind, + arguments, + return_type, + }) + } +} + +fn get_description(entity: clang::Entity) -> Result { + let name = entity + .get_display_name() + .ok_or_else(|| anyhow!("Entity has no name: {:?}", entity))?; + + // TODO: is that the best? + if let (Some(brief), Some(comment)) = (entity.get_comment_brief(), entity.get_comment()) { + Ok(Description { + name, + brief, + detailed: parse_comment(comment), + }) + } else { + Ok(Description { + name, + brief: String::new(), + detailed: String::new(), + }) + } +} + +pub fn parse_comment(raw: String) -> String { + #[derive(Debug)] + enum CommentStyle { + // Comments of type `/**` or `/*!` + Starred, + // Comments of type `///` + SingleLine, + } + + let mut chars = raw.chars(); + let style = match &chars.as_str()[..3] { + "/*!" | "/**" => CommentStyle::Starred, + "///" => CommentStyle::SingleLine, + _ => panic!("Comment is empty or doesn't start with either `///`, `/**`, or `/*!`"), + }; + + chars.nth(2); + + let mut result = String::new(); + + 'parse_loop: loop { + let maybe_space = chars.next(); + let mut empty_line = false; + match maybe_space { + // TODO: Warn on empty comments + None => break, + Some(' ') => {} + Some('\n') => { + empty_line = true; + result.push('\n'); + } + Some(ch) => result.push(ch), + } + + if !empty_line { + let rest = chars.as_str(); + match rest.find('\n') { + None => { + result.push_str(rest); + break; + } + Some(position) => { + result.push_str(&rest[..=position]); + chars.nth(position); + } + } + } + + // Beginning of the line + let first_non_ws_ch = 'ws_loop: loop { + let maybe_whitespace = chars.next(); + match maybe_whitespace { + None => break 'parse_loop, + Some(ch) if ch.is_whitespace() => continue, + Some(ch) => break 'ws_loop ch, + } + }; + + match style { + CommentStyle::Starred if first_non_ws_ch == '*' => { + if &chars.as_str()[..1] == "/" { + break; + } + } + CommentStyle::Starred => result.push(first_non_ws_ch), + CommentStyle::SingleLine => { + assert!(first_non_ws_ch == '/'); + let rest = chars.as_str(); + if &rest[..2] == "//" { + chars.nth(1); + } else if &rest[..1] == "/" { + chars.nth(0); + } else { + panic!("Could not parse comment"); + } + } + } + } + + result +} + +#[derive(Debug, Clone, Error)] +#[error("Failed to load 'compile_commands.json' at path: {:?}", path)] +pub(crate) struct CompileCommandsLoadError { + path: PathBuf, +} diff --git a/src/parsing/mod.rs b/src/parsing/mod.rs new file mode 100644 index 0000000..0736eee --- /dev/null +++ b/src/parsing/mod.rs @@ -0,0 +1 @@ +pub(crate) mod clang; diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..82f9c65 --- /dev/null +++ b/src/types.rs @@ -0,0 +1,22 @@ +use serde_derive::{Serialize, Deserialize}; + +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Poseidoc { + pub config: toml::Value, + pub entities: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Entity { + pub name: String, + //location: SourceLocation + pub language: String, + pub kind: String, + pub brief_description: String, + pub documentation: String, + pub children: BTreeMap>, +} + +// TODO: use newtype for language, entity kind, entity ID -- cgit v1.2.3