From 5d6f163000bf34d12ec9587b2d8572319d27607d Mon Sep 17 00:00:00 2001 From: Minijackson Date: Sun, 12 Jan 2020 13:28:40 +0100 Subject: clang: allow parse single file + parse typedefs --- src/cli.rs | 7 +- src/main.rs | 21 +++- src/parser/clang/entities.rs | 55 ++++++++- src/parser/clang/mod.rs | 2 +- src/parser/clang/parsing.rs | 282 +++++++++++++++++++++++++++++++------------ 5 files changed, 275 insertions(+), 92 deletions(-) (limited to 'src') diff --git a/src/cli.rs b/src/cli.rs index 0000644..946ebc9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -5,6 +5,7 @@ use structopt::StructOpt; use std::path::PathBuf; #[derive(Debug, Clone, StructOpt)] +#[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] pub(crate) struct Cli { #[structopt(long, short, parse(from_occurrences))] pub(crate) verbosity: u8, @@ -22,9 +23,11 @@ pub(crate) struct Cli { #[derive(Debug, Clone, StructOpt)] pub(crate) enum Command { Generate { - file: String, + file: Option, + }, + Inspect { + file: Option, }, - Inspect, Config { #[structopt(subcommand)] command: ConfigCommand, diff --git a/src/main.rs b/src/main.rs index 5c613e0..5a0b175 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +#![deny(unsafe_code)] #![warn(clippy::all)] mod cli; @@ -83,19 +84,27 @@ fn start(codemap: &mut CodeMap) -> Result<()> { .with_context(|| format!("Cannot change current directory to: {:?}", cli.directory))?; match &cli.command { - Command::Generate { file: _ } => { - //let file = file.clone(); + Command::Generate { file } => { + let file = file.clone(); let config = load_effective_config(cli, codemap)?; - let entities = parser::clang::parse_compile_commands(&config.clang, codemap)?; - //let manager = parse_file(file, &config.extra_clang_args); + let entities = if let Some(file) = file { + parser::clang::parse_file(file, &config.clang, codemap)? + } else { + parser::clang::parse_compile_commands(&config.clang, codemap)? + }; let base_output_dir = std::path::Path::new("doc"); generate(&base_output_dir, entities, &config.generator)?; } - Command::Inspect => { + Command::Inspect { file } => { + let file = file.clone(); let config = load_effective_config(cli, codemap)?; - let entities = parser::clang::parse_compile_commands(&config.clang, codemap)?; + let entities = if let Some(file) = file { + parser::clang::parse_file(file, &config.clang, codemap)? + } else { + parser::clang::parse_compile_commands(&config.clang, codemap)? + }; serde_json::to_writer_pretty(std::io::stdout().lock(), &entities)?; } Command::Config { diff --git a/src/parser/clang/entities.rs b/src/parser/clang/entities.rs index cbc17b7..a5b6462 100644 --- a/src/parser/clang/entities.rs +++ b/src/parser/clang/entities.rs @@ -14,6 +14,7 @@ pub(super) struct Children { variables: Option>>, structs: Option>>, functions: Option>>, + typedefs: Option>>, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -22,6 +23,7 @@ pub(super) enum ClangEntityKind { Variable(VariableKind), Struct(StructKind), Function(FunctionKind), + Typedef, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -52,6 +54,7 @@ impl ClangEntityKind { ClangEntityKind::Struct(StructKind::Class) => "class", ClangEntityKind::Function(FunctionKind::Function) => "function", ClangEntityKind::Function(FunctionKind::Method) => "method", + ClangEntityKind::Typedef => "typedef", } } @@ -64,6 +67,7 @@ impl ClangEntityKind { ClangEntityKind::Struct(StructKind::Class) => "classes", ClangEntityKind::Function(FunctionKind::Function) => "functions", ClangEntityKind::Function(FunctionKind::Method) => "methods", + ClangEntityKind::Typedef => "typedefs", } } } @@ -80,12 +84,19 @@ impl TryFrom for ClangEntityKind { EntityKind::Namespace => ClangEntityKind::Namespace, EntityKind::VarDecl => ClangEntityKind::Variable(VariableKind::Variable), EntityKind::FieldDecl => ClangEntityKind::Variable(VariableKind::Field), - EntityKind::FunctionDecl => ClangEntityKind::Function(FunctionKind::Function), - EntityKind::Method | EntityKind::Constructor => { + EntityKind::FunctionDecl | EntityKind::FunctionTemplate => { + ClangEntityKind::Function(FunctionKind::Function) + } + EntityKind::Method | EntityKind::Constructor | EntityKind::Destructor => { ClangEntityKind::Function(FunctionKind::Method) } EntityKind::StructDecl => ClangEntityKind::Struct(StructKind::Struct), - EntityKind::ClassDecl => ClangEntityKind::Struct(StructKind::Class), + EntityKind::ClassDecl | EntityKind::ClassTemplate => { + ClangEntityKind::Struct(StructKind::Class) + } + EntityKind::TypedefDecl + | EntityKind::TypeAliasDecl + | EntityKind::TypeAliasTemplateDecl => ClangEntityKind::Typedef, kind => return Err(TryFromEntityKindError(format!("{:?}", kind))), }) } @@ -106,6 +117,9 @@ pub(super) trait ClangEntity { fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { None } + fn get_member_typedefs(&mut self) -> Option<&mut BTreeMap>> { + None + } fn into_children(self) -> Children where @@ -116,6 +130,7 @@ pub(super) trait ClangEntity { variables: None, structs: None, functions: None, + typedefs: None, } } } @@ -169,12 +184,23 @@ where } } +impl NamespaceParentManipulation for U +where + U: ClangEntity, +{ + fn get_members_mut(&mut self) -> Option<&mut BTreeMap>> { + self.get_member_typedefs() + } +} + fn entity_id(usr: Usr) -> EntityId { // This is a somewhat ugly workaround, because Pandoc parses markdown identifiers as alphanum // or one of "-_:.", which means Pandoc can't parse libclang's USRs in title ids and related. // // - EntityId(usr.0.replace("@", "::").replace("#", ".").replace("$", "-")) + EntityId( + usr.0.replace("@", "::").replace("#", ".").replace("$", "-") + ) } fn append_children( @@ -243,6 +269,7 @@ pub(super) struct Namespace { pub(super) member_variables: BTreeMap>, pub(super) member_structs: BTreeMap>, pub(super) member_functions: BTreeMap>, + pub(super) member_typedefs: BTreeMap>, } impl ClangEntity for Namespace { @@ -262,6 +289,9 @@ impl ClangEntity for Namespace { fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { Some(&mut self.member_functions) } + fn get_member_typedefs(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_typedefs) + } fn into_children(self) -> Children { Children { @@ -269,6 +299,7 @@ impl ClangEntity for Namespace { variables: Some(self.member_variables), structs: Some(self.member_structs), functions: Some(self.member_functions), + typedefs: Some(self.member_typedefs), } } } @@ -314,6 +345,7 @@ pub(super) struct Struct { pub(super) member_variables: BTreeMap>, pub(super) member_structs: BTreeMap>, pub(super) member_functions: BTreeMap>, + pub(super) member_typedefs: BTreeMap>, } impl ClangEntity for Struct { @@ -330,6 +362,9 @@ impl ClangEntity for Struct { fn get_member_functions(&mut self) -> Option<&mut BTreeMap>> { Some(&mut self.member_functions) } + fn get_member_typedefs(&mut self) -> Option<&mut BTreeMap>> { + Some(&mut self.member_typedefs) + } fn into_children(self) -> Children { Children { @@ -337,6 +372,7 @@ impl ClangEntity for Struct { variables: Some(self.member_variables), structs: Some(self.member_structs), functions: Some(self.member_functions), + typedefs: Some(self.member_typedefs), } } } @@ -381,3 +417,14 @@ pub(super) struct FunctionArgument { pub(super) name: String, pub(super) r#type: String, } + +#[derive(Debug, Clone)] +pub(super) struct Typedef { + pub(super) referee: String, +} + +impl ClangEntity for Typedef { + fn kind(&self) -> ClangEntityKind { + ClangEntityKind::Typedef + } +} diff --git a/src/parser/clang/mod.rs b/src/parser/clang/mod.rs index 047c49e..da49462 100644 --- a/src/parser/clang/mod.rs +++ b/src/parser/clang/mod.rs @@ -2,4 +2,4 @@ pub(crate) mod config; mod entities; mod parsing; -pub(crate) use parsing::parse_compile_commands; +pub(crate) use parsing::{parse_compile_commands, parse_file}; diff --git a/src/parser/clang/parsing.rs b/src/parser/clang/parsing.rs index a654681..1f1691f 100644 --- a/src/parser/clang/parsing.rs +++ b/src/parser/clang/parsing.rs @@ -11,12 +11,15 @@ use std::collections::BTreeMap; use std::convert::{TryFrom, TryInto}; use std::path::{Path, PathBuf}; +// TODO: check for use of Extend instead of loop+insert + #[derive(Debug, Default)] struct TopLevel { namespaces: BTreeMap>, variables: BTreeMap>, structs: BTreeMap>, functions: BTreeMap>, + typedefs: BTreeMap>, } /* @@ -82,6 +85,9 @@ impl TopLevel { ClangEntityKind::Struct(_) => { &mut parent.get_member_structs()?.get_mut(&usr)?.entity } + ClangEntityKind::Typedef => { + &mut parent.get_member_typedefs()?.get_mut(&usr)?.entity + } }) } else { Some(match path.get_kind().try_into().ok()? { @@ -89,6 +95,7 @@ impl TopLevel { ClangEntityKind::Variable(_) => &mut self.variables.get_mut(&usr)?.entity, ClangEntityKind::Struct(_) => &mut self.structs.get_mut(&usr)?.entity, ClangEntityKind::Function(_) => &mut self.functions.get_mut(&usr)?.entity, + ClangEntityKind::Typedef => &mut self.typedefs.get_mut(&usr)?.entity, }) } } @@ -121,7 +128,10 @@ impl TopLevel { .insert(usr, entity); Ok(()) } else { - Err(anyhow!("has parent but no parent in tree")) + Err(anyhow!( + "has parent: {:?} but no parent in tree", + parent_path + )) } } else { self.insert_toplevel(usr, entity); @@ -175,6 +185,12 @@ impl TopLevelManipulation for TopLevel { } } +impl TopLevelManipulation for TopLevel { + fn insert_toplevel(&mut self, usr: Usr, entity: Described) { + self.typedefs.insert(usr, entity); + } +} + /* trait FromTopLevel: ClangEntity + Sized { fn from_toplevel<'a>(toplevel: &'a mut TopLevel, usr: &Usr) -> Option<&'a mut Described>; @@ -268,7 +284,7 @@ pub(crate) fn parse_compile_commands( &codemap, )?; - trace!("Changing directory to: {:?}", directory); + trace!("Changing directory to: {:?}", toplevel_directory); std::env::set_current_dir(&toplevel_directory).with_context(|| { format!( "Cannot change current directory to: {:?}", @@ -277,74 +293,77 @@ pub(crate) fn parse_compile_commands( })?; } - let normalized_entities = entities - .namespaces - .into_iter() - .map(|(usr, entity)| (EntityId(usr.0), entity.into())) - .chain( - entities - .variables - .into_iter() - .map(|(usr, entity)| (EntityId(usr.0), entity.into())), - ) - .chain( - entities - .structs - .into_iter() - .map(|(usr, entity)| (EntityId(usr.0), entity.into())), - ) - .chain( - entities - .functions - .into_iter() - .map(|(usr, entity)| (EntityId(usr.0), entity.into())), - ) - .collect(); - - Ok(normalized_entities) + Ok(entities.into()) } -/* -pub(crate) fn parse_file(path: T, extra_args: &[S]) -> EntitiesManager +pub(crate) fn parse_file( + path: T, + config: &Config, + codemap: &mut CodeMap, +) -> Result> where T: Into, T: AsRef, T: ToString, - S: AsRef, - S: std::fmt::Debug, { - let mut codemap = CodeMap::new(); - let file_map = codemap.add_file(path.to_string(), std::fs::read_to_string(&path).unwrap()); - let file_span = file_map.span; - let clang = Clang::new().unwrap(); let index = Index::new(&clang, true, false); - let mut parser = index.parser(path); - parser.skip_function_bodies(true); - parser.arguments(&extra_args); + // as provided in the command line + let filename = path.to_string(); + let file_map = codemap.add_file(filename.clone(), std::fs::read_to_string(&path)?); - if log_enabled!(log::Level::Debug) { - for extra_arg in extra_args { - debug!("Extra libclang argument: {:?}", extra_arg); - } + let path = path.as_ref().canonicalize()?; + let toplevel_directory = std::env::current_dir().context("Cannot read current directory")?; + + let maybe_commands = CompilationDatabase::from_directory(&config.compile_commands_location) + .and_then(|database| database.get_compile_commands(&path)) + .ok(); + let maybe_command = maybe_commands + .as_ref() + .and_then(|commands| commands.get_commands().pop()); + + let mut clang_arguments = maybe_command + .map(|command| command.get_arguments()) + .unwrap_or_default(); + clang_arguments.extend_from_slice(&config.extra_args); + + if let Some(command) = maybe_command { + let directory = command.get_directory(); + trace!("Changing directory to: {:?}", directory); + std::env::set_current_dir(&directory) + .with_context(|| format!("Cannot change current directory to: {:?}", directory))?; } - let trans_unit = parser.parse().unwrap(); - let mut entities = EntitiesManager::new(); + let mut parser = index.parser(""); + parser.skip_function_bodies(true); + + parser.arguments(&clang_arguments); + + trace!("Parsing with libclang arguments: {:?}", clang_arguments); + + let mut entities = TopLevel::default(); parse_unit( - &trans_unit, + &parser + .parse() + .with_context(|| format!("Could not parse file: {:?}", filename))?, &mut entities, - &std::env::current_dir().unwrap(), - file_span, + &toplevel_directory, + file_map.span, &codemap, - ) - .unwrap(); + )?; - entities + trace!("Changing directory to: {:?}", toplevel_directory); + std::env::set_current_dir(&toplevel_directory).with_context(|| { + format!( + "Cannot change current directory to: {:?}", + toplevel_directory + ) + })?; + + Ok(entities.into()) } -*/ fn parse_unit( trans_unit: &TranslationUnit, @@ -355,10 +374,7 @@ fn parse_unit( ) -> Result<()> { trans_unit.get_entity().visit_children(|entity, _parent| { if is_in_system_header(entity, &base_dir) { - trace!( - "Entity is in system header, skipping: USR = {:?}", - entity.get_display_name() - ); + trace!("Entity is in system header, skipping: {:?}", entity); return clang::EntityVisitResult::Continue; } @@ -374,12 +390,13 @@ fn parse_unit( add_entity(entity, entities, file_span, codemap) }); - /* use codemap_diagnostic::{ColorConfig, Emitter}; let mut emitter = Emitter::stderr(ColorConfig::Auto, Some(&codemap)); for diagnostic in trans_unit.get_diagnostics().iter() { + warn!("{}", diagnostic); + /* let main_diag = match clang_diag_to_codemap_diag(&diagnostic, file_span) { Some(diag) => diag, None => continue, @@ -401,8 +418,8 @@ fn parse_unit( .chain(fix_it_diags) .collect::>(), ); + */ } - */ Ok(()) } @@ -412,11 +429,12 @@ fn is_in_system_header(entity: clang::Entity, base_dir: impl AsRef) -> boo true } else if let Some(location) = entity.get_location() { if let Some(file) = location.get_file_location().file { - !file - .get_path() - .canonicalize() - .unwrap() - .starts_with(base_dir) + // !file + // .get_path() + // .canonicalize() + // .unwrap() + // .starts_with(base_dir) + false } else { // Not defined in a file? probably shouldn't document true @@ -481,7 +499,8 @@ fn add_entity( // if current.has_documentation && !tree.has_documentation { // append_documentation // } - } else if libclang_entity.is_definition() { + } else { + //if libclang_entity.is_definition() { // TODO: This probably means that you can't put documentation in forward declarations. // TODO: Ad-hoc toplevel functions are not documented // @@ -492,19 +511,32 @@ fn add_entity( // when we see the definition. let result = match kind { - ClangEntityKind::Namespace => Described::::try_from(libclang_entity) - .and_then(|namespace| toplevel.insert(libclang_entity, namespace)), + ClangEntityKind::Namespace => { + if libclang_entity.get_name().is_some() { + Described::::try_from(libclang_entity) + .and_then(|namespace| toplevel.insert(libclang_entity, namespace)) + } else { + Ok(()) + } + } ClangEntityKind::Variable(_) => Described::::try_from(libclang_entity) .and_then(|variable| toplevel.insert(libclang_entity, variable)), ClangEntityKind::Struct(_) => Described::::try_from(libclang_entity) .and_then(|r#struct| toplevel.insert(libclang_entity, r#struct)), ClangEntityKind::Function(_) => Described::::try_from(libclang_entity) .and_then(|function| toplevel.insert(libclang_entity, function)), + ClangEntityKind::Typedef => Described::::try_from(libclang_entity) + .and_then(|function| toplevel.insert(libclang_entity, function)), }; // TODO: check result + + if let Err(err) = result { + error!("{}: {:?}", err, libclang_entity); + return ::clang::EntityVisitResult::Continue; + } } - if kind == ClangEntityKind::Namespace { + if kind == ClangEntityKind::Namespace && libclang_entity.get_name().is_some() { // Recurse here since namespace definitions are allowed to change between translation units. ::clang::EntityVisitResult::Recurse } else { @@ -512,6 +544,40 @@ fn add_entity( } } +impl From for BTreeMap { + fn from(toplevel: TopLevel) -> Self { + toplevel + .namespaces + .into_iter() + .map(|(usr, entity)| (EntityId(usr.0), entity.into())) + .chain( + toplevel + .variables + .into_iter() + .map(|(usr, entity)| (EntityId(usr.0), entity.into())), + ) + .chain( + toplevel + .structs + .into_iter() + .map(|(usr, entity)| (EntityId(usr.0), entity.into())), + ) + .chain( + toplevel + .functions + .into_iter() + .map(|(usr, entity)| (EntityId(usr.0), entity.into())), + ) + .chain( + toplevel + .typedefs + .into_iter() + .map(|(usr, entity)| (EntityId(usr.0), entity.into())), + ) + .collect() + } +} + impl<'a, T> TryFrom> for Described where T: TryFrom, Error = Error>, @@ -544,6 +610,7 @@ impl<'a> TryFrom> for Namespace { member_variables: Default::default(), member_structs: Default::default(), member_functions: Default::default(), + member_typedefs: Default::default(), }) } } @@ -580,31 +647,65 @@ impl<'a> TryFrom> for Struct { Ok(ClangEntityKind::Struct(kind)) => { struct_kind = kind; } - _ => panic!("Trying to parse a non-class into a class"), + _ => panic!("Trying to parse a non-struct into a struct"), } debug!("Parsing Struct: {:?}", entity); let mut member_variables = BTreeMap::new(); let mut member_structs = BTreeMap::new(); let mut member_functions = BTreeMap::new(); + let mut member_typedefs = BTreeMap::new(); for child in entity.get_children() { trace!("Struct has child: {:?}", child); - match child.get_kind().try_into() { - Ok(ClangEntityKind::Variable(_)) => { - let child_usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; - member_variables.insert(child_usr, Described::::try_from(child)?); + let kind = child.get_kind(); + + match kind { + ::clang::EntityKind::AccessSpecifier | ::clang::EntityKind::BaseSpecifier => { + continue } - Ok(ClangEntityKind::Struct(_)) => { - let child_usr: Usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; - member_structs.insert(child_usr, Described::::try_from(child)?); + _ => {} + } + + let mut parse_child = || -> Result<()> { + match kind.try_into() { + Ok(ClangEntityKind::Variable(_)) => { + let child_usr = child + .get_usr() + .ok_or_else(|| anyhow!("no usr for: {:?}", child))?; + member_variables.insert(child_usr, Described::::try_from(child)?); + } + Ok(ClangEntityKind::Struct(_)) => { + let child_usr: Usr = child + .get_usr() + .ok_or_else(|| anyhow!("no usr for: {:?}", child))?; + member_structs.insert(child_usr, Described::::try_from(child)?); + } + Ok(ClangEntityKind::Function(_)) => { + let child_usr = child + .get_usr() + .ok_or_else(|| anyhow!("no usr for: {:?}", child))?; + member_functions.insert(child_usr, Described::::try_from(child)?); + } + Ok(ClangEntityKind::Typedef) => { + let child_usr = child + .get_usr() + .ok_or_else(|| anyhow!("no usr for: {:?}", child))?; + member_typedefs.insert(child_usr, Described::::try_from(child)?); + } + Ok(other) => warn!("Unsupported child of struct {:?}: {:?}", other, child), + Err(err) => info!("Error while parsing entity {:?}: {}", child, err), } - Ok(ClangEntityKind::Function(_)) => { - let child_usr = child.get_usr().ok_or_else(|| anyhow!("no usr"))?; - member_functions.insert(child_usr, Described::::try_from(child)?); + + Ok(()) + }; + + match parse_child() { + Ok(()) => {} + Err(err) => { + warn!("Error while parsing child {:?}: {}", child, err); } - _ => trace!("Skipping child"), } } @@ -613,6 +714,7 @@ impl<'a> TryFrom> for Struct { member_functions, member_structs, member_variables, + member_typedefs, }) } } @@ -634,7 +736,9 @@ impl<'a> TryFrom> for Function { trace!("Function has return type: {:?}", return_type); let arguments = entity .get_arguments() - .unwrap() + // TODO: this seems weird, but it fixes a None for a function that takes only a + // variadic argument from its own template declaration. + .unwrap_or_else(|| vec![]) .into_iter() .map(|arg| { let name = arg @@ -654,6 +758,26 @@ impl<'a> TryFrom> for Function { } } +impl<'a> TryFrom> for Typedef { + type Error = Error; + + fn try_from(entity: clang::Entity) -> Result { + match entity.get_kind().try_into() { + Ok(ClangEntityKind::Typedef) => {} + _ => panic!("Trying to parse a non-typedef into a typedef"), + } + debug!("Parsing typedef: {:?}", entity); + + // TODO: unwrap (and unwrap in other similar places too) + let referee = entity + .get_typedef_underlying_type() + .ok_or_else(|| anyhow!("No underlying type"))? + .get_display_name(); + + Ok(Typedef { referee }) + } +} + fn get_description(entity: clang::Entity) -> Result { let name = entity .get_display_name() -- cgit v1.2.3