Use Clang's -fmodules

Clang has something called "modules", which are a mechanism to describe
and combine a set of headers into a single cohesive unit:
https://clang.llvm.org/docs/Modules.html

This is what Swift uses when mapping frameworks, so it makes sense for
us to do the same. Objective-C also recently gained the `@import` syntax
to do the same.

This should unblock our support for umbrella frameworks, makes it easier
to use API notes in the future, and is generally more correct.

crates/header-translator/Cargo.toml

@@ -23,6 +23,7 @@ heck = "0.5"
 semver = { version = "1.0", features = ["serde"] }
 lenient_semver_parser = "0.4"
 four-char-code = "2.2.0"
+regex = "1.6"
 
 [package.metadata.release]
 release = false

crates/header-translator/src/config.rs

@@ -92,6 +92,18 @@ fn get_version<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<Vers
     deserializer.deserialize_str(VersionVisitor)
 }
 
+#[derive(Deserialize, Debug, Default, Clone, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct ExternalData {
+    pub module: String,
+    #[serde(rename = "thread-safety")]
+    #[serde(default)]
+    pub thread_safety: Option<String>,
+    #[serde(rename = "required-items")]
+    #[serde(default)]
+    pub required_items: Vec<String>,
+}
+
 #[derive(Deserialize, Debug, Default, Clone, PartialEq, Eq)]
 #[serde(deny_unknown_fields)]
 pub struct LibraryConfig {
@@ -137,6 +149,12 @@ pub struct LibraryConfig {
     #[serde(default = "link_default")]
     pub link: bool,
 
+    /// Data about an external class or protocol whose header isn't imported.
+    ///
+    /// I.e. a bare `@protocol X;` or `@class X;`.
+    #[serde(default)]
+    pub external: BTreeMap<String, ExternalData>,
+
     #[serde(rename = "class")]
     #[serde(default)]
     pub class_data: HashMap<String, ClassData>,

crates/header-translator/src/context.rs

@@ -1,84 +1,62 @@
-use std::borrow::Cow;
 use std::collections::HashMap;
 use std::ops;
-use std::path::{Path, PathBuf};
 
-use apple_sdk::SdkPath;
 use clang::Entity;
 
 use crate::config::Config;
-use crate::id::Location;
+use crate::ItemIdentifier;
 
-pub struct Context<'a> {
-    config: &'a Config,
-    pub macro_invocations: HashMap<clang::source::Location<'a>, Entity<'a>>,
-    framework_dir: PathBuf,
-    include_dir: PathBuf,
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct MacroLocation {
+    file_id: Option<(u64, u64, u64)>,
+    line: u32,
+    column: u32,
+    offset: u32,
 }
 
-impl<'a> Context<'a> {
-    pub fn new(config: &'a Config, sdk: &SdkPath) -> Self {
+impl MacroLocation {
+    pub fn from_location(location: &clang::source::SourceLocation<'_>) -> Self {
+        let clang::source::Location {
+            file,
+            line,
+            column,
+            offset,
+        } = location.get_expansion_location();
         Self {
-            config,
-            macro_invocations: Default::default(),
-            framework_dir: sdk.path.join("System/Library/Frameworks"),
-            include_dir: sdk.path.join("usr/include"),
+            file_id: file.map(|f| f.get_id()),
+            line,
+            column,
+            offset,
         }
     }
+}
 
-    pub fn get_location(&self, entity: &Entity<'_>) -> Option<Location> {
-        if let Some(location) = entity.get_location() {
-            if let Some(file) = location.get_file_location().file {
-                let path = file.get_path();
-                if let Ok(path) = path.strip_prefix(&self.framework_dir) {
-                    let mut components: Vec<Cow<'_, str>> = path
-                        .components()
-                        .filter(|component| {
-                            component.as_os_str() != "Headers"
-                                && component.as_os_str() != "Frameworks"
-                        })
-                        .map(|component| component.as_os_str().to_str().expect("component to_str"))
-                        .map(|component| component.strip_suffix(".framework").unwrap_or(component))
-                        .map(|component| component.strip_suffix(".h").unwrap_or(component))
-                        .map(|s| s.to_string().into())
-                        .collect();
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct MacroEntity {
+    pub(crate) id: ItemIdentifier,
+    pub(crate) is_function_like: bool,
+}
 
-                    // Put items in umbrella header in `mod.rs`
-                    if let [.., innermost_framework_name, file_name] = &*components {
-                        let umbrella_header = self
-                            .libraries
-                            .get(&**innermost_framework_name)
-                            .and_then(|lib| lib.umbrella_header.as_deref())
-                            .unwrap_or(innermost_framework_name);
+impl MacroEntity {
+    pub fn from_entity(entity: &Entity<'_>, context: &Context<'_>) -> Self {
+        Self {
+            id: ItemIdentifier::new(entity, context),
+            is_function_like: entity.is_function_like_macro(),
+        }
+    }
+}
 
-                        if file_name == umbrella_header {
-                            let _ = components.pop();
-                        }
-                    }
+pub struct Context<'config> {
+    config: &'config Config,
+    pub macro_invocations: HashMap<MacroLocation, MacroEntity>,
+}
 
-                    return Some(Location::from_components(components));
-                } else if let Ok(path) = path.strip_prefix(&self.include_dir) {
-                    if path.starts_with("objc") {
-                        return Some(Location::from_components(vec!["objc2".into()]));
-                    }
-                    if path == Path::new("MacTypes.h") {
-                        return Some(Location::from_components(vec!["System".into()]));
-                    }
-                    if path.starts_with("sys") {
-                        return Some(Location::from_components(vec!["libc".into()]));
-                    }
-                    if path.starts_with("mach") {
-                        // Will be moved to the `mach` crate in `libc` v1.0
-                        return Some(Location::from_components(vec!["libc".into()]));
-                    }
-                    if path.starts_with("arm") {
-                        // Temporary
-                        return Some(Location::from_components(vec!["System".into()]));
-                    }
-                }
-            }
+impl<'config> Context<'config> {
+    pub fn new(config: &'config Config) -> Self {
+        Self {
+            config,
+            macro_invocations: Default::default(),
         }
-        None
     }
 }
 

crates/header-translator/src/expr.rs

@@ -4,6 +4,7 @@ use std::fmt;
 use clang::token::TokenKind;
 use clang::{Entity, EntityKind, EntityVisitResult, EvaluationResult};
 
+use crate::context::MacroLocation;
 use crate::rust_type::Ty;
 use crate::stmt::{enum_constant_name, new_enum_id};
 use crate::unexposed_attr::UnexposedAttr;
@@ -101,10 +102,10 @@ impl Expr {
             let location = entity.get_location().expect("expr location");
             if let Some(macro_invocation) = context
                 .macro_invocations
-                .get(&location.get_spelling_location())
+                .get(&MacroLocation::from_location(&location))
             {
                 return Expr::MacroInvocation {
-                    id: ItemIdentifier::new(macro_invocation, context),
+                    id: macro_invocation.id.clone(),
                     evaluated: Some(Box::new(Self::from_evaluated(entity))),
                 };
             } else {
@@ -122,10 +123,10 @@ impl Expr {
                     } else {
                         let macro_invocation = context
                             .macro_invocations
-                            .get(&token.get_location().get_spelling_location())
+                            .get(&MacroLocation::from_location(&token.get_location()))
                             .expect("expr macro invocation");
                         Expr::MacroInvocation {
-                            id: ItemIdentifier::new(macro_invocation, context),
+                            id: macro_invocation.id.clone(),
                             evaluated: None,
                         }
                     })

crates/header-translator/src/id.rs

@@ -4,6 +4,7 @@ use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::collections::BTreeSet;
 
+use clang::source::Module;
 use clang::Entity;
 
 use crate::cfgs::PlatformCfg;
@@ -122,6 +123,35 @@ impl<'config> LocationLibrary<'_, 'config> {
 }
 
 impl Location {
+    fn from_module(module: Module<'_>) -> Self {
+        let full_name = module.get_full_name();
+
+        Location::from_components(match &*full_name {
+            // Objective-C
+            name if name.starts_with("ObjectiveC") => vec!["objc2".into()],
+
+            // Redefined in the framework crate itself.
+            "Darwin.MacTypes" => vec!["System".into()],
+
+            // Built-in
+            "DarwinFoundation.types.machine_types" => vec!["System".into()],
+
+            // Libc
+            name if name.starts_with("sys_types") => vec!["libc".into()],
+            "DarwinFoundation.types.sys_types" => vec!["libc".into()],
+            name if name.starts_with("Darwin.POSIX") => vec!["libc".into()],
+
+            // Will be moved to the `mach2` crate in `libc` v1.0
+            name if name.starts_with("Darwin.Mach") => vec!["libc".into()],
+            "_mach_port_t" => vec!["libc".into()],
+
+            full_name => full_name
+                .split('.')
+                .map(|component| Cow::Owned(component.to_string()))
+                .collect(),
+        })
+    }
+
     pub(crate) fn from_components(path_components: Vec<Cow<'static, str>>) -> Self {
         Self { path_components }
     }
@@ -237,13 +267,23 @@ impl<N: ToOptionString> ItemIdentifier<N> {
         }
     }
 
-    pub fn with_name(name: N, entity: &Entity<'_>, context: &Context<'_>) -> Self {
-        let mut location = context.get_location(entity).unwrap_or_else(|| {
-            error!(?entity, "ItemIdentifier from unknown header");
-            Location::from_components(vec!["__Unknown__".into()])
-        });
+    pub fn with_name(name: N, entity: &Entity<'_>, _context: &Context<'_>) -> Self {
+        let file = entity
+            .get_location()
+            .expect("entity location")
+            .get_expansion_location()
+            .file
+            .expect("expanded location file");
+
+        let mut location = if let Some(module) = file.get_module() {
+            Location::from_module(module)
+        } else {
+            // If file module is not available, the item is likely a built-in macro.
+            Location::from_components(vec!["System".into()])
+        };
 
-        if let Some("IOSurfaceRef") = name.to_option() {
+        // Defined in multiple places for some reason.
+        if let Some("IOSurfaceRef" | "__IOSurface") = name.to_option() {
             location = Location::from_components(vec!["IOSurface".into(), "IOSurfaceRef".into()]);
         }
 

crates/header-translator/src/lib.rs

@@ -28,7 +28,7 @@ mod thread_safety;
 mod unexposed_attr;
 
 pub use self::config::{Config, LibraryConfig};
-pub use self::context::Context;
+pub use self::context::{Context, MacroEntity, MacroLocation};
 pub use self::global_analysis::global_analysis;
 pub use self::id::{ItemIdentifier, Location};
 pub use self::library::Library;

crates/header-translator/src/library.rs

@@ -15,7 +15,6 @@ use crate::config::LibraryConfig;
 use crate::display_helper::FormatterFn;
 use crate::module::Module;
 use crate::Config;
-use crate::Location;
 use crate::VERSION;
 
 #[derive(Debug, PartialEq)]
@@ -36,20 +35,26 @@ impl Library {
         }
     }
 
-    pub fn add_module(&mut self, path: Vec<String>) {
+    pub fn add_module(&mut self, components: Vec<String>) {
         let mut current = &mut self.module;
-        for p in path {
-            current = current.submodules.entry(p.clone()).or_default();
+        for component in components {
+            current = current.submodules.entry(component).or_default();
         }
     }
 
-    pub fn module_mut(&mut self, location: &Location) -> &mut Module {
+    pub fn module_mut(&mut self, mut module: clang::source::Module<'_>) -> &mut Module {
+        let mut components = vec![];
+        while let Some(parent) = module.get_parent() {
+            components.insert(0, module.get_name());
+            module = parent;
+        }
+
         let mut current = &mut self.module;
-        for p in location.modules() {
-            current = match current.submodules.entry(p.to_string()) {
+        for component in components {
+            current = match current.submodules.entry(component) {
                 Entry::Occupied(entry) => entry.into_mut(),
                 Entry::Vacant(entry) => {
-                    warn!(?location, "expected module to be available in library");
+                    error!(?module, "expected module to be available in library");
                     entry.insert(Default::default())
                 }
             };