diff --git a/crates/ty_project/src/metadata.rs b/crates/ty_project/src/metadata.rs index 4a10bafef6..19a0a2e7ac 100644 --- a/crates/ty_project/src/metadata.rs +++ b/crates/ty_project/src/metadata.rs @@ -21,7 +21,7 @@ pub mod pyproject; pub mod settings; pub mod value; -#[derive(Debug, PartialEq, Eq, get_size2::GetSize)] +#[derive(Clone, Debug, PartialEq, Eq, get_size2::GetSize)] #[cfg_attr(test, derive(serde::Serialize))] pub struct ProjectMetadata { pub(super) name: Name, @@ -134,13 +134,25 @@ impl ProjectMetadata { path: &SystemPath, system: &dyn System, ) -> Result { + if !system.is_directory(path) { + return Err(ProjectMetadataError::NotADirectory(path.to_path_buf())); + } + + let closest = Self::discover_closest(path, system)?; + Ok(closest.into_metadata()) + } + + fn discover_closest( + path: &SystemPath, + system: &dyn System, + ) -> Result { tracing::debug!("Searching for a project in '{path}'"); if !system.is_directory(path) { return Err(ProjectMetadataError::NotADirectory(path.to_path_buf())); } - let mut closest_project: Option = None; + let mut closest_project: Option = None; for project_root in path.ancestors() { let Some(discovered) = Self::discover_in(project_root, system)? else { @@ -150,51 +162,52 @@ impl ProjectMetadata { match discovered { DiscoveredProject::PyProject { has_ty_section: true, - metadata, + .. } - | DiscoveredProject::Ty { metadata } => { + | DiscoveredProject::Ty { .. } => { tracing::debug!("Found project at '{}'", project_root); - return Ok(metadata); + return Ok(discovered); } - DiscoveredProject::PyProject { metadata, .. } => { + DiscoveredProject::PyProject { .. } => { // Not a project itself, keep looking for an enclosing project. if closest_project.is_none() { - closest_project = Some(metadata); + closest_project = Some(discovered); } } } } - // No project found, but maybe a pyproject.toml was found. - let metadata = if let Some(closest_project) = closest_project { + if let Some(closest_project) = closest_project { tracing::debug!( "Project without `tool.ty` section: '{}'", closest_project.root() ); - closest_project - } else { - tracing::debug!( - "The ancestor directories contain no `pyproject.toml`. Falling back to a virtual project." - ); + return Ok(closest_project); + } - // Create a project with a default configuration - Self::new( + tracing::debug!( + "The ancestor directories contain no `pyproject.toml`. Falling back to a virtual project." + ); + + // Create a project with a default configuration + Ok(DiscoveredProject::PyProject { + metadata: Self::new( path.file_name().unwrap_or("root").into(), path.to_path_buf(), - ) - }; - - Ok(metadata) + ), + has_ty_section: false, + }) } + /// Discovers a project in `project_root`. Unlike [`Self::discover`], this function + /// only searches for a configuration in `project_root` + /// without traversing the ancestor directories. fn discover_in( project_root: &SystemPath, system: &dyn System, ) -> Result, ProjectMetadataError> { - tracing::debug!("Searching for a project in '{project_root}'"); - if !system.is_directory(project_root) { return Err(ProjectMetadataError::NotADirectory( project_root.to_path_buf(), @@ -284,6 +297,89 @@ impl ProjectMetadata { Ok(None) } + /// Discovers all project in `root`, recursively. + pub fn discover_all( + root: &SystemPath, + system: &dyn System, + ) -> Result, ProjectMetadataError> { + tracing::debug!("Searching for all projects in '{root}'"); + + // FIXME: We need to know if it is a pyproject toml or not :( + let root_project = Self::discover_closest(root, system)?; + + let projects: BTreeMap<_, _> = [(root.to_path_buf(), root_project)].into_iter().collect(); + + // Hmm, what's complicated about this is that + // `check_project` now descends into sub directories so that + // a single file now belongs to multiple projects. + // We would need to exclude the inner projects from the outer project, + // but that seems annoying. + // The alternative is that we skip a directory as soon as we've found a project. But that + // still doesn't solve the case where we have sub folders that each are a project + // with an outermost default database + let projects = std::sync::Mutex::new(projects); + + system + .walk_directory(root) + .standard_filters(true) + .ignore_hidden(true) + .run(|| { + Box::new(|entry| { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + tracing::debug!("Failed to walk directory {error}'"); + return WalkState::Skip; + } + }; + + if entry.depth() == 0 { + return WalkState::Continue; + } + + if !entry.file_type().is_directory() { + return WalkState::Skip; + } + + // TODO: Do propage the error somehow + let discovered = match Self::discover_in(entry.path(), system) { + Ok(Some(discovered)) => discovered, + Ok(None) => return WalkState::Continue, + Err(error) => { + tracing::debug!( + "Failed to discover project in {path}: {error}", + path = entry.path() + ); + + return WalkState::Skip; + } + }; + + let mut projects = projects.lock().unwrap(); + + // Skip the project if there's an outer project that uses either a `ty.toml` or + // `pyproject.toml` with a `tool.ty` section. + if let Some((parent_path, parent_project)) = projects.range(..entry.path().to_path_buf()).last() { + if parent_project.takes_priority_over(&discovered) { + tracing::debug!("Ignoring project at {path} because the parent configuration at {parent_path} takes priority", path = entry.path()); + return WalkState::Continue; + } + } + + projects.insert(entry.path().to_path_buf(), discovered); + + WalkState::Continue + }) + }); + + Ok(projects + .into_inner() + .unwrap() + .into_iter() + .map(|(path, discovered)| (path, discovered.into_metadata())) + .collect()) + } + pub fn root(&self) -> &SystemPath { &self.root } @@ -388,19 +484,36 @@ enum DiscoveredProject { } impl DiscoveredProject { - fn is_ty_or_project_with_ty_section(&self) -> bool { + fn is_ty_or_has_ty_seciton(&self) -> bool { match self { DiscoveredProject::PyProject { - has_ty_section: tool_ty, - .. - } => *tool_ty, + has_ty_section, + metadata: _, + } => *has_ty_section, DiscoveredProject::Ty { .. } => true, } } + fn takes_priority_over(&self, other: &DiscoveredProject) -> bool { + self.is_ty_or_has_ty_seciton() && !other.is_ty_or_has_ty_seciton() + } + + fn root(&self) -> &SystemPath { + match self { + DiscoveredProject::PyProject { + has_ty_section: _, + metadata, + } => &metadata.root(), + DiscoveredProject::Ty { metadata } => metadata.root(), + } + } + fn into_metadata(self) -> ProjectMetadata { match self { - DiscoveredProject::PyProject { metadata, .. } => metadata, + DiscoveredProject::PyProject { + has_ty_section: _, + metadata, + } => metadata, DiscoveredProject::Ty { metadata } => metadata, } } @@ -432,7 +545,7 @@ mod tests { assert_eq!(project.root(), &*root); - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(&project, @r#" ProjectMetadata( name: Name("app"), @@ -470,7 +583,7 @@ mod tests { assert_eq!(project.root(), &*root); - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(&project, @r#" ProjectMetadata( name: Name("backend"), @@ -562,7 +675,7 @@ unclosed table, expected `]` let sub_project = ProjectMetadata::discover(&root.join("packages/a"), &system)?; - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(sub_project, @r#" ProjectMetadata( name: Name("nested-project"), @@ -612,7 +725,7 @@ unclosed table, expected `]` let root = ProjectMetadata::discover(&root, &system)?; - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(root, @r#" ProjectMetadata( name: Name("project-root"), @@ -656,7 +769,7 @@ unclosed table, expected `]` let sub_project = ProjectMetadata::discover(&root.join("packages/a"), &system)?; - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(sub_project, @r#" ProjectMetadata( name: Name("nested-project"), @@ -699,7 +812,7 @@ unclosed table, expected `]` let root = ProjectMetadata::discover(&root.join("packages/a"), &system)?; - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(root, @r#" ProjectMetadata( name: Name("project-root"), @@ -751,7 +864,7 @@ unclosed table, expected `]` let root = ProjectMetadata::discover(&root, &system)?; - with_escaped_paths(|| { + with_sanitized_paths(|| { assert_ron_snapshot!(root, @r#" ProjectMetadata( name: Name("super-app"), @@ -770,6 +883,266 @@ unclosed table, expected `]` Ok(()) } + + #[test] + fn discover_all_nested_projects_with_ty_sections() -> anyhow::Result<()> { + let system = TestSystem::default(); + let root = SystemPathBuf::from("/app"); + + system + .memory_file_system() + .write_files_all([ + ( + root.join("pyproject.toml"), + r#" + [project] + name = "project-root" + + [tool.ty.src] + root = "src" + "#, + ), + ( + root.join("packages/a/pyproject.toml"), + r#" + [project] + name = "nested-project" + + [tool.ty.src] + root = "src" + "#, + ), + ]) + .context("Failed to write files")?; + + let projects = ProjectMetadata::discover_all(&root, &system)?; + + with_sanitized_paths(|| { + assert_ron_snapshot!(projects, @r###" + { + "/app": ProjectMetadata( + name: Name("project-root"), + root: "/app", + options: Options( + src: Some(SrcOptions( + root: Some("src"), + )), + ), + ), + "/app/packages/a": ProjectMetadata( + name: Name("nested-project"), + root: "/app/packages/a", + options: Options( + src: Some(SrcOptions( + root: Some("src"), + )), + ), + ), + } + "###); + }); + + Ok(()) + } + + #[test] + fn discover_all_nested_project_without_ty_section() -> anyhow::Result<()> { + let system = TestSystem::default(); + let root = SystemPathBuf::from("/app"); + + system + .memory_file_system() + .write_files_all([ + ( + root.join("pyproject.toml"), + r#" + [project] + name = "project-root" + + [tool.ty.src] + root = "src" + "#, + ), + ( + root.join("packages/a/pyproject.toml"), + r#" + [project] + name = "nested-project" + "#, + ), + ]) + .context("Failed to write files")?; + + let projects = ProjectMetadata::discover_all(&root, &system)?; + + with_sanitized_paths(|| { + assert_ron_snapshot!(projects, @r###" + { + "/app": ProjectMetadata( + name: Name("project-root"), + root: "/app", + options: Options( + src: Some(SrcOptions( + root: Some("src"), + )), + ), + ), + } + "###); + }); + + Ok(()) + } + + #[test] + fn discover_all_nested_projects_with_ty_toml() -> anyhow::Result<()> { + let system = TestSystem::default(); + let root = SystemPathBuf::from("/app"); + + system + .memory_file_system() + .write_files_all([ + ( + root.join("pyproject.toml"), + r#" + [project] + name = "project-root" + + [tool.ty.src] + root = "src" + "#, + ), + ( + root.join("packages/a/ty.toml"), + r#" + [src] + root = "src" + "#, + ), + ]) + .context("Failed to write files")?; + + let projects = ProjectMetadata::discover_all(&root, &system)?; + + with_sanitized_paths(|| { + assert_ron_snapshot!(projects, @r###" + { + "/app": ProjectMetadata( + name: Name("project-root"), + root: "/app", + options: Options( + src: Some(SrcOptions( + root: Some("src"), + )), + ), + ), + "/app/packages/a": ProjectMetadata( + name: Name("a"), + root: "/app/packages/a", + options: Options( + src: Some(SrcOptions( + root: Some("src"), + )), + ), + ), + } + "###); + }); + + Ok(()) + } + + #[test] + fn discover_all_nested_without_ty_sections() -> anyhow::Result<()> { + let system = TestSystem::default(); + let root = SystemPathBuf::from("/app"); + + system + .memory_file_system() + .write_files_all([ + ( + root.join("pyproject.toml"), + r#" + [project] + name = "project-root" + + [tool.ty] + "#, + ), + ( + root.join("packages/a/pyproject.toml"), + r#" + [project] + name = "sub-project" + "#, + ), + ]) + .context("Failed to write files")?; + + let projects = ProjectMetadata::discover_all(&root, &system)?; + + with_sanitized_paths(|| { + assert_ron_snapshot!(projects, @r###" + { + "/app": ProjectMetadata( + name: Name("project-root"), + root: "/app", + options: Options(), + ), + } + "###); + }); + + Ok(()) + } + + #[test] + fn discover_all_nested_all_projects_without_ty_sections() -> anyhow::Result<()> { + let system = TestSystem::default(); + let root = SystemPathBuf::from("/app"); + + system + .memory_file_system() + .write_files_all([ + ( + root.join("pyproject.toml"), + r#" + [project] + name = "project-root" + "#, + ), + ( + root.join("packages/a/pyproject.toml"), + r#" + [project] + name = "sub-project" + "#, + ), + ]) + .context("Failed to write files")?; + + let projects = ProjectMetadata::discover_all(&root, &system)?; + + with_sanitized_paths(|| { + assert_ron_snapshot!(projects, @r###" + { + "/app": ProjectMetadata( + name: Name("project-root"), + root: "/app", + options: Options(), + ), + "/app/packages/a": ProjectMetadata( + name: Name("sub-project"), + root: "/app/packages/a", + options: Options(), + ), + } + "###); + }); + + Ok(()) + } + #[test] fn requires_python_major_minor() -> anyhow::Result<()> { let system = TestSystem::default(); @@ -1053,7 +1426,7 @@ unclosed table, expected `]` assert_eq!(error.to_string().replace('\\', "/"), message); } - fn with_escaped_paths(f: impl FnOnce() -> R) -> R { + fn with_sanitized_paths(f: impl FnOnce() -> R) -> R { let mut settings = insta::Settings::clone_current(); settings.add_dynamic_redaction(".root", |content, _path| { content.as_str().unwrap().replace('\\', "/") diff --git a/crates/ty_server/src/server/api/diagnostics.rs b/crates/ty_server/src/server/api/diagnostics.rs index d25e6f5243..28127374e9 100644 --- a/crates/ty_server/src/server/api/diagnostics.rs +++ b/crates/ty_server/src/server/api/diagnostics.rs @@ -12,7 +12,7 @@ use rustc_hash::FxHashMap; use ruff_db::diagnostic::{Annotation, Severity, SubDiagnostic}; use ruff_db::files::{File, FileRange}; -use ruff_db::system::SystemPathBuf; +use ruff_db::system::{SystemPath, SystemPathBuf}; use serde::{Deserialize, Serialize}; use ty_project::{Db as _, ProjectDatabase}; @@ -200,7 +200,7 @@ pub(super) fn publish_diagnostics(document: &DocumentHandle, session: &Session, pub(crate) fn publish_settings_diagnostics( session: &mut Session, client: &Client, - path: SystemPathBuf, + path: &SystemPath, ) { // Don't publish settings diagnostics for workspace that are already doing full diagnostics. // @@ -212,7 +212,7 @@ pub(crate) fn publish_settings_diagnostics( } let session_encoding = session.position_encoding(); - let state = session.project_state_mut(&AnySystemPath::System(path)); + let state = session.project_state_mut(&AnySystemPath::System(path.to_path_buf())); let db = &state.db; let project = db.project(); let settings_diagnostics = project.check_settings(db); diff --git a/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs b/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs index c67bbd8e70..70bf3e63c4 100644 --- a/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs +++ b/crates/ty_server/src/server/api/notifications/did_change_watched_files.rs @@ -79,7 +79,7 @@ impl SyncNotificationHandler for DidChangeWatchedFiles { tracing::debug!("Applying changes to `{root}`"); session.apply_changes(&AnySystemPath::System(root.clone()), changes); - publish_settings_diagnostics(session, client, root); + publish_settings_diagnostics(session, client, &root); } let client_capabilities = session.client_capabilities(); diff --git a/crates/ty_server/src/session.rs b/crates/ty_server/src/session.rs index ab68d17964..8e6ab26ca0 100644 --- a/crates/ty_server/src/session.rs +++ b/crates/ty_server/src/session.rs @@ -22,6 +22,8 @@ use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{System, SystemPath, SystemPathBuf}; use ty_combine::Combine; use ty_project::metadata::Options; +use ty_project::metadata::options::{ProjectOptionsOverrides, SrcOptions}; +use ty_project::metadata::value::{RangedValue, RelativeGlobPattern}; use ty_project::watch::{ChangeEvent, CreatedKind}; use ty_project::{ChangeResult, CheckMode, Db as _, ProjectDatabase, ProjectMetadata}; @@ -494,13 +496,13 @@ impl Session { combined_global_options.combine_with(Some(global)); let workspace_settings = workspace.into_settings(); - let Some((root, workspace)) = self.workspaces.initialize(&url, workspace_settings) + let Some((workspace_root, workspace)) = + self.workspaces.initialize(&url, workspace_settings) else { continue; }; - // TODO: Discover all projects within this workspace, starting from root. - // + let workspace_overrides = workspace.settings().project_options_overrides().cloned(); // For now, create one project database per workspace. // In the future, index the workspace directories to find all projects @@ -510,61 +512,112 @@ impl Session { self.native_system.clone(), ); - let project = ProjectMetadata::discover(&root, &system) - .context("Failed to discover project configuration") - .and_then(|mut metadata| { - metadata - .apply_configuration_files(&system) - .context("Failed to apply configuration files")?; + // We probably want something similar to our current logic but that now loops over every project. - if let Some(overrides) = workspace.settings.project_options_overrides() { - metadata.apply_overrides(overrides); - } - - ProjectDatabase::new(metadata, system.clone()) - }); - - let (root, db) = match project { - Ok(db) => (root, db), + let projects = match ProjectMetadata::discover_all(&workspace_root, &system) + .context("Failed to discover projects") + { + Ok(projects) => projects, Err(err) => { - tracing::error!( - "Failed to create project for `{root}`: {err:#}. \ - Falling back to default settings" - ); + tracing::error!("Failed to discover projects: {}", err); client.show_error_message(format!( - "Failed to load project rooted at {root}. \ + "Failed to discover projects in {workspace_root}. \ Please refer to the logs for more details.", )); - let db_with_default_settings = - ProjectMetadata::from_options(Options::default(), root, None) - .context("Failed to convert default options to metadata") - .and_then(|metadata| ProjectDatabase::new(metadata, system)) - .expect("Default configuration to be valid"); - let default_root = db_with_default_settings - .project() - .root(&db_with_default_settings) - .to_path_buf(); - - (default_root, db_with_default_settings) + BTreeMap::from_iter([( + workspace_root.clone(), + ProjectMetadata::new( + workspace_root.file_name().unwrap_or("root").into(), + workspace_root, + ), + )]) } }; - // Carry forward diagnostic state if any exists - let previous = self.projects.remove(&root); - let untracked = previous - .map(|state| state.untracked_files_with_pushed_diagnostics) - .unwrap_or_default(); - self.projects.insert( - root.clone(), - ProjectState { - db, - untracked_files_with_pushed_diagnostics: untracked, - }, - ); + for (project_root, metadata) in &projects { + let mut metadata = metadata.clone(); + if let Err(err) = metadata.apply_configuration_files(&system) { + tracing::error!( + "Failed to apply configuration files for `{project_root}`: {err:#}." + ); - publish_settings_diagnostics(self, client, root); + client.show_error_message(format!( + "Failed to apply configuration files for `{project_root}` \ + Please refer to the logs for more details.", + )); + }; + + // TODO: Request python interpreter path per project. + if let Some(overrides) = workspace_overrides.as_ref() { + metadata.apply_overrides(overrides); + } + + // Only add outer most escape + for (nested, _) in projects + .range(project_root.to_path_buf()..) + .take_while(|(path, _)| path.starts_with(project_root)) + { + let exclude = Some(RangedValue::cli(vec![ + RelativeGlobPattern::cli(format!("{nested}/**/*")), + RelativeGlobPattern::cli("**/*.ipynb"), + ])); + + tracing::debug!("Adding exclude pattern: {:?}", exclude); + metadata.apply_overrides(&ProjectOptionsOverrides { + config_file_override: None, + fallback_python_version: None, + fallback_python: None, + options: Options { + src: Some(SrcOptions { + exclude, + ..SrcOptions::default() + }), + ..Options::default() + }, + }); + } + + let db = match ProjectDatabase::new(metadata, system.clone()) { + Ok(db) => db, + Err(err) => { + tracing::error!( + "Failed to create project for `{project_root}`: {err:#}. \ + Falling back to default settings" + ); + + client.show_error_message(format!( + "Failed to load project rooted at {project_root}. \ + Please refer to the logs for more details.", + )); + + ProjectMetadata::from_options( + Options::default(), + project_root.clone(), + None, + ) + .context("Failed to convert default options to metadata") + .and_then(|metadata| ProjectDatabase::new(metadata, system.clone())) + .expect("Default configuration to be valid") + } + }; + + // Carry forward diagnostic state if any exists + let previous = self.projects.remove(project_root); + let untracked = previous + .map(|state| state.untracked_files_with_pushed_diagnostics) + .unwrap_or_default(); + self.projects.insert( + project_root.clone(), + ProjectState { + db, + untracked_files_with_pushed_diagnostics: untracked, + }, + ); + + publish_settings_diagnostics(self, client, project_root); + } } if let Some(global_options) = combined_global_options.take() {