From e59432fb401dad33313b7a769b9740e8574b2851 Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 11 Dec 2025 14:16:45 -0500 Subject: [PATCH] [ty] Add evaluation tasks for a few cases I went through https://github.com/astral-sh/ty/issues/1274 and tried to extract what I could into eval tasks. Some of the suggestions from that issue have already been done, but most haven't. This captures the status quo. --- .../completion-evaluation-tasks.csv | 7 +++++++ .../truth/exact-over-fuzzy/completion.toml | 2 ++ .../truth/exact-over-fuzzy/main.py | 3 +++ .../truth/exact-over-fuzzy/pyproject.toml | 5 +++++ .../truth/exact-over-fuzzy/uv.lock | 8 ++++++++ .../truth/local-over-auto-import/completion.toml | 2 ++ .../truth/local-over-auto-import/main.py | 11 +++++++++++ .../truth/local-over-auto-import/pyproject.toml | 5 +++++ .../truth/local-over-auto-import/uv.lock | 8 ++++++++ .../truth/tighter-over-looser-scope/completion.toml | 2 ++ .../truth/tighter-over-looser-scope/main.py | 10 ++++++++++ .../truth/tighter-over-looser-scope/pyproject.toml | 5 +++++ .../truth/tighter-over-looser-scope/uv.lock | 8 ++++++++ .../truth/typing-gets-priority/completion.toml | 2 ++ .../truth/typing-gets-priority/main.py | 12 ++++++++++++ .../truth/typing-gets-priority/pyproject.toml | 5 +++++ .../truth/typing-gets-priority/uv.lock | 8 ++++++++ 17 files changed, 103 insertions(+) create mode 100644 crates/ty_completion_eval/truth/exact-over-fuzzy/completion.toml create mode 100644 crates/ty_completion_eval/truth/exact-over-fuzzy/main.py create mode 100644 crates/ty_completion_eval/truth/exact-over-fuzzy/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/exact-over-fuzzy/uv.lock create mode 100644 crates/ty_completion_eval/truth/local-over-auto-import/completion.toml create mode 100644 crates/ty_completion_eval/truth/local-over-auto-import/main.py create mode 100644 crates/ty_completion_eval/truth/local-over-auto-import/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/local-over-auto-import/uv.lock create mode 100644 crates/ty_completion_eval/truth/tighter-over-looser-scope/completion.toml create mode 100644 crates/ty_completion_eval/truth/tighter-over-looser-scope/main.py create mode 100644 crates/ty_completion_eval/truth/tighter-over-looser-scope/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/tighter-over-looser-scope/uv.lock create mode 100644 crates/ty_completion_eval/truth/typing-gets-priority/completion.toml create mode 100644 crates/ty_completion_eval/truth/typing-gets-priority/main.py create mode 100644 crates/ty_completion_eval/truth/typing-gets-priority/pyproject.toml create mode 100644 crates/ty_completion_eval/truth/typing-gets-priority/uv.lock diff --git a/crates/ty_completion_eval/completion-evaluation-tasks.csv b/crates/ty_completion_eval/completion-evaluation-tasks.csv index ed036b44f6..c546e6a7c0 100644 --- a/crates/ty_completion_eval/completion-evaluation-tasks.csv +++ b/crates/ty_completion_eval/completion-evaluation-tasks.csv @@ -3,6 +3,7 @@ auto-import-includes-modules,main.py,0,1 auto-import-includes-modules,main.py,1,7 auto-import-includes-modules,main.py,2,1 auto-import-skips-current-module,main.py,0,1 +exact-over-fuzzy,main.py,0,2 fstring-completions,main.py,0,1 higher-level-symbols-preferred,main.py,0, higher-level-symbols-preferred,main.py,1,1 @@ -15,6 +16,7 @@ import-deprioritizes-type_check_only,main.py,3,2 import-deprioritizes-type_check_only,main.py,4,3 import-keyword-completion,main.py,0,1 internal-typeshed-hidden,main.py,0,2 +local-over-auto-import,main.py,0,2 none-completion,main.py,0,2 numpy-array,main.py,0,159 numpy-array,main.py,1,1 @@ -25,7 +27,12 @@ raise-uses-base-exception,main.py,0,1 scope-existing-over-new-import,main.py,0,1 scope-prioritize-closer,main.py,0,2 scope-simple-long-identifier,main.py,0,1 +tighter-over-looser-scope,main.py,0,3 tstring-completions,main.py,0,1 ty-extensions-lower-stdlib,main.py,0,9 type-var-typing-over-ast,main.py,0,3 type-var-typing-over-ast,main.py,1,251 +typing-gets-priority,main.py,0,30 +typing-gets-priority,main.py,1,3 +typing-gets-priority,main.py,2,3 +typing-gets-priority,main.py,3,95 diff --git a/crates/ty_completion_eval/truth/exact-over-fuzzy/completion.toml b/crates/ty_completion_eval/truth/exact-over-fuzzy/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/exact-over-fuzzy/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/exact-over-fuzzy/main.py b/crates/ty_completion_eval/truth/exact-over-fuzzy/main.py new file mode 100644 index 0000000000..d4eea9a01e --- /dev/null +++ b/crates/ty_completion_eval/truth/exact-over-fuzzy/main.py @@ -0,0 +1,3 @@ +pattern = 1 +pttn = 1 +pttn diff --git a/crates/ty_completion_eval/truth/exact-over-fuzzy/pyproject.toml b/crates/ty_completion_eval/truth/exact-over-fuzzy/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/exact-over-fuzzy/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/exact-over-fuzzy/uv.lock b/crates/ty_completion_eval/truth/exact-over-fuzzy/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/exact-over-fuzzy/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." } diff --git a/crates/ty_completion_eval/truth/local-over-auto-import/completion.toml b/crates/ty_completion_eval/truth/local-over-auto-import/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/local-over-auto-import/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/local-over-auto-import/main.py b/crates/ty_completion_eval/truth/local-over-auto-import/main.py new file mode 100644 index 0000000000..551bcdbcb9 --- /dev/null +++ b/crates/ty_completion_eval/truth/local-over-auto-import/main.py @@ -0,0 +1,11 @@ +def foo(x): + # We specifically want the local `x` to be + # suggested first here, and NOT an `x` or an + # `X` from some other module (via auto-import). + # We'd also like this to come before `except`, + # which is a keyword that contains `x`, but is + # not an exact match (where as `x` is). `except` + # also isn't legal in this context, although + # that sort of context sensitivity is a bit + # trickier. + return x diff --git a/crates/ty_completion_eval/truth/local-over-auto-import/pyproject.toml b/crates/ty_completion_eval/truth/local-over-auto-import/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/local-over-auto-import/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/local-over-auto-import/uv.lock b/crates/ty_completion_eval/truth/local-over-auto-import/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/local-over-auto-import/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." } diff --git a/crates/ty_completion_eval/truth/tighter-over-looser-scope/completion.toml b/crates/ty_completion_eval/truth/tighter-over-looser-scope/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/tighter-over-looser-scope/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/tighter-over-looser-scope/main.py b/crates/ty_completion_eval/truth/tighter-over-looser-scope/main.py new file mode 100644 index 0000000000..13ba31f323 --- /dev/null +++ b/crates/ty_completion_eval/truth/tighter-over-looser-scope/main.py @@ -0,0 +1,10 @@ +scope1 = 1 +def x(): + scope2 = 1 + def xx(): + scope3 = 1 + def xxx(): + # We specifically want `scope3` to be + # suggested first here, since that's in + # the "tighter" scope. + scope diff --git a/crates/ty_completion_eval/truth/tighter-over-looser-scope/pyproject.toml b/crates/ty_completion_eval/truth/tighter-over-looser-scope/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/tighter-over-looser-scope/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/tighter-over-looser-scope/uv.lock b/crates/ty_completion_eval/truth/tighter-over-looser-scope/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/tighter-over-looser-scope/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." } diff --git a/crates/ty_completion_eval/truth/typing-gets-priority/completion.toml b/crates/ty_completion_eval/truth/typing-gets-priority/completion.toml new file mode 100644 index 0000000000..cbd5805f07 --- /dev/null +++ b/crates/ty_completion_eval/truth/typing-gets-priority/completion.toml @@ -0,0 +1,2 @@ +[settings] +auto-import = true diff --git a/crates/ty_completion_eval/truth/typing-gets-priority/main.py b/crates/ty_completion_eval/truth/typing-gets-priority/main.py new file mode 100644 index 0000000000..bf7fa62a93 --- /dev/null +++ b/crates/ty_completion_eval/truth/typing-gets-priority/main.py @@ -0,0 +1,12 @@ +# We should prefer `typing` over `asyncio` here. +class Foo(Protoco): ... + +# We should prefer `typing` over `ty_extensions` +# or `typing_extensions`. +reveal_ + +# We should prefer `typing` over `ast`. +TypeVa + +# We should prefer `typing` over `ctypes`. +cast diff --git a/crates/ty_completion_eval/truth/typing-gets-priority/pyproject.toml b/crates/ty_completion_eval/truth/typing-gets-priority/pyproject.toml new file mode 100644 index 0000000000..cd277d8097 --- /dev/null +++ b/crates/ty_completion_eval/truth/typing-gets-priority/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "test" +version = "0.1.0" +requires-python = ">=3.13" +dependencies = [] diff --git a/crates/ty_completion_eval/truth/typing-gets-priority/uv.lock b/crates/ty_completion_eval/truth/typing-gets-priority/uv.lock new file mode 100644 index 0000000000..a4937d10d3 --- /dev/null +++ b/crates/ty_completion_eval/truth/typing-gets-priority/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "test" +version = "0.1.0" +source = { virtual = "." }