From 4fd82d5f35d4cd0d2202e80444387dd23a2e4caf Mon Sep 17 00:00:00 2001
From: David Peter <sharkdp@users.noreply.github.com>
Date: Wed, 8 Jan 2025 22:24:57 +0100
Subject: [PATCH] [red-knot] Property test improvements (#15358)

## Summary

- Add a workflow to run property tests on a daily basis (based on
`daily_fuzz.yaml`)
- Mark `assignable_to_is_reflexive` as flaky (related to #14899)
- Add new (failing) `intersection_assignable_to_both` test (also related
to #14899)

## Test Plan

Ran:

```bash
export QUICKCHECK_TESTS=100000
while cargo test --release -p red_knot_python_semantic -- \
  --ignored types::property_tests::stable; do :; done
```

Observed successful property_tests CI run
---
 .github/workflows/daily_property_tests.yaml   | 71 +++++++++++++++++++
 .../src/types/property_tests.rs               | 32 +++++++--
 2 files changed, 97 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/daily_property_tests.yaml

diff --git a/.github/workflows/daily_property_tests.yaml b/.github/workflows/daily_property_tests.yaml
new file mode 100644
index 0000000000..7ed7c5db58
--- /dev/null
+++ b/.github/workflows/daily_property_tests.yaml
@@ -0,0 +1,71 @@
+name: Daily property test run
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * *"
+  pull_request:
+    paths:
+      - ".github/workflows/daily_property_tests.yaml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  CARGO_TERM_COLOR: always
+  RUSTUP_MAX_RETRIES: 10
+  FORCE_COLOR: 1
+
+jobs:
+  property_tests:
+    name: Property tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    # Don't run the cron job on forks:
+    if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: "Install Rust toolchain"
+        run: rustup show
+      - name: "Install mold"
+        uses: rui314/setup-mold@v1
+      - uses: Swatinem/rust-cache@v2
+      - name: Build Red Knot
+        # A release build takes longer (2 min vs 1 min), but the property tests run much faster in release
+        # mode (1.5 min vs 14 min), so the overall time is shorter with a release build.
+        run: cargo build --locked --release --package red_knot_python_semantic --tests
+      - name: Run property tests
+        shell: bash
+        run: |
+          export QUICKCHECK_TESTS=100000
+          for _ in {1..5}; do
+            cargo test --locked --release --package red_knot_python_semantic -- --ignored types::property_tests::stable
+          done
+
+  create-issue-on-failure:
+    name: Create an issue if the daily property test run surfaced any bugs
+    runs-on: ubuntu-latest
+    needs: property_tests
+    if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && needs.property_tests.result == 'failure' }}
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.create({
+              owner: "astral-sh",
+              repo: "ruff",
+              title: `Daily property test run failed on ${new Date().toDateString()}`,
+              body: "Runs listed here: https://github.com/astral-sh/ruff/actions/workflows/daily_property_tests.yaml",
+              labels: ["bug", "red_knot", "testing"],
+            })
diff --git a/crates/red_knot_python_semantic/src/types/property_tests.rs b/crates/red_knot_python_semantic/src/types/property_tests.rs
index 93f9044909..ba739c4647 100644
--- a/crates/red_knot_python_semantic/src/types/property_tests.rs
+++ b/crates/red_knot_python_semantic/src/types/property_tests.rs
@@ -258,12 +258,6 @@ mod stable {
         forall types s, t. s.is_subtype_of(db, t) => !s.is_disjoint_from(db, t) || s.is_never()
     );
 
-    // `T` can be assigned to itself.
-    type_property_test!(
-        assignable_to_is_reflexive, db,
-        forall types t. t.is_assignable_to(db, t)
-    );
-
     // `S <: T` implies that `S` can be assigned to `T`.
     type_property_test!(
         subtype_of_implies_assignable_to, db,
@@ -321,6 +315,32 @@ mod stable {
 /// tests to the `stable` section. In the meantime, it can still be useful to run these
 /// tests (using [`types::property_tests::flaky`]), to see if there are any new obvious bugs.
 mod flaky {
+    use crate::{
+        db::tests::TestDb,
+        types::{IntersectionBuilder, Type},
+    };
+
+    // Currently fails due to https://github.com/astral-sh/ruff/issues/14899
+    // `T` can be assigned to itself.
+    type_property_test!(
+        assignable_to_is_reflexive, db,
+        forall types t. t.is_assignable_to(db, t)
+    );
+
+    // Currently fails due to https://github.com/astral-sh/ruff/issues/14899
+    // An intersection of two types should be assignable to both of them
+    fn intersection<'db>(db: &'db TestDb, s: Type<'db>, t: Type<'db>) -> Type<'db> {
+        IntersectionBuilder::new(db)
+            .add_positive(s)
+            .add_positive(t)
+            .build()
+    }
+
+    type_property_test!(
+        intersection_assignable_to_both, db,
+        forall types s, t. intersection(db, s, t).is_assignable_to(db, s) && intersection(db, s, t).is_assignable_to(db, t)
+    );
+
     // `S <: T` and `T <: S` implies that `S` is equivalent to `T`.
     type_property_test!(
         subtype_of_is_antisymmetric, db,