Embedded stubs track upstream master

AJenbo · AJenbo · commit fc4f37e68d4e · 2026-03-18T13:10:49.000+01:00
diff --git a/build.rs b/build.rs
@@ -42,7 +42,16 @@ use flate2::read::GzDecoder;
 use serde::Deserialize;
 use tar::Archive;
 
-const GITHUB_REPO: &str = "JetBrains/phpstorm-stubs";
+/// Direct tarball URL for the `master` branch.
+///
+/// Unlike the releases API endpoint this does not require authentication
+/// and always points at the latest commit.  PHPStan uses the same
+/// approach (see their `update-phpstorm-stubs.yml` workflow).
+const MASTER_TARBALL_URL: &str =
+    "https://github.com/JetBrains/phpstorm-stubs/archive/refs/heads/master.tar.gz";
+
+/// GitHub API URL to query the latest commit SHA on `master`.
+const MASTER_SHA_URL: &str = "https://api.github.com/repos/JetBrains/phpstorm-stubs/commits/master";
 
 /// Relative path from the crate root to the stubs map file.
 const MAP_FILE: &str = "stubs/jetbrains/phpstorm-stubs/PhpStormStubsMap.php";
@@ -51,11 +60,10 @@ const MAP_FILE: &str = "stubs/jetbrains/phpstorm-stubs/PhpStormStubsMap.php";
 /// relative paths found in the map file).
 const STUBS_DIR: &str = "stubs/jetbrains/phpstorm-stubs";
 
-/// GitHub API response for latest release.
+/// GitHub API response for a single commit (only the fields we need).
 #[derive(Deserialize)]
-struct GitHubRelease {
-    tag_name: String,
-    tarball_url: String,
+struct GitHubCommit {
+    sha: String,
 }
 
 fn main() {
@@ -229,23 +237,26 @@ fn main() {
 }
 
 fn fetch_stubs(manifest_dir: &str) -> Result<(), Box<dyn std::error::Error>> {
-    let api_url = format!(
-        "https://api.github.com/repos/{}/releases/latest",
-        GITHUB_REPO
-    );
-
-    let mut response = ureq::get(&api_url)
+    // Try to fetch the commit SHA for a meaningful version string.
+    // This is best-effort — if it fails we still download the tarball.
+    let commit_sha = match ureq::get(MASTER_SHA_URL)
         .header("User-Agent", "phpantom-lsp-build")
         .header("Accept", "application/vnd.github.v3+json")
-        .call()?;
+        .call()
+    {
+        Ok(mut resp) => {
+            let commit: GitHubCommit = resp.body_mut().read_json()?;
+            commit.sha
+        }
+        Err(_) => String::from("unknown"),
+    };
 
-    let release: GitHubRelease = response.body_mut().read_json()?;
     eprintln!(
-        "cargo:warning=Downloading phpstorm-stubs {}",
-        release.tag_name
+        "cargo:warning=Downloading phpstorm-stubs master ({})",
+        &commit_sha[..commit_sha.len().min(10)]
     );
 
-    let mut tarball_response = ureq::get(&release.tarball_url)
+    let mut tarball_response = ureq::get(MASTER_TARBALL_URL)
         .header("User-Agent", "phpantom-lsp-build")
         .call()?;
 
@@ -290,11 +301,12 @@ fn fetch_stubs(manifest_dir: &str) -> Result<(), Box<dyn std::error::Error>> {
     // Record which version was fetched so subsequent builds (that skip
     // the download because stubs/ already exists) can still read it.
     let version_file = Path::new(manifest_dir).join("stubs/.version");
-    let _ = fs::write(&version_file, &release.tag_name);
+    let version_label = format!("master@{}", &commit_sha[..commit_sha.len().min(10)]);
+    let _ = fs::write(&version_file, &version_label);
 
     eprintln!(
         "cargo:warning=Successfully downloaded phpstorm-stubs {}",
-        release.tag_name
+        version_label
     );
     Ok(())
 }
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -33,6 +33,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- **Embedded stubs track upstream master.** The bundled phpstorm-stubs are now pulled from the `master` branch instead of the latest GitHub release, matching what PHPStan does. This brings in upstream fixes and new PHP version annotations weeks or months before a formal release.
+- **Version-aware stub types.** `#[LanguageLevelTypeAware]` attributes in phpstorm-stubs are now resolved against the project's PHP version. Functions, methods, parameters, and properties whose types changed across PHP versions (e.g. `int|false` in 7.x becoming `int` in 8.0) now show the correct type for your version. This eliminates false-positive diagnostics and incorrect completions from stale type annotations.
+
 - **Pull diagnostics.** Diagnostics are now delivered via the LSP 3.17 pull model (`textDocument/diagnostic`) when the editor supports it. The editor requests diagnostics only for visible files, and cross-file invalidation uses `workspace/diagnostic/refresh` instead of recomputing every open tab. Clients without pull support fall back to the previous push model automatically.
 - **Class name completion ranking.** Completions now rank by match quality first (exact match, then starts-with, then substring), so typing `Order` puts `Order` above `OrderLine` above `CheckOrderFlowJob` regardless of where the class comes from. Within each match quality group, use-imported and same-namespace classes appear first, followed by everything else sorted by namespace affinity (classes from heavily-imported namespaces rank higher).
 - **Use-import completion.** Same-namespace classes no longer appear in `use` statement completions (PHP auto-resolves them without an import). Classes that are already imported are filtered out. Namespace affinity still ranks the remaining candidates.
diff --git a/docs/todo.md b/docs/todo.md
@@ -19,12 +19,6 @@ within the same impact tier.
 
 # Scheduled Sprints
 
-## Sprint 3 — Diagnostics quality (finishing touches)
-
-| #   | Item                                                                                                                                                           | Impact | Effort |
-| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------ |
-| E0  | [Switch embedded stubs to master + LanguageLevelTypeAware](todo/external-stubs.md#e0-switch-embedded-stubs-to-master-and-apply-languageleveltypeaware-patches) | High   | Low    |
-
 ## Sprint 4 — Refactoring toolkit
 
 | #   | Item                                                                               | Impact | Effort |
diff --git a/docs/todo/external-stubs.md b/docs/todo/external-stubs.md
@@ -16,81 +16,6 @@ within the same impact tier.
 
 ---
 
-## E0. Switch embedded stubs to `master` and apply `LanguageLevelTypeAware` patches
-
-**Impact: High · Effort: Low**
-
-The embedded phpstorm-stubs are pinned to the latest GitHub **release**,
-which ships infrequently (the current release is about six months old).
-In the meantime the stubs `master` branch receives ongoing fixes and new
-PHP version additions. Using a stale release causes false-positive
-diagnostics and missing completions for PHP features that were already
-corrected upstream.
-
-PHPStan takes `dev-master` of the stubs on a weekly automated schedule
-(see their `update-phpstorm-stubs.yml` workflow). We should do the same.
-
-### Part 1 — Pull `master` instead of `latest release`
-
-`build.rs` currently calls the GitHub API for the latest release and
-downloads its tarball. Change `fetch_stubs` to clone or download the
-`master` branch HEAD instead:
-
-- Replace the `releases/latest` API call with a direct tarball download
-  of `master`:
-  `https://github.com/JetBrains/phpstorm-stubs/archive/refs/heads/master.tar.gz`
-- No API call needed — the tarball URL is stable and does not require
-  authentication.
-- Record the commit SHA (from the archive's embedded git metadata or a
-  separate HEAD query) in `stubs/.version` so `STUBS_VERSION` remains
-  meaningful.
-- The rest of `build.rs` (map parsing, code generation) is unchanged.
-
-### Part 2 — Patch `LanguageLevelTypeAware` overloads
-
-phpstorm-stubs uses `#[LanguageLevelTypeAware]` to annotate functions
-and parameters whose types differ by PHP version. When two overloads of
-the same function exist (one for an older PHP version, one for a newer),
-PHPantom's parser can end up with the wrong variant — or both, causing
-duplicate entries.
-
-The `PhpStormStubsElementAvailable` filter (`parser/mod.rs`) already
-handles this for version-gated elements. However `LanguageLevelTypeAware`
-on return types and parameter types is not yet processed: the attribute
-is present in the stub but PHPantom reads the raw type annotation
-(which may be the oldest-PHP fallback string) instead of selecting the
-correct variant for the target PHP version.
-
-Inspect how the attribute is used in the stubs:
-
-```
-#[LanguageLevelTypeAware(['8.0' => 'string|false'], default: 'string')]
-```
-
-The attribute's first argument is an array mapping minimum PHP version
-strings to the type annotation to use from that version onward. The
-`default` named argument is the fallback for older versions.
-
-**Implementation:** In `parser/mod.rs`, after the existing
-`PhpStormStubsElementAvailable` filter, add a pass that reads
-`LanguageLevelTypeAware` attributes on return types and parameters,
-selects the appropriate type string for the configured `php_version`,
-and substitutes it in place of the raw annotation. This mirrors the
-logic PHPStan implements in their stub-loading layer.
-
-If fully implementing the attribute parsing is complex, a pragmatic
-first step is to always select the **newest** type variant (highest
-version key) rather than version-matching. This eliminates the most
-common false positives (stubs that gained union types or `never` return
-types in recent PHP versions) without requiring per-project version
-configuration.
-
-### Prerequisites
-
-None. Both changes are isolated to `build.rs` and `parser/mod.rs`.
-
----
-
 ## Current state
 
 PHPantom embeds JetBrains phpstorm-stubs at compile time via
diff --git a/src/parser/classes.rs b/src/parser/classes.rs
@@ -1522,10 +1522,23 @@ impl Backend {
                         php_version,
                         doc_ctx,
                     );
-                    let native_return_type = method
+                    let raw_native_return_type = method
                         .return_type_hint
                         .as_ref()
                         .map(|rth| extract_hint_string(&rth.hint));
+
+                    // Check for a #[LanguageLevelTypeAware] override on the
+                    // method's return type.  When present, it replaces the
+                    // native type hint with the version-appropriate string.
+                    let native_return_type = if let Some(ctx) = doc_ctx
+                        && let Some(ver) = ctx.php_version
+                        && let Some(override_type) =
+                            super::extract_language_level_type(&method.attribute_lists, ctx, ver)
+                    {
+                        Some(override_type)
+                    } else {
+                        raw_native_return_type
+                    };
                     let is_static = method.modifiers.iter().any(|m| m.is_static());
                     let visibility = extract_visibility(method.modifiers.iter());
 
@@ -1875,6 +1888,21 @@ impl Backend {
                         Property::Hooked(h) => Some(&h.attribute_lists),
                     };
 
+                    // Apply #[LanguageLevelTypeAware] override to property types.
+                    // When present, the attribute's version-appropriate type
+                    // string replaces the native type hint.
+                    if let Some(ctx) = doc_ctx
+                        && let Some(ver) = ctx.php_version
+                        && let Some(attr_lists) = prop_attr_lists
+                        && let Some(override_type) =
+                            super::extract_language_level_type(attr_lists, ctx, ver)
+                    {
+                        for prop in &mut prop_infos {
+                            prop.type_hint = Some(override_type.clone());
+                            prop.native_type_hint = Some(override_type.clone());
+                        }
+                    }
+
                     // Apply PHPDoc `@var` override, `@deprecated`, `@see`, and
                     // description for each property.
                     if let Some(ctx) = doc_ctx
diff --git a/src/parser/functions.rs b/src/parser/functions.rs
@@ -48,11 +48,24 @@ impl Backend {
                         php_version,
                         doc_ctx,
                     );
-                    let native_return_type = func
+                    let raw_native_return_type = func
                         .return_type_hint
                         .as_ref()
                         .map(|rth| extract_hint_string(&rth.hint));
 
+                    // Check for a #[LanguageLevelTypeAware] override on the
+                    // function's return type.  When present, it replaces the
+                    // native type hint with the version-appropriate string.
+                    let native_return_type = if let Some(ctx) = doc_ctx
+                        && let Some(ver) = ctx.php_version
+                        && let Some(override_type) =
+                            super::extract_language_level_type(&func.attribute_lists, ctx, ver)
+                    {
+                        Some(override_type)
+                    } else {
+                        raw_native_return_type
+                    };
+
                     // Apply PHPDoc `@return` override for the function.
                     // Also extract PHPStan conditional return types,
                     // type assertion annotations, and `@deprecated` if present.
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
diff --git a/src/types.rs b/src/types.rs
diff --git a/tests/php_version.rs b/tests/php_version.rs
