Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
cd303a5
feat(pypi): store only selectable wheels in the lockfile facts
solosito Nov 9, 2026
31128d4
fix(pypi): key the wheel filter by major and minor python version
solosito Nov 9, 2026
d285768
perf(pypi): memoize the wheel filter by the wheel tags
solosito Nov 11, 2026
011bb79
test(pypi): cover the wheel filter for many platforms and yanked wheels
solosito Nov 11, 2026
f0be680
test(pypi): regenerate the bzlmod_lockfile lockfile
solosito Nov 17, 2026
0d6f976
fix(pypi): keep the wheels_filtered flag in simpleapi_download results
solosito Nov 17, 2026
199c07d
fix(pypi): keep all dists of a version that no platform can select
solosito Nov 17, 2026
088ce6c
fix(pypi): leave uv_lock hubs out of the wheel filter
solosito Nov 17, 2026
8e10fb3
fix(pypi): bump the facts version to v4
solosito Nov 17, 2026
b82d92b
docs(pypi): clarify the wheel filter comments and news
solosito Nov 17, 2026
eed5e15
style(pypi): wrap the long wheel filter lines
solosito Nov 17, 2026
bcf6fc5
feat(pypi): store dist urls relative to their common directory in the…
solosito Nov 9, 2026
032dd86
fix(pypi): keep relative dist urls in the facts unambiguous
solosito Nov 11, 2026
3516533
fix(pypi): fetch the index again if a dist fact is malformed
solosito Nov 11, 2026
ed52bd5
docs(pypi): note why relative dist urls are safe to merge
solosito Nov 11, 2026
9a8534b
refactor(pypi): group the relative dist urls under their common direc…
solosito Nov 11, 2026
092dbcf
refactor(pypi): compact the dist URLs inside the facts cache
solosito Nov 17, 2026
4a27602
docs(pypi): clarify the relative dist URL comments
solosito Nov 17, 2026
78a5549
docs(pypi): note the index refetch in the relative URL news
solosito Nov 17, 2026
eab0163
tests(pypi): wrap long lines and write URL in the relative URL tests
solosito Nov 17, 2026
398e4ea
tests(pypi): cover the empty and the merged common directories
solosito Nov 17, 2026
0414b70
test(pypi): regenerate the bzlmod_lockfile lockfile
solosito Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions news/facts-filter-wheels-by-platform.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
(pypi) The `MODULE.bazel.lock` facts now store all sdists, but only the wheels
that a configured {obj}`pip.default` platform can select for one of the
requested Python versions. This makes the lockfile smaller. The first run after
the upgrade fetches the index again, so update the lockfile if you use
`--lockfile_mode=error`. After that, the extension fetches the index again only
if you change the platforms or the major or minor Python versions.
4 changes: 4 additions & 0 deletions news/facts-relative-dist-urls.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
(pypi) The `MODULE.bazel.lock` facts now store the distribution URLs of each
package relative to the longest directory that the URLs share. This makes the
lockfile smaller. The first run after the upgrade fetches the index again, so
update the lockfile if you use `--lockfile_mode=error`.
7 changes: 6 additions & 1 deletion python/private/pypi/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ bzl_library(
":unified_hub_repo",
":whl_library",
"//python/private:auth",
"//python/private:full_version",
"//python/private:normalize_name",
"//python/private:pyproject_utils",
"//python/private:repo_utils",
Expand Down Expand Up @@ -371,7 +372,11 @@ bzl_library(
bzl_library(
name = "pypi_cache",
srcs = ["pypi_cache.bzl"],
deps = [":version_from_filename"],
deps = [
":parse_whl_name",
":select_whl",
":version_from_filename",
],
)

bzl_library(
Expand Down
41 changes: 33 additions & 8 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ load("@pythons_hub//:versions.bzl", "MINOR_MAPPING")
load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
load("@toml.bzl", "toml")
load("//python/private:auth.bzl", "AUTH_ATTRS")
load("//python/private:full_version.bzl", "full_version")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python")
load("//python/private:repo_utils.bzl", "repo_utils")
load(":default_platforms.bzl", "default_platforms")
load(":hub_builder.bzl", "hub_builder")
load(":hub_builder.bzl", "hub_builder", "hub_platforms")
load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
load(":parse_whl_name.bzl", "parse_whl_name")
load(":pep508_env.bzl", "env")
Expand Down Expand Up @@ -157,6 +158,14 @@ def build_config(
unified_hub_requirements_bzl = unified_hub_requirements_bzl,
)

def _python_version(module_ctx, pip_attr, config):
python_version = pip_attr.python_version
if not python_version and pip_attr.pyproject_toml:
pyproject = read_pyproject(module_ctx, pip_attr.pyproject_toml)
if pyproject.requires_python:
python_version = version_from_requires_python(pyproject.requires_python)
return python_version or config.python_version

def parse_modules(
module_ctx,
_fail = fail,
Expand Down Expand Up @@ -245,19 +254,35 @@ You cannot use both the additive_build_content and additive_build_content_file a
# dict[str repo, HubBuilder]
# See `hub_builder.bzl%hub_builder()` for `HubBuilder`
pip_hub_map = {}
simpleapi_cache = pypi_cache(mctx = module_ctx)

# The facts store all sdists, but only the wheels that a configured platform can select for
# one of the requested Python versions. The host is not used, so the lockfile is the same on
# all hosts. A hub with a `uv_lock` does not read the index, so its Python version is not used.
wheel_platforms = {}
for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
if pip_attr.uv_lock:
continue

full_python_version = full_version(
version = _python_version(module_ctx, pip_attr, config) or "",
minor_mapping = kwargs.get("minor_mapping", MINOR_MAPPING),
fail_on_err = False,
)
if full_python_version:
wheel_platforms.update(hub_platforms(
module_ctx,
python_version = full_python_version,
config = config,
))
simpleapi_cache = pypi_cache(mctx = module_ctx, wheel_platforms = wheel_platforms.values())

is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1"
renamed_default_hub = None

for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
python_version = pip_attr.python_version
if not python_version and pip_attr.pyproject_toml:
pyproject = read_pyproject(module_ctx, pip_attr.pyproject_toml)
if pyproject.requires_python:
python_version = version_from_requires_python(pyproject.requires_python)
python_version = python_version or config.python_version
python_version = _python_version(module_ctx, pip_attr, config)
if not python_version:
_fail("pip.parse() requires one of `python_version`, `pyproject_toml`, or `pip.default(pyproject_toml=...)` to be set")

Expand Down
18 changes: 18 additions & 0 deletions python/private/pypi/hub_builder.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,24 @@ def _detect_interpreter(self, pip_attr, python_version, full_python_version):
path = pip_attr.python_interpreter,
)

def hub_platforms(module_ctx, *, python_version, config):
"""Return all of the target platforms in `config` for a Python version.

Args:
module_ctx: {type}`module_ctx` the module context.
python_version: {type}`str` the full Python version.
config: the platform configuration.

Returns:
{type}`dict[str, struct]` the target platforms, keyed by the ABI and the platform name.
"""
return _platforms(
module_ctx,
python_version = python_version,
config = config,
target_platforms = [],
)

def _platforms(module_ctx, *, python_version, config, target_platforms):
platforms = {}
python_version = version.parse(
Expand Down
5 changes: 5 additions & 0 deletions python/private/pypi/parse_requirements.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,11 @@ def _add_dists(*, requirement, index_urls, target_platform, logger = None):
sdist = maybe_sdist
continue

if getattr(index_urls, "wheels_filtered", False):
# The digest belongs to a wheel that no configured platform can select, so the
# lockfile facts left it out on purpose. Do not warn.
continue

logger.warn(lambda: "Could not find a whl or an sdist with hash {}; note that the index may be advertising digests calculated with a different hash algorithm".format(digest))

yanked = {}
Expand Down
Loading