Restore _group_ package overrides for Hydra 1.4

hydra/_internal/defaults_list.py

@@ -74,6 +74,19 @@ def __init__(self, repo: IConfigRepository, overrides_list: list[Override]) -> N
                 continue
             is_group = repo.group_exists(override.key_or_group)
             is_config = repo.config_exists(override.key_or_group)
+            key = override.get_key_element()
+            if override.is_delete():
+                key = key[1:]
+            if (
+                is_group
+                and override.package is not None
+                and override.package.partition(".")[0] == "_group_"
+            ):
+                default = GroupDefault(
+                    group=override.key_or_group, package=override.package
+                )
+                default.update_parent("", "")
+                key = default.get_override_key()
             value = override.value()
             is_dict = isinstance(override.value(), dict)
             if is_group and isinstance(value, (str, list)):
@@ -86,7 +99,6 @@ def __init__(self, repo: IConfigRepository, overrides_list: list[Override]) -> N
                 else:
                     value = value_node._value()
             if override.is_delete() and (is_group or is_config):
-                key = override.get_key_element()[1:]
                 if is_group:
                     if value is not None and not isinstance(value, str):
                         raise ValueError(
@@ -120,7 +132,6 @@ def __init__(self, repo: IConfigRepository, overrides_list: list[Override]) -> N
                     )
                 )
             else:
-                key = override.get_key_element()
                 self.override_choices[key] = value
                 self.override_metadata[key] = OverrideMetadata(external_override=True)
 

hydra/core/default_element.py

@@ -144,6 +144,13 @@ def _get_final_package(
 
         if package is None:
             package = self._relative_group_path().replace("/", ".")
+        elif self.package is not None:
+            keyword, _, suffix = package.partition(".")
+            if keyword == "_group_":
+                resolved = self.get_default_package()
+                if suffix:
+                    resolved = f"{resolved}.{suffix}" if resolved else suffix
+                package = f"_global_.{resolved}"
 
         if parent_package == "":
             ret = package

news/rc/1.4.0rc2/3535.bugfix

@@ -0,0 +1 @@
+Restore `_group_` package overrides broken in Hydra 1.4.0rc1.

news/rc/1.4.0rc2/towncrier.toml

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Contributors to Hydra
+# SPDX-License-Identifier: MIT
+[tool.towncrier]
+directory = "news/rc/1.4.0rc2"
+filename = "NEWS-RC.md"
+title_format = "{version} ({project_date})"
+template = "news/_template.rst"
+issue_format = "[#{issue}](https://github.com/hydra-ecosystem/hydra/issues/{issue})"
+ignore = ["towncrier.toml"]
+
+[[tool.towncrier.type]]
+directory = "bugfix"
+name = "Bug Fixes"
+showcontent = true

pyproject.toml

@@ -3,6 +3,7 @@
     package_dir = ""
     filename = "NEWS.md"
     directory = "news/"
+    ignore = ["rc"]
     title_format = "{version} ({project_date})"
     template = "news/_template.rst"
     issue_format = "[#{issue}](https://github.com/hydra-ecosystem/hydra/issues/{issue})"

tests/defaults_list/test_defaults_list.py

@@ -362,21 +362,21 @@ def test_get_default_package(default: InputDefault, expected: Any) -> None:
             GroupDefault(group="foo", value="bar", package="_group_"),
             "",
             "",
-            "_group_",
+            "foo",
             id="group_default:parent_package=, package=_group_",
         ),
         param(
             ConfigDefault(path="foo/bar", package="_group_"),
             "",
             "",
-            "_group_",
+            "foo",
             id="config_default:parent_package=, package=_group_",
         ),
         param(
             GroupDefault(group="foo", value="bar", package="_group_.zoo"),
             "",
             "",
-            "_group_.zoo",
+            "foo.zoo",
             id="group_default:parent_package=, package=_group_.zoo",
         ),
         param(
@@ -386,7 +386,7 @@ def test_get_default_package(default: InputDefault, expected: Any) -> None:
             ),
             "",
             "",
-            "_group_.zoo",
+            "foo.zoo",
             id="config_default:parent_package=, package=_group_.zoo",
         ),
     ],

tests/defaults_list/test_group_package.py

@@ -0,0 +1,252 @@
+# SPDX-FileCopyrightText: Contributors to Hydra
+# SPDX-License-Identifier: MIT
+from pathlib import Path
+from typing import Any
+
+from pytest import mark, param
+
+from hydra import compose, initialize_config_dir
+from hydra.core.default_element import ConfigDefault, GroupDefault, InputDefault
+from hydra.core.plugins import Plugins
+
+Plugins.instance()
+
+
+@mark.parametrize(
+    "entry,root_defaults,overrides,expected",
+    [
+        param(
+            "/preprocess@_group_: v1",
+            "  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"version": "v1"}, "relocated": {}},
+            id="group_include",
+        ),
+        param(
+            "/preprocess/v1@_group_",
+            "  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"version": "v1"}, "relocated": {}},
+            id="config_include",
+        ),
+        param(
+            "/preprocess@_group_.suffix: v1",
+            "  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"suffix": {"version": "v1"}}, "relocated": {}},
+            id="group_suffix",
+        ),
+        param(
+            "/preprocess/v1@_group_.suffix",
+            "  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"suffix": {"version": "v1"}}, "relocated": {}},
+            id="config_suffix",
+        ),
+        param(
+            "/preprocess@my_group_target: v1",
+            "  - feature@relocated: default\n",
+            [],
+            {"relocated": {"my_group_target": {"version": "v1"}}},
+            id="literal_group_package",
+        ),
+        param(
+            "/preprocess/v1@my_group_target",
+            "  - feature@relocated: default\n",
+            [],
+            {"relocated": {"my_group_target": {"version": "v1"}}},
+            id="literal_config_package",
+        ),
+        param(
+            "/preprocess@_group_.my_group_target: v1",
+            "  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"my_group_target": {"version": "v1"}}, "relocated": {}},
+            id="literal_suffix",
+        ),
+        param(
+            "override /preprocess@_group_: v1",
+            "  - preprocess: v2\n  - feature@relocated: default\n",
+            [],
+            {"preprocess": {"version": "v1"}, "relocated": {}},
+            id="nested_override",
+        ),
+        param(
+            "override /preprocess@_group_: v1",
+            "  - preprocess: v2\n  - feature@relocated: default\n",
+            ["preprocess=v2"],
+            {"preprocess": {"version": "v2"}, "relocated": {}},
+            id="cli_selection_wins",
+        ),
+        param(
+            "override /preprocess@_group_: ${variant}",
+            "  - variant: v1\n  - preprocess: v2\n  - feature@relocated: default\n",
+            [],
+            {"variant": {}, "preprocess": {"version": "v1"}, "relocated": {}},
+            id="interpolated_override_value",
+        ),
+        param(
+            None,
+            "  - _self_\n",
+            ["+preprocess@_group_=v1"],
+            {"preprocess": {"version": "v1"}},
+            id="cli_package",
+        ),
+        param(
+            None,
+            "  - _self_\n",
+            ["+preprocess@_group_.suffix=v1"],
+            {"preprocess": {"suffix": {"version": "v1"}}},
+            id="cli_package_suffix",
+        ),
+        param(
+            None,
+            "  - _self_\n",
+            ["+preprocess@my_group_target=v1"],
+            {"my_group_target": {"version": "v1"}},
+            id="literal_cli_package",
+        ),
+        param(
+            "/preprocess@_group_: v1",
+            "  - feature@relocated: default\n",
+            ["preprocess@_group_=v2"],
+            {"preprocess": {"version": "v2"}, "relocated": {}},
+            id="cli_symbolic_key",
+        ),
+        param(
+            "/preprocess@_group_.suffix: v1",
+            "  - feature@relocated: default\n",
+            ["preprocess@_group_.suffix=v2"],
+            {"preprocess": {"suffix": {"version": "v2"}}, "relocated": {}},
+            id="cli_symbolic_suffix_key",
+        ),
+        param(
+            "/preprocess@_group_: ${variant}",
+            "  - variant: v1\n  - feature@relocated: default\n",
+            ["preprocess=v2"],
+            {"variant": {}, "preprocess": {"version": "v2"}, "relocated": {}},
+            id="cli_overrides_interpolated_selection",
+        ),
+        param(
+            "/preprocess@_group_: v1",
+            "  - feature@relocated: default\n",
+            ["~preprocess@_group_"],
+            {"relocated": {}},
+            id="cli_symbolic_delete",
+        ),
+    ],
+)
+def test_group_package_composition(
+    hydra_restore_singletons: Any,
+    tmp_path: Path,
+    recwarn: Any,
+    entry: str | None,
+    root_defaults: str,
+    overrides: list[str],
+    expected: Any,
+) -> None:
+    (tmp_path / "preprocess").mkdir()
+    (tmp_path / "feature").mkdir()
+    (tmp_path / "variant").mkdir()
+    (tmp_path / "variant/v1.yaml").write_text("{}\n", encoding="utf-8")
+    (tmp_path / "preprocess/v1.yaml").write_text("version: v1\n", encoding="utf-8")
+    (tmp_path / "preprocess/v2.yaml").write_text("version: v2\n", encoding="utf-8")
+    (tmp_path / "feature/default.yaml").write_text(
+        "{}\n" if entry is None else f"defaults:\n  - {entry}\n",
+        encoding="utf-8",
+    )
+    (tmp_path / "config.yaml").write_text(
+        f"defaults:\n{root_defaults}", encoding="utf-8"
+    )
+
+    with initialize_config_dir(config_dir=str(tmp_path)):
+        assert compose(config_name="config", overrides=overrides) == expected
+    assert len(recwarn) == 0
+
+
+@mark.parametrize(
+    "default",
+    [
+        GroupDefault(group="user_group_settings", value="v1", package="_group_"),
+        ConfigDefault(path="user_group_settings/v1", package="_group_"),
+    ],
+)
+def test_group_package_parent_resolution_is_idempotent(default: InputDefault) -> None:
+    default.update_parent("", "relocated")
+    assert default.get_final_package() == "user_group_settings"
+    default.update_parent("", "relocated")
+    assert default.get_final_package() == "user_group_settings"
+
+
+@mark.parametrize(
+    "entry,overrides,expected_group",
+    [
+        param(
+            "/group_a@_group_: ${prefix_group}/item",
+            [],
+            {"sub_group": {"x": 42}},
+            id="group",
+        ),
+        param(
+            "/group_a@_group_: ${prefix_group}/item",
+            ["prefix_group=sub_other"],
+            {"sub_other": {"x": 43}},
+            id="cli_prefix_selection",
+        ),
+        param(
+            "/group_a@_group_.suffix: ${prefix_group}/item",
+            [],
+            {"sub_group": {"suffix": {"x": 42}}},
+            id="group_suffix",
+        ),
+        param(
+            "/group_a/${prefix_group}/item@_group_",
+            [],
+            {"sub_group": {"x": 42}},
+            id="config",
+        ),
+        param(
+            "/group_a/${prefix_group}/item@_group_.suffix",
+            [],
+            {"sub_group": {"suffix": {"x": 42}}},
+            id="config_suffix",
+        ),
+    ],
+)
+def test_group_package_interpolation(
+    hydra_restore_singletons: Any,
+    tmp_path: Path,
+    entry: str,
+    overrides: list[str],
+    expected_group: Any,
+) -> None:
+    (tmp_path / "prefix_group").mkdir()
+    (tmp_path / "feature").mkdir()
+    (tmp_path / "group_a/sub_group").mkdir(parents=True)
+    (tmp_path / "group_a/sub_other").mkdir()
+    (tmp_path / "prefix_group/sub_group.yaml").write_text("{}\n", encoding="utf-8")
+    (tmp_path / "prefix_group/sub_other.yaml").write_text("{}\n", encoding="utf-8")
+    (tmp_path / "group_a/sub_group/item.yaml").write_text("x: 42\n", encoding="utf-8")
+    (tmp_path / "group_a/sub_other/item.yaml").write_text("x: 43\n", encoding="utf-8")
+    (tmp_path / "feature/default.yaml").write_text(
+        f"defaults:\n  - {entry}\n", encoding="utf-8"
+    )
+    (tmp_path / "config.yaml").write_text(
+        "defaults:\n  - prefix_group: sub_group\n  - feature@relocated: default\n",
+        encoding="utf-8",
+    )
+    with initialize_config_dir(config_dir=str(tmp_path)):
+        assert compose(config_name="config", overrides=overrides) == {
+            "prefix_group": {},
+            "group_a": expected_group,
+            "relocated": {},
+        }
+
+
+def test_group_package_suffix_with_empty_group(tmp_path: Path) -> None:
+    (tmp_path / "item.yaml").write_text("x: 42\n", encoding="utf-8")
+    (tmp_path / "config.yaml").write_text(
+        "defaults:\n  - /item@_group_.suffix\n", encoding="utf-8"
+    )
+    with initialize_config_dir(config_dir=str(tmp_path)):
+        assert compose(config_name="config") == {"suffix": {"x": 42}}

website/docs/advanced/overriding_packages.md

@@ -127,6 +127,8 @@ defaults:
 
 Without a package override, the resulting package is `config_group.server.db`.
 With the **@\_here\_** keyword, the resulting package is the same as the containing config (`config_group`).
+With **@\_group\_**, the resulting package is the absolute default package of the selected config (`server.db`), independent of the containing config's package.
+Anything following \_group\_ is appended to that package; for example, **@\_group\_.backup** becomes `server.db.backup`.
 With **@\_global\_**, the resulting package is the global package. Anything following \_global\_ is absolute; for example, **@\_global\_.foo** becomes `foo`.
 
 ### Overriding the package via the package directive

website/docs/development/release.md

@@ -294,6 +294,23 @@ trigger publishing.
 
 ## Release summaries
 
+### Release-candidate corrections
+
+Corrections to regressions introduced in a release candidate belong in that
+candidate's news collection, rather than the final release's `news/` collection.
+For Hydra 1.4.0rc2, keep these fragments under `news/rc/1.4.0rc2/` and render
+the candidate notes with:
+
+```shell
+towncrier build --dir . --draft --version 1.4.0rc2 \
+  --config news/rc/1.4.0rc2/towncrier.toml
+```
+
+Include that output in the rc2 release notes. The normal Towncrier configuration
+ignores `news/rc/`, so these corrections are excluded from the final 1.4.0 notes.
+
+### Publish workflow summary
+
 The publish workflow writes a release summary to the GitHub Actions run. The
 summary includes: