From 8eb9bd0f890bf296450d7e25d902f237c0472d26 Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Wed, 23 Apr 2025 17:09:37 -0400
Subject: [PATCH 01/10] create context attribute

---
 paramtools/schema.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index ebbb453..e2081b5 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -192,6 +192,9 @@ class Meta:
         "when": "_get_when_validator",
     }
 
+    def __init__(self):
+        self.context = {}
+
     def validate_only(self, data):
         """
         Bypass deserialization and just run field validators. This is taken

From fd01b74fcadc425b444485875895deff76375cff Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Wed, 23 Apr 2025 17:17:08 -0400
Subject: [PATCH 02/10] remove ordered

---
 paramtools/schema.py                | 31 ++++++++---------------------
 paramtools/schema_factory.py        |  9 +++------
 paramtools/tests/test_parameters.py |  9 ---------
 3 files changed, 11 insertions(+), 38 deletions(-)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index e2081b5..9259f3c 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -28,14 +28,14 @@ class RangeSchema(Schema):
     }
     """
 
-    _min = fields.Field(attribute="min", data_key="min")
-    _max = fields.Field(attribute="max", data_key="max")
-    step = fields.Field()
+    _min = fields.Raw(attribute="min", data_key="min")
+    _max = fields.Raw(attribute="max", data_key="max")
+    step = fields.Raw()
     level = fields.String(validate=[validate.OneOf(["warn", "error"])])
 
 
 class ChoiceSchema(Schema):
-    choices = fields.List(fields.Field)
+    choices = fields.List(fields.Raw)
     level = fields.String(validate=[validate.OneOf(["warn", "error"])])
 
 
@@ -53,9 +53,9 @@ class ValueValidatorSchema(Schema):
 
 
 class IsSchema(Schema):
-    equal_to = fields.Field(required=False)
-    greater_than = fields.Field(required=False)
-    less_than = fields.Field(required=False)
+    equal_to = fields.Raw(required=False)
+    greater_than = fields.Raw(required=False)
+    less_than = fields.Raw(required=False)
 
     @validates_schema
     def just_one(self, data, **kwargs):
@@ -107,15 +107,12 @@ class BaseParamSchema(Schema):
         data_key="type",
     )
     number_dims = fields.Integer(required=False, load_default=0)
-    value = fields.Field(required=True)  # will be specified later
+    value = fields.Raw(required=True)  # will be specified later
     validators = fields.Nested(
         ValueValidatorSchema(), required=False, load_default={}
     )
     indexed = fields.Boolean(required=False)
 
-    class Meta:
-        ordered = True
-
 
 class EmptySchema(Schema):
     """
@@ -126,15 +123,6 @@ class EmptySchema(Schema):
     pass
 
 
-class OrderedSchema(Schema):
-    """
-    Same as `EmptySchema`, but preserves the order of its fields.
-    """
-
-    class Meta:
-        ordered = True
-
-
 class ValueObject(fields.Nested):
     """
     Schema for value objects
@@ -182,9 +170,6 @@ class BaseValidatorSchema(Schema):
     class.
     """
 
-    class Meta:
-        ordered = True
-
     WRAPPER_MAP = {
         "range": "_get_range_validator",
         "date_range": "_get_range_validator",
diff --git a/paramtools/schema_factory.py b/paramtools/schema_factory.py
index b841169..dc61dbb 100644
--- a/paramtools/schema_factory.py
+++ b/paramtools/schema_factory.py
@@ -1,7 +1,6 @@
-from marshmallow import fields
+from marshmallow import fields, Schema
 
 from paramtools.schema import (
-    OrderedSchema,
     BaseValidatorSchema,
     ValueObject,
     get_type,
@@ -67,9 +66,7 @@ def schemas(self):
             # if not isinstance(v["value"], list):
             #     v["value"] = [{"value": v["value"]}]
 
-            validator_dict[k] = type(
-                "ValidatorItem", (OrderedSchema,), classattrs
-            )
+            validator_dict[k] = type("ValidatorItem", (Schema,), classattrs)
 
             classattrs = {"value": ValueObject(validator_dict[k], many=True)}
             param_dict[k] = type(
@@ -77,7 +74,7 @@ def schemas(self):
             )
 
         classattrs = {k: fields.Nested(v) for k, v in param_dict.items()}
-        DefaultsSchema = type("DefaultsSchema", (OrderedSchema,), classattrs)
+        DefaultsSchema = type("DefaultsSchema", (Schema,), classattrs)
         defaults_schema = DefaultsSchema()
 
         classattrs = {
diff --git a/paramtools/tests/test_parameters.py b/paramtools/tests/test_parameters.py
index 1c5a6ec..d84ef04 100644
--- a/paramtools/tests/test_parameters.py
+++ b/paramtools/tests/test_parameters.py
@@ -166,7 +166,6 @@ def get_defaults(self):
         assert params.hello_world == "hello world"
         assert params.label_grid == {"somelabel": [0, 1, 2, 3, 4, 5]}
 
-
     def test_schema_not_dropped(self, defaults_spec_path):
         with open(defaults_spec_path, "r") as f:
             defaults_ = json.loads(f.read())
@@ -379,14 +378,6 @@ def test_specification(self, TestParams, defaults_spec_path):
 
         assert spec1["min_int_param"] == exp["min_int_param"]["value"]
 
-    def test_is_ordered(self, TestParams):
-        params = TestParams()
-        spec1 = params.specification()
-        assert isinstance(spec1, OrderedDict)
-
-        spec2 = params.specification(meta_data=True, serializable=True)
-        assert isinstance(spec2, OrderedDict)
-
     def test_specification_query(self, TestParams):
         params = TestParams()
         spec1 = params.specification()

From 0fa55f50669c5ee584c1e0de17564dd5cfbdf70e Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Wed, 23 Apr 2025 17:31:27 -0400
Subject: [PATCH 03/10] create custom fields

---
 paramtools/schema.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index 9259f3c..7707179 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -179,6 +179,7 @@ class BaseValidatorSchema(Schema):
 
     def __init__(self):
         self.context = {}
+        self.fields = {}
 
     def validate_only(self, data):
         """

From 4451118be6957b7743ca361527bac80c8a516d14 Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Thu, 24 Apr 2025 08:37:21 -0400
Subject: [PATCH 04/10] another try at setting fields

---
 paramtools/schema.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index 7707179..f67b8f3 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -177,9 +177,8 @@ class BaseValidatorSchema(Schema):
         "when": "_get_when_validator",
     }
 
-    def __init__(self):
+    def __init__(self, data):
         self.context = {}
-        self.fields = {}
 
     def validate_only(self, data):
         """
@@ -187,6 +186,7 @@ def validate_only(self, data):
         from the marshmallow _do_load function:
         https://github.com/marshmallow-code/marshmallow/blob/3.5.2/src/marshmallow/schema.py#L807
         """
+        self.fields = data.keys()
         error_store = ErrorStore()
         # Run field-level validation
         self._invoke_field_validators(

From bbbf9fc83273f477323b9aeb7eeacc46ecb43d5f Mon Sep 17 00:00:00 2001
From: hdoupe <henrymdoupe@gmail.com>
Date: Sun, 11 May 2025 06:40:44 -0400
Subject: [PATCH 05/10] Rename context to pt_context and fix parent class logic

---
 paramtools/parameters.py |  8 ++++----
 paramtools/schema.py     | 33 +++++++++++++++++----------------
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/paramtools/parameters.py b/paramtools/parameters.py
index fe3e908..0830049 100644
--- a/paramtools/parameters.py
+++ b/paramtools/parameters.py
@@ -101,7 +101,7 @@ def __init__(
             else:
                 self._stateless_label_grid[name] = []
         self.label_grid = copy.deepcopy(self._stateless_label_grid)
-        self._validator_schema.context["spec"] = self
+        self._validator_schema.pt_context["spec"] = self
         self._warnings = {}
         self._errors = {}
         self._defer_validation = False
@@ -364,7 +364,7 @@ def _adjust(
                 for param, value in parsed_params.items():
                     self._update_param(param, value)
 
-        self._validator_schema.context["spec"] = self
+        self._validator_schema.pt_context["spec"] = self
 
         has_errors = bool(self._errors.get("messages"))
         has_warnings = bool(self._warnings.get("messages"))
@@ -525,7 +525,7 @@ def _delete(
         if self.label_to_extend is not None and extend_adj:
             self.extend()
 
-        self._validator_schema.context["spec"] = self
+        self._validator_schema.pt_context["spec"] = self
 
         has_errors = bool(self._errors.get("messages"))
         has_warnings = bool(self._warnings.get("messages"))
@@ -1414,4 +1414,4 @@ def get_defaults(self):
           - `params`: String if URL or file path. Dict if this is the loaded params
             dict.
         """
-        return utils.read_json(self.defaults)
\ No newline at end of file
+        return utils.read_json(self.defaults)
diff --git a/paramtools/schema.py b/paramtools/schema.py
index f67b8f3..b4c5122 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -177,8 +177,9 @@ class BaseValidatorSchema(Schema):
         "when": "_get_when_validator",
     }
 
-    def __init__(self, data):
-        self.context = {}
+    def __init__(self, *args, **kwargs):
+        self.pt_context = {}
+        super().__init__(*args, **kwargs)
 
     def validate_only(self, data):
         """
@@ -186,7 +187,7 @@ def validate_only(self, data):
         from the marshmallow _do_load function:
         https://github.com/marshmallow-code/marshmallow/blob/3.5.2/src/marshmallow/schema.py#L807
         """
-        self.fields = data.keys()
+        # self.fields = data.keys()
         error_store = ErrorStore()
         # Run field-level validation
         self._invoke_field_validators(
@@ -260,7 +261,7 @@ def validate_param(self, param_name, param_spec, raw_data):
         Do range validation for a parameter.
         """
         validate_schema = not getattr(
-            self.context["spec"], "_defer_validation", False
+            self.pt_context["spec"], "_defer_validation", False
         )
         validators = self.validators(
             param_name, param_spec, raw_data, validate_schema=validate_schema
@@ -279,7 +280,7 @@ def validate_param(self, param_name, param_spec, raw_data):
         return warnings, errors
 
     def field_keyfunc(self, param_name):
-        data = self.context["spec"]._data[param_name]
+        data = self.pt_context["spec"]._data[param_name]
         field = get_type(data, self.validators(param_name))
         try:
             return field.cmp_funcs()["key"]
@@ -287,7 +288,7 @@ def field_keyfunc(self, param_name):
             return None
 
     def field(self, param_name):
-        data = self.context["spec"]._data[param_name]
+        data = self.pt_context["spec"]._data[param_name]
         return get_type(data, self.validators(param_name))
 
     def validators(
@@ -298,7 +299,7 @@ def validators(
         if raw_data is None:
             raw_data = {}
 
-        param_info = self.context["spec"]._data[param_name]
+        param_info = self.pt_context["spec"]._data[param_name]
         # sort keys to guarantee order.
         validator_spec = param_info.get("validators", {})
         validators = []
@@ -336,7 +337,7 @@ def _get_when_validator(
         when_param = when_dict["param"]
 
         if (
-            when_param not in self.context["spec"]._data.keys()
+            when_param not in self.pt_context["spec"]._data.keys()
             and when_param != "default"
         ):
             raise MarshmallowValidationError(
@@ -371,8 +372,8 @@ def _get_when_validator(
                 )
             )
 
-        _type = self.context["spec"]._data[oth_param]["type"]
-        number_dims = self.context["spec"]._data[oth_param]["number_dims"]
+        _type = self.pt_context["spec"]._data[oth_param]["type"]
+        number_dims = self.pt_context["spec"]._data[oth_param]["number_dims"]
 
         error_then = (
             f"When {oth_param}{{when_labels}}{{ix}} is {{is_val}}, "
@@ -458,9 +459,9 @@ def _get_range_validator(
         )
 
     def _sort_by_label_to_extend(self, vos):
-        label_to_extend = self.context["spec"].label_to_extend
+        label_to_extend = self.pt_context["spec"].label_to_extend
         if label_to_extend is not None:
-            label_grid = self.context["spec"]._stateless_label_grid
+            label_grid = self.pt_context["spec"]._stateless_label_grid
             extend_vals = label_grid[label_to_extend]
             return sorted(
                 vos,
@@ -522,9 +523,9 @@ def _get_related_value(
             # If comparing against the "default" value then get the current
             # value of the parameter being updated.
             if oth_param_name == "default":
-                oth_param = self.context["spec"]._data[param_name]
+                oth_param = self.pt_context["spec"]._data[param_name]
             else:
-                oth_param = self.context["spec"]._data[oth_param_name]
+                oth_param = self.pt_context["spec"]._data[oth_param_name]
             vals = oth_param["value"]
         labs_to_check = {k for k in param_spec if k not in ("value", "_auto")}
         if labs_to_check:
@@ -549,11 +550,11 @@ def _check_ndim_restriction(
                 if other_param is None:
                     continue
                 if other_param == "default":
-                    ndims = self.context["spec"]._data[param_name][
+                    ndims = self.pt_context["spec"]._data[param_name][
                         "number_dims"
                     ]
                 else:
-                    ndims = self.context["spec"]._data[other_param][
+                    ndims = self.pt_context["spec"]._data[other_param][
                         "number_dims"
                     ]
                 if ndims > 0:

From e2fc2da53d59493870adbf0349d332412364a56c Mon Sep 17 00:00:00 2001
From: hdoupe <henrymdoupe@gmail.com>
Date: Sun, 11 May 2025 06:41:32 -0400
Subject: [PATCH 06/10] pass_many --> pass_collection per
 https://github.com/marshmallow-code/marshmallow/commit/cf9c3a86f07de69d921214fab3dc4e29bff459e8

---
 paramtools/schema.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index b4c5122..dcd5442 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -198,7 +198,7 @@ def validate_only(self, data):
             field_errors = bool(error_store.errors)
             self._invoke_schema_validators(
                 error_store=error_store,
-                pass_many=True,
+                pass_collection=True,
                 data=data,
                 original_data=data,
                 many=None,
@@ -207,7 +207,7 @@ def validate_only(self, data):
             )
             self._invoke_schema_validators(
                 error_store=error_store,
-                pass_many=False,
+                pass_collection=False,
                 data=data,
                 original_data=data,
                 many=None,

From 1ed480a6d8d7a1a612cebf4262aa32903b8f6a7c Mon Sep 17 00:00:00 2001
From: hdoupe <henrymdoupe@gmail.com>
Date: Sun, 11 May 2025 06:43:37 -0400
Subject: [PATCH 07/10] Pass default value for uknown logic

---
 paramtools/schema.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index dcd5442..4d6c3f7 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -7,6 +7,7 @@
     validates_schema,
     ValidationError as MarshmallowValidationError,
     decorators,
+    RAISE as RAISEUNKNOWNOPTION,
 )
 from marshmallow.error_store import ErrorStore
 
@@ -204,6 +205,7 @@ def validate_only(self, data):
                 many=None,
                 partial=None,
                 field_errors=field_errors,
+                unknown=RAISEUNKNOWNOPTION,
             )
             self._invoke_schema_validators(
                 error_store=error_store,
@@ -213,6 +215,7 @@ def validate_only(self, data):
                 many=None,
                 partial=None,
                 field_errors=field_errors,
+                unknown=RAISEUNKNOWNOPTION,
             )
         errors = error_store.errors
         if errors:

From daafc2390e3b85a65063ea74f3f77e38788161bb Mon Sep 17 00:00:00 2001
From: hdoupe <henrymdoupe@gmail.com>
Date: Sun, 11 May 2025 06:47:32 -0400
Subject: [PATCH 08/10] Require marshmallow > 4

---
 conda.recipe/meta.yaml | 4 ++--
 environment.yml        | 2 +-
 setup.py               | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index a69167a..88a554f 100755
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -5,13 +5,13 @@ package:
 requirements:
   build:
     - python
-    - "marshmallow>=3.0.0rc4"
+    - "marshmallow>=4.0.0"
     - "numpy>=1.13"
     - "python-dateutil>=2.8.0"
 
   run:
     - python
-    - "marshmallow>=3.0.0rc4"
+    - "marshmallow>=4.0.0"
     - "numpy>=1.13"
     - "python-dateutil>=2.8.0"
 
diff --git a/environment.yml b/environment.yml
index 7903a13..5c09981 100644
--- a/environment.yml
+++ b/environment.yml
@@ -2,7 +2,7 @@ name: paramtools-dev
 channels:
   - conda-forge
 dependencies:
-  - "marshmallow>=3.22.0"
+  - "marshmallow>=4.0.0"
   - "numpy>=2.1.0"
   - "python-dateutil>=2.8.0"
   - "pytest>=6.0.0"
diff --git a/setup.py b/setup.py
index 4cbb390..468a191 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@
     url="https://github.com/hdoupe/ParamTools",
     packages=setuptools.find_packages(),
     install_requires=[
-        "marshmallow>=3.0.0",
+        "marshmallow>=4.0.0",
         "numpy",
         "python-dateutil>=2.8.0",
         "fsspec",

From 95a6756e66e6fea2deeaa1a5b97bab9a823a047e Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Tue, 13 May 2025 10:00:54 -0400
Subject: [PATCH 09/10] remove uncommented line

---
 paramtools/schema.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/paramtools/schema.py b/paramtools/schema.py
index 4d6c3f7..2df76ed 100644
--- a/paramtools/schema.py
+++ b/paramtools/schema.py
@@ -188,7 +188,6 @@ def validate_only(self, data):
         from the marshmallow _do_load function:
         https://github.com/marshmallow-code/marshmallow/blob/3.5.2/src/marshmallow/schema.py#L807
         """
-        # self.fields = data.keys()
         error_store = ErrorStore()
         # Run field-level validation
         self._invoke_field_validators(

From 58e2ad9800b5c2af43be1343409e0f551148bf22 Mon Sep 17 00:00:00 2001
From: Jason DeBacker <jdebacker@users.noreply.github.com>
Date: Tue, 13 May 2025 10:55:36 -0400
Subject: [PATCH 10/10] bump version

---
 paramtools/__init__.py | 2 +-
 setup.py               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/paramtools/__init__.py b/paramtools/__init__.py
index 292fa9f..283acdc 100644
--- a/paramtools/__init__.py
+++ b/paramtools/__init__.py
@@ -53,7 +53,7 @@
 
 
 name = "paramtools"
-__version__ = "0.19.0"
+__version__ = "0.20.0"
 
 __all__ = [
     "SchemaFactory",
diff --git a/setup.py b/setup.py
index 468a191..e2b0abb 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
 
 setuptools.setup(
     name="paramtools",
-    version=os.environ.get("VERSION", "0.19.0"),
+    version=os.environ.get("VERSION", "0.20.0"),
     author="Hank Doupe",
     author_email="henrymdoupe@gmail.com",
     description=(