Carefully handle overwriting secrets in Fireworks account

eval_protocol/cli.py

@@ -156,6 +156,11 @@ def _configure_parser(parser: argparse.ArgumentParser) -> argparse.ArgumentParse
         metavar="",
         help="Extra flags to pass to 'docker run' when validating evaluator (quoted string, e.g. \"--env-file .env --memory=8g\")",
     )
+    rft_parser.add_argument(
+        "--env-file",
+        metavar="",
+        help="Path to .env file containing secrets to upload to Fireworks (default: .env in project root)",
+    )
 
     # The flags below are Eval Protocol CLI workflow controls (not part of the Fireworks SDK `create()` signature),
     # so they can’t be auto-generated via signature introspection and must be maintained here.

eval_protocol/cli_commands/create_rft.py

@@ -19,7 +19,7 @@
     materialize_dataset_via_builder,
 )
 from ..models import EvaluationRow
-from .upload import upload_command
+from .upload import upload_command, upload_secrets_to_fireworks
 from .utils import (
     _build_entry_point,
     _build_trimmed_dataset_id,
@@ -569,8 +569,18 @@ def _upload_and_ensure_evaluator(
     api_key: str,
     api_base: str,
     force: bool,
+    env_file: Optional[str] = None,
+    non_interactive: bool = False,
 ) -> bool:
     """Ensure the evaluator exists and is ACTIVE, uploading it if needed."""
+    # Always upload secrets before checking evaluator status, as secrets may be
+    # needed for evaluation even if the evaluator already exists.
+    upload_secrets_to_fireworks(
+        root=project_root,
+        env_file=env_file,
+        non_interactive=non_interactive,
+    )
+
     # Optional short-circuit: if evaluator already exists and not forcing, skip upload path
     if not force:
         try:
@@ -625,13 +635,13 @@ def _upload_and_ensure_evaluator(
             description=None,
             force=force,  # Pass through the --force flag
             yes=True,
-            env_file=None,  # Add the new env_file parameter
+            env_file=None,  # Secrets already handled above via upload_secrets_to_fireworks
         )
 
         if force:
             print(f"🔄 Force flag enabled - will overwrite existing evaluator '{evaluator_id}'")
 
-        rc = upload_command(upload_args)
+        rc = upload_command(upload_args, skip_secrets=True)  # Secrets already handled above
         if rc == 0:
             print(f"✓ Uploaded/ensured evaluator: {evaluator_id}")
 
@@ -811,13 +821,16 @@ def create_rft_command(args) -> int:
         return 1
 
     # 5) Ensure evaluator exists and is ACTIVE (upload + poll if needed)
+    env_file: Optional[str] = getattr(args, "env_file", None)
     if not _upload_and_ensure_evaluator(
         project_root=project_root,
         evaluator_id=evaluator_id,
         evaluator_resource_name=evaluator_resource_name,
         api_key=api_key,
         api_base=api_base,
         force=force,
+        env_file=env_file,
+        non_interactive=non_interactive,
     ):
         return 1