From bcb7c0bbbb1c531e1ed70558e9e21e0e7dd46710 Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 12:58:44 -0500 Subject: [PATCH 1/9] Now *also* echo any errors to file. --- examples-proposed/024-aggregated-compute-ensemble/gen_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py index f8bd70b3..5bfc97ce 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py +++ b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py @@ -12,6 +12,7 @@ import matplotlib.pyplot as plt from ipsframework.resourceHelper import get_platform_info +from ipsframework.services import add_analysis_data_file def main(instance: str, alpha: float, L:float, T_final:float, Nx:int, Nt:int) -> dict[str, Any]: @@ -114,5 +115,6 @@ def main(instance: str, except Exception as e: print(f'Encountered error: {e}') - print(f'Encountered error: {e}', file='gen_data_error.txt') + with open('gen_data_error.txt', 'w') as f: + print(f'Encountered error: {e!s}', file=f) print_exc() From fb3aab3993f3e042c57fcb04d661b91add17fcfb Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 13:00:13 -0500 Subject: [PATCH 2/9] Let's use pathlib Path objects correctly --- .../024-aggregated-compute-ensemble/instance_component.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py index 3e1940b7..61300ef5 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py @@ -26,8 +26,8 @@ def create_cmd(instance: str, path: Path, alpha: float, L:float, T_final:float, :param Nt: number of time steps :returns: list of command line arguments to be executed in step() """ - executable = f'{path!s}/gen_data.py' - cmd = ['python3', executable, '--instance', instance, + executable = path / 'gen_data.py' + cmd = ['python3', str(executable), '--instance', instance, '--alpha', alpha, '--L', L, '--T_final', T_final, '--Nx', Nx, '--Nt', Nt] return cmd From 44d779cbf0d3cbd802ef94200b38b31a3c490c81 Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 14:28:59 -0500 Subject: [PATCH 3/9] Add Jupyter notebook for instance data visualization and update data generation process --- .../gen_data.py | 17 +-- .../instance_base_notebook.ipynb | 101 ++++++++++++++++++ .../instance_component.py | 14 +-- .../instance_driver.py | 11 ++ 4 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb diff --git a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py index 5bfc97ce..f54c0360 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py +++ b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py @@ -1,6 +1,14 @@ #!/usr/bin/env python3 """ Used to generate synthetic data as an example. + + Writes two output files: + + * `_solution.json`: contains the synthetic data + * `_stats.csv`: contains provenance information about the run + + The JSON file is, in turn, read by the a per-instance jupyter notebook + available on the Portal to generate a plot of the data. """ import argparse from typing import Any @@ -9,7 +17,6 @@ from time import time from traceback import print_exc import numpy as np -import matplotlib.pyplot as plt from ipsframework.resourceHelper import get_platform_info from ipsframework.services import add_analysis_data_file @@ -58,13 +65,7 @@ def main(instance: str, u_new[i] = u[i] + r * (u[i + 1] - 2 * u[i] + u[i - 1]) u = u_new - # Plotting the result - plt.plot(x, u) - plt.xlabel("Position (x)") - plt.ylabel("Temperature (u)") - plt.title("Solution of 1D Heat Equation") - plt.grid(True) - plt.savefig(f"{instance}_solution.png") + # Save some per-component stats stats_fname = f'{instance}_stats.csv' diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb b/examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb new file mode 100644 index 00000000..294b0dd9 --- /dev/null +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb @@ -0,0 +1,101 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "# Instance base notebook\n", + "\n", + "This notebook replicates for each ensemble instance." + ], + "id": "56c3997f94a366d5" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-12-18T18:45:35.472364Z", + "start_time": "2025-12-18T18:45:35.462785Z" + } + }, + "cell_type": "code", + "source": [ + "import json\n", + "from pathlib import Path\n", + "\n", + "%matplotlib notebook\n", + "import matplotlib.pyplot as plt" + ], + "id": "937aff95abd359fa", + "outputs": [], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-12-18T18:40:41.874866Z", + "start_time": "2025-12-18T18:40:41.864743Z" + } + }, + "cell_type": "code", + "source": [ + "# Find the JSON output from `gen_data.py` that was invoked as a sub-task. It will\n", + "# follow the naming pattern `_solution.json`. It will contain two\n", + "# arrays, `x` and `u`, which we will plot.\n", + "solution_json_file = list(Path(\".\").rglob(\"*solution.json\"))\n", + "\n", + "if len(solution_json) != 1:\n", + " raise RuntimeError(\n", + " f\"Expected exactly one solution JSON file, found {len(solution_json_file)}\"\n", + " )\n", + "solution_json_file = solution_json_file[0]\n", + "\n", + "# Now read the JSON file and extract the data.\n", + "with open(solution_json_file, \"r\") as f:\n", + " solution_data = json.load(f)\n", + "\n", + "x = solution_data[\"x\"]\n", + "u = solution_data[\"u\"]\n", + "\n", + "# Now plot the data.\n", + "plt.plot(x, u)\n", + "plt.xlabel(\"Position (x)\")\n", + "plt.ylabel(\"Temperature (u)\")\n", + "plt.title(\"Solution of 1D Heat Equation\")\n", + "plt.grid(True)\n", + "plt.savefig(f\"{instance}_solution.png\")" + ], + "id": "b1a40da330ba6b3e", + "outputs": [], + "execution_count": 1 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "66002c188c5ab473" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py index 61300ef5..341551f5 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_component.py +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_component.py @@ -71,12 +71,12 @@ def step(self, timestamp: float = 0.0, **keywords): self.services.info(f'{instance_id}: Completed MPI executable with ' f'return value: {return_value}.') - # TODO temporarily commenting this out until the actual - # example is ready to consider adding data files to the portal. This - # originally came from code Lance wrote in a previous example. - # try: - # self.services.add_analysis_data_files([data_fname, stats_fname], timestamp) - # except Exception: - # print('did not add data files to portal, check logs') + # Add the generated data JSON and CSV files to the portal + try: + self.services.add_analysis_data_files([f'{instance_id}_solution.json', + f'{instance_id}_stats.csv'], + replace=True) + except Exception: + print('did not add data files to portal, check logs') self.services.info(f'{instance_id}: End of step of instance component.') diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py b/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py index 33f1954b..8b8f76d0 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py +++ b/examples-proposed/024-aggregated-compute-ensemble/instance_driver.py @@ -5,11 +5,22 @@ from ipsframework import Component +# The notebook that will be copied for each ensemble instance +SOURCE_NOTEBOOK_NAME='instance_base_notebook.ipynb' + class InstanceDriver(Component): """ Instance driver component that steps the main component """ + def init(self, timestamp: float = 0.0, **keywords): + self.services.stage_input_files([SOURCE_NOTEBOOK_NAME]) + + self.services.initialize_jupyter_notebook( + dest_notebook_name='jupyterhub_instance_notebook.ipynb', + source_notebook_path=SOURCE_NOTEBOOK_NAME, + ) + def step(self, timestamp: float = 0.0, **keywords): instance_component = self.services.get_port('WORKER') From 098bb23289a27d13eec5069db556d2e3c839384d Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 14:36:51 -0500 Subject: [PATCH 4/9] The notebook needs to be copied over to the instance, too. --- examples-proposed/024-aggregated-compute-ensemble/template.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/template.conf b/examples-proposed/024-aggregated-compute-ensemble/template.conf index 1e699128..b74fd872 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/template.conf +++ b/examples-proposed/024-aggregated-compute-ensemble/template.conf @@ -18,7 +18,7 @@ SIMULATION_MODE = NORMAL NAME = InstanceDriver NPROC = 1 BIN_PATH = - INPUT_FILES = + INPUT_FILES = $PWD/instance_base_notebook.ipynb OUTPUT_FILES = SCRIPT = $PWD/instance_driver.py MODULE = From 3cb0d590a007903253fdcf16c9b03a6c7be8a7bc Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 15:20:45 -0500 Subject: [PATCH 5/9] Move base ensemble instance notebook to input dir so that it will be "seen" by instances for registration. --- examples-proposed/024-aggregated-compute-ensemble/ensemble.conf | 1 + .../{ => input_dir}/instance_base_notebook.ipynb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) rename examples-proposed/024-aggregated-compute-ensemble/{ => input_dir}/instance_base_notebook.ipynb (97%) diff --git a/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf b/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf index b01655f0..b366327f 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf +++ b/examples-proposed/024-aggregated-compute-ensemble/ensemble.conf @@ -4,6 +4,7 @@ LOG_FILE = log LOG_LEVEL = INFO SIMULATION_MODE = NORMAL +# This is where we have the base jupyter notebook for the ensemble instances. INPUT_DIR = $PWD/input_dir/ USE_PORTAL = True diff --git a/examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb similarity index 97% rename from examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb rename to examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb index 294b0dd9..a428a33d 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/instance_base_notebook.ipynb +++ b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb @@ -41,7 +41,7 @@ "# Find the JSON output from `gen_data.py` that was invoked as a sub-task. It will\n", "# follow the naming pattern `_solution.json`. It will contain two\n", "# arrays, `x` and `u`, which we will plot.\n", - "solution_json_file = list(Path(\".\").rglob(\"*solution.json\"))\n", + "solution_json_file = list(Path(\"..\").rglob(\"*solution.json\"))\n", "\n", "if len(solution_json) != 1:\n", " raise RuntimeError(\n", From 4dbe915ab16be6c0ac82939e336c9ce006250f34 Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 15:27:17 -0500 Subject: [PATCH 6/9] Another attempt at getting the notebook to show up where we can get at it. --- .../024-aggregated-compute-ensemble/template.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/template.conf b/examples-proposed/024-aggregated-compute-ensemble/template.conf index b74fd872..ee25e943 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/template.conf +++ b/examples-proposed/024-aggregated-compute-ensemble/template.conf @@ -4,6 +4,10 @@ LOG_FILE = log LOG_LEVEL = INFO SIMULATION_MODE = NORMAL +# This is where we have the base jupyter notebook for the ensemble instances. +INPUT_DIR = $PWD/input_dir/ + + [PORTS] NAMES = DRIVER WORKER [[DRIVER]] @@ -18,7 +22,7 @@ SIMULATION_MODE = NORMAL NAME = InstanceDriver NPROC = 1 BIN_PATH = - INPUT_FILES = $PWD/instance_base_notebook.ipynb + INPUT_FILES = OUTPUT_FILES = SCRIPT = $PWD/instance_driver.py MODULE = From 1f9107627db63aeff2a95fca558f14d489917f32 Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Thu, 18 Dec 2025 15:45:33 -0500 Subject: [PATCH 7/9] Removed errant import --- examples-proposed/024-aggregated-compute-ensemble/gen_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py index f54c0360..0c9f5bf0 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/gen_data.py +++ b/examples-proposed/024-aggregated-compute-ensemble/gen_data.py @@ -19,7 +19,7 @@ import numpy as np from ipsframework.resourceHelper import get_platform_info -from ipsframework.services import add_analysis_data_file + def main(instance: str, alpha: float, L:float, T_final:float, Nx:int, Nt:int) -> dict[str, Any]: From 1c55db31dd24672d2ad71948ae0cf17b3388c50d Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Fri, 9 Jan 2026 10:33:29 -0500 Subject: [PATCH 8/9] Add data generation code reference to README --- examples-proposed/024-aggregated-compute-ensemble/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples-proposed/024-aggregated-compute-ensemble/README.md b/examples-proposed/024-aggregated-compute-ensemble/README.md index cc68ad63..377a982e 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/README.md +++ b/examples-proposed/024-aggregated-compute-ensemble/README.md @@ -14,6 +14,7 @@ ignored. These are due to Dask not having a clean shutdown. * `driver.py` -- top-level driver * `instance_component.py` -- component worker code * `instance_driver.py` -- component driver code +* `gen_data.py` -- data generation code for the component * `ensemble.conf` -- top-level configuration file * `platform.conf` -- platform configuration file From d6aa14ee232093b42f161b5726863ef363f2cb67 Mon Sep 17 00:00:00 2001 From: Mark Coletti Date: Fri, 9 Jan 2026 10:33:36 -0500 Subject: [PATCH 9/9] Fix variable name in solution JSON file check and update plot save path --- .../input_dir/instance_base_notebook.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb index a428a33d..6323404d 100644 --- a/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb +++ b/examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb @@ -43,7 +43,7 @@ "# arrays, `x` and `u`, which we will plot.\n", "solution_json_file = list(Path(\"..\").rglob(\"*solution.json\"))\n", "\n", - "if len(solution_json) != 1:\n", + "if len(solution_json_file) != 1:\n", " raise RuntimeError(\n", " f\"Expected exactly one solution JSON file, found {len(solution_json_file)}\"\n", " )\n", @@ -62,7 +62,7 @@ "plt.ylabel(\"Temperature (u)\")\n", "plt.title(\"Solution of 1D Heat Equation\")\n", "plt.grid(True)\n", - "plt.savefig(f\"{instance}_solution.png\")" + "plt.savefig(f\"solution.png\")" ], "id": "b1a40da330ba6b3e", "outputs": [],