234 jupyter notebooks for example 024

examples-proposed/024-aggregated-compute-ensemble/README.md

@@ -14,6 +14,7 @@ ignored.  These are due to Dask not having a clean shutdown.
 * `driver.py` -- top-level driver
 * `instance_component.py` -- component worker code
 * `instance_driver.py` -- component driver code
+* `gen_data.py` -- data generation code for the component
 
 * `ensemble.conf` -- top-level configuration file
 * `platform.conf` -- platform configuration file

examples-proposed/024-aggregated-compute-ensemble/ensemble.conf

@@ -4,6 +4,7 @@ LOG_FILE = log
 LOG_LEVEL = INFO
 SIMULATION_MODE = NORMAL
 
+# This is where we have the base jupyter notebook for the ensemble instances.
 INPUT_DIR = $PWD/input_dir/
 
 USE_PORTAL = True

examples-proposed/024-aggregated-compute-ensemble/gen_data.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python3
 """
     Used to generate synthetic data as an example.
+
+    Writes two output files:
+
+    * `<instance ID>_solution.json`: contains the synthetic data
+    * `<instance ID>_stats.csv`: contains provenance information about the run
+
+    The JSON file is, in turn, read by the a per-instance jupyter notebook
+    available on the Portal to generate a plot of the data.
 """
 import argparse
 from typing import Any
@@ -9,10 +17,10 @@
 from time import time
 from traceback import print_exc
 import numpy as np
-import matplotlib.pyplot as plt
 
 from ipsframework.resourceHelper import get_platform_info
 
+
 def main(instance: str,
          alpha: float, L:float, T_final:float, Nx:int, Nt:int) -> dict[str, Any]:
     """ Generate synthetic data to emulate an actual simulation or complex
@@ -57,13 +65,7 @@ def main(instance: str,
             u_new[i] = u[i] + r * (u[i + 1] - 2 * u[i] + u[i - 1])
         u = u_new
 
-    # Plotting the result
-    plt.plot(x, u)
-    plt.xlabel("Position (x)")
-    plt.ylabel("Temperature (u)")
-    plt.title("Solution of 1D Heat Equation")
-    plt.grid(True)
-    plt.savefig(f"{instance}_solution.png")
+
 
     # Save some per-component stats
     stats_fname = f'{instance}_stats.csv'
@@ -114,5 +116,6 @@ def main(instance: str,
 
     except Exception as e:
         print(f'Encountered error: {e}')
-        print(f'Encountered error: {e}', file='gen_data_error.txt')
+        with open('gen_data_error.txt', 'w') as f:
+            print(f'Encountered error: {e!s}', file=f)
         print_exc()

examples-proposed/024-aggregated-compute-ensemble/input_dir/instance_base_notebook.ipynb

@@ -0,0 +1,101 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "# Instance base notebook\n",
+    "\n",
+    "This notebook replicates for each ensemble instance."
+   ],
+   "id": "56c3997f94a366d5"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-12-18T18:45:35.472364Z",
+     "start_time": "2025-12-18T18:45:35.462785Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import json\n",
+    "from pathlib import Path\n",
+    "\n",
+    "%matplotlib notebook\n",
+    "import matplotlib.pyplot as plt"
+   ],
+   "id": "937aff95abd359fa",
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-12-18T18:40:41.874866Z",
+     "start_time": "2025-12-18T18:40:41.864743Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# Find the JSON output from `gen_data.py` that was invoked as a sub-task. It will\n",
+    "# follow the naming pattern `<instance ID>_solution.json`.  It will contain two\n",
+    "# arrays, `x` and `u`, which we will plot.\n",
+    "solution_json_file = list(Path(\"..\").rglob(\"*solution.json\"))\n",
+    "\n",
+    "if len(solution_json_file) != 1:\n",
+    "    raise RuntimeError(\n",
+    "        f\"Expected exactly one solution JSON file, found {len(solution_json_file)}\"\n",
+    "    )\n",
+    "solution_json_file = solution_json_file[0]\n",
+    "\n",
+    "# Now read the JSON file and extract the data.\n",
+    "with open(solution_json_file, \"r\") as f:\n",
+    "    solution_data = json.load(f)\n",
+    "\n",
+    "x = solution_data[\"x\"]\n",
+    "u = solution_data[\"u\"]\n",
+    "\n",
+    "# Now plot the data.\n",
+    "plt.plot(x, u)\n",
+    "plt.xlabel(\"Position (x)\")\n",
+    "plt.ylabel(\"Temperature (u)\")\n",
+    "plt.title(\"Solution of 1D Heat Equation\")\n",
+    "plt.grid(True)\n",
+    "plt.savefig(f\"solution.png\")"
+   ],
+   "id": "b1a40da330ba6b3e",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "66002c188c5ab473"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples-proposed/024-aggregated-compute-ensemble/instance_component.py

@@ -26,8 +26,8 @@ def create_cmd(instance: str, path: Path, alpha: float, L:float, T_final:float,
     :param Nt: number of time steps
     :returns: list of command line arguments to be executed in step()
     """
-    executable = f'{path!s}/gen_data.py'
-    cmd = ['python3', executable, '--instance', instance,
+    executable = path / 'gen_data.py'
+    cmd = ['python3', str(executable), '--instance', instance,
            '--alpha', alpha, '--L', L, '--T_final', T_final,
            '--Nx', Nx, '--Nt', Nt]
     return cmd
@@ -71,12 +71,12 @@ def step(self, timestamp: float = 0.0, **keywords):
         self.services.info(f'{instance_id}: Completed MPI executable with '
                            f'return value: {return_value}.')
 
-        # TODO temporarily commenting this out until the actual
-        # example is ready to consider adding data files to the portal.  This
-        # originally came from code Lance wrote in a previous example.
-        # try:
-        #     self.services.add_analysis_data_files([data_fname, stats_fname], timestamp)
-        # except Exception:
-        #     print('did not add data files to portal, check logs')
+        # Add the generated data JSON and CSV files to the portal
+        try:
+            self.services.add_analysis_data_files([f'{instance_id}_solution.json',
+                                                   f'{instance_id}_stats.csv'],
+                                                  replace=True)
+        except Exception:
+            print('did not add data files to portal, check logs')
 
         self.services.info(f'{instance_id}: End of step of instance component.')

examples-proposed/024-aggregated-compute-ensemble/instance_driver.py

@@ -5,11 +5,22 @@
 
 from ipsframework import Component
 
+# The notebook that will be copied for each ensemble instance
+SOURCE_NOTEBOOK_NAME='instance_base_notebook.ipynb'
+
 
 class InstanceDriver(Component):
     """
     Instance driver component that steps the main component
     """
+    def init(self, timestamp: float = 0.0, **keywords):
+        self.services.stage_input_files([SOURCE_NOTEBOOK_NAME])
+
+        self.services.initialize_jupyter_notebook(
+                dest_notebook_name='jupyterhub_instance_notebook.ipynb',
+                source_notebook_path=SOURCE_NOTEBOOK_NAME,
+        )
+
 
     def step(self, timestamp: float = 0.0, **keywords):
         instance_component = self.services.get_port('WORKER')

examples-proposed/024-aggregated-compute-ensemble/template.conf

@@ -4,6 +4,10 @@ LOG_FILE = log
 LOG_LEVEL = INFO
 SIMULATION_MODE = NORMAL
 
+# This is where we have the base jupyter notebook for the ensemble instances.
+INPUT_DIR = $PWD/input_dir/
+
+
 [PORTS]
     NAMES = DRIVER WORKER
     [[DRIVER]]