From 46a33a854238281fc6d39c038873e76f17841e1a Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 11:42:02 +0000 Subject: [PATCH 1/7] Make changes to post process script for min and max --- CONTRIBUTORS.md | 3 ++- post-processing/vernier/vernier_data.py | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 577a19a0..a30b1730 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -5,4 +5,5 @@ | andrewcoughtrie | Andrew Coughtrie | Met Office | 2026-03-05 | | mo-marqh | mark Hedley | Met Office | 2026-03-10 | | oakleybrunt | Oakley Brunt | Met Office | 2026-03-09 | -| EdHone | Ed Hone | Met Office | 2026-03-20 | \ No newline at end of file +| EdHone | Ed Hone | Met Office | 2026-03-20 | +| MetBenjaminWent | Benjamin Went | Met Office | 2026-03-25 | \ No newline at end of file diff --git a/post-processing/vernier/vernier_data.py b/post-processing/vernier/vernier_data.py index 0509623b..87659133 100644 --- a/post-processing/vernier/vernier_data.py +++ b/post-processing/vernier/vernier_data.py @@ -142,7 +142,11 @@ def reduce(self) -> list: return [ self.name, # calliper name round(np.mean(self.total_time), 5), # mean total time across calls - round(np.mean(self.self_time), 5), # mean self time across calls + round(np.min(self.total_time), 5), # min total time across calls + round(np.max(self.total_time), 5), # max total time across calls + round(np.mean(self.self_time), 5), # mean self time across calls + round(np.min(self.self_time), 5), # min self time across calls + round(np.max(self.self_time), 5), # max self time across calls round(np.mean(self.cumul_time), 5), # mean cumulative time across calls self.n_calls[0], # number of calls (should be the same for all entries, so just take the first) round(np.mean(self.time_percent), 5), # mean percentage of time across calls @@ -151,8 +155,8 @@ def reduce(self) -> list: @classmethod def labels(self): - return ["Routine", "Total time (s)", "Self (s)", "Cumul time (s)", - "Max no. calls", "% time", "Time per call (s)"] + return ["Routine", "Total Mean(s)", "Total Min(s)", "Total Max(s)","Self Mean(s)", "Self Min(s)", "Self Max(s)", "Cumul time(s)", + "Max no. calls", "% time", "Time per call(s)"] class VernierData(): @@ -239,8 +243,10 @@ def write_txt_output(self, txt_path: Optional[Path] = None): else: out = open(txt_path, 'w') + # The issue is here, the write is manually formatted..... for row in txt_table: - out.write('| {:>{}} | {:>14} | {:>12} | {:>14} | {:>13} | {:>8} | {:>17} |\n'.format(row[0], max_calliper_len, *row[1:])) + out.write('| {:>{}} | {:>14} | {:>14} | {:>14} | {:>14} | {:>14} | {:>12} | {:>14} | {:>13} | {:>8} | {:>17} |\ + \n'.format(row[0], max_calliper_len, *row[1:])) if txt_path is not None: out.close() From a355ec5aef2c2a38c6b567aa9cf0ae4f1dbdaf7f Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 11:42:34 +0000 Subject: [PATCH 2/7] Update testing Wip --- post-processing/tests/test_vernier_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/post-processing/tests/test_vernier_data.py b/post-processing/tests/test_vernier_data.py index 3735152b..fee9a6ec 100644 --- a/post-processing/tests/test_vernier_data.py +++ b/post-processing/tests/test_vernier_data.py @@ -182,8 +182,8 @@ def test_write_txt_output_file(self): self.test_data.write_txt_output(Path(tmp_file.name)) contents = Path(tmp_file.name).read_text().splitlines() # pylint: disable=line-too-long - self.assertEqual("| Routine | Total time (s) | Self (s) | Cumul time (s) | Max no. calls | % time | Time per call (s) |", contents[0]) - self.assertEqual("| test_calliper | 30.0 | 10.0 | 35.0 | 2 | 15.0 | 15.0 |", contents[1]) + self.assertEqual("| Routine | Total Mean(s) | Total Min(s) | Total Max(s) | Self Mean(s) | Self Min(s) | Self Max(s) | Cumul time(s) | Max no. calls | % time | Time per call(s) |", contents[0]) + self.assertEqual("| test_calliper | 30.0 | 25.0 | 35.0 | 10.0 | 5.0 | 15.0 | 35.0 | 2 | 15.0 | 15.0 |", contents[1]) def test_write_txt_output_terminal(self): """ @@ -205,8 +205,8 @@ def test_write_txt_output_terminal(self): sys.stdout = sys.__stdout__ # pylint: disable=line-too-long - self.assertEqual("| Routine | Total time (s) | Self (s) | Cumul time (s) | Max no. calls | % time | Time per call (s) |", write_output.getvalue().splitlines()[0]) - self.assertEqual("| test_calliper | 35.0 | 3.5 | 11.0 | 2 | 45.0 | 17.5 |", write_output.getvalue().splitlines()[1]) + self.assertEqual("| Routine | Total Mean(s) | Total Min(s) | Total Max(s) | Self Mean(s) | Self Min(s) | Self Max(s) | Cumul time(s) | Max no. calls | % time | Time per call(s) |", write_output.getvalue().splitlines()[0]) + self.assertEqual("| test_calliper | 35.0 | 15.0 | 55.0 | 3.5 | 3.0 | 4.0 | 11.0 | 2 | 45.0 | 17.5 |", write_output.getvalue().splitlines()[1]) def test_aggregate(self): """ From efad94a871db9115daff91d0e0a75b6e0931616e Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 11:46:53 +0000 Subject: [PATCH 3/7] Adjust end formatting --- post-processing/vernier/vernier_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/post-processing/vernier/vernier_data.py b/post-processing/vernier/vernier_data.py index 87659133..fdd64a4f 100644 --- a/post-processing/vernier/vernier_data.py +++ b/post-processing/vernier/vernier_data.py @@ -245,8 +245,7 @@ def write_txt_output(self, txt_path: Optional[Path] = None): # The issue is here, the write is manually formatted..... for row in txt_table: - out.write('| {:>{}} | {:>14} | {:>14} | {:>14} | {:>14} | {:>14} | {:>12} | {:>14} | {:>13} | {:>8} | {:>17} |\ - \n'.format(row[0], max_calliper_len, *row[1:])) + out.write('| {:>{}} | {:>14} | {:>14} | {:>14} | {:>14} | {:>14} | {:>12} | {:>14} | {:>13} | {:>8} | {:>17} |\n'.format(row[0], max_calliper_len, *row[1:])) if txt_path is not None: out.close() From e3565f891df3840b2bea99abbd0d93a792f42b1a Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 12:39:33 +0000 Subject: [PATCH 4/7] Update tests --- post-processing/tests/test_vernier_calliper.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/post-processing/tests/test_vernier_calliper.py b/post-processing/tests/test_vernier_calliper.py index 3f9bf916..4e552f37 100644 --- a/post-processing/tests/test_vernier_calliper.py +++ b/post-processing/tests/test_vernier_calliper.py @@ -52,11 +52,15 @@ def test_reduce(self): reduced_data = self.calliper_a.reduce() self.assertEqual(reduced_data[0], "test_calliper_a") self.assertEqual(reduced_data[1], 30.0) - self.assertEqual(reduced_data[2], 10.0) + self.assertEqual(reduced_data[2], 25.0) self.assertEqual(reduced_data[3], 35.0) - self.assertEqual(reduced_data[4], 2) - self.assertEqual(reduced_data[5], 15.0) + self.assertEqual(reduced_data[4], 10.0) + self.assertEqual(reduced_data[5], 5.0) self.assertEqual(reduced_data[6], 15.0) + self.assertEqual(reduced_data[7], 35.0) + self.assertEqual(reduced_data[8], 2) + self.assertEqual(reduced_data[9], 15.0) + self.assertEqual(reduced_data[10], 15.0) def test_compare(self): """ From 601c09d60c8abb84fc8ee817a0251e22058e0d5d Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 12:52:00 +0000 Subject: [PATCH 5/7] Remove debugging comment --- post-processing/vernier/vernier_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/post-processing/vernier/vernier_data.py b/post-processing/vernier/vernier_data.py index fdd64a4f..29afd97b 100644 --- a/post-processing/vernier/vernier_data.py +++ b/post-processing/vernier/vernier_data.py @@ -243,7 +243,6 @@ def write_txt_output(self, txt_path: Optional[Path] = None): else: out = open(txt_path, 'w') - # The issue is here, the write is manually formatted..... for row in txt_table: out.write('| {:>{}} | {:>14} | {:>14} | {:>14} | {:>14} | {:>14} | {:>12} | {:>14} | {:>13} | {:>8} | {:>17} |\n'.format(row[0], max_calliper_len, *row[1:])) From 961901047dd828449ca5e7151027598498bffd93 Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Wed, 25 Mar 2026 12:54:59 +0000 Subject: [PATCH 6/7] Some minor flake8 fixes --- post-processing/vernier/vernier_data.py | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/post-processing/vernier/vernier_data.py b/post-processing/vernier/vernier_data.py index 29afd97b..d3ce9906 100644 --- a/post-processing/vernier/vernier_data.py +++ b/post-processing/vernier/vernier_data.py @@ -55,7 +55,7 @@ def __len__(self): """ result = None if (len(self.time_percent) == len(self.cumul_time) == - len(self.self_time) == len(self.total_time) == len(self.n_calls)): + len(self.self_time) == len(self.total_time) == len(self.n_calls)): result = len(self.time_percent) return result @@ -140,22 +140,22 @@ def reduce(self) -> list: """ return [ - self.name, # calliper name - round(np.mean(self.total_time), 5), # mean total time across calls - round(np.min(self.total_time), 5), # min total time across calls - round(np.max(self.total_time), 5), # max total time across calls - round(np.mean(self.self_time), 5), # mean self time across calls - round(np.min(self.self_time), 5), # min self time across calls - round(np.max(self.self_time), 5), # max self time across calls - round(np.mean(self.cumul_time), 5), # mean cumulative time across calls - self.n_calls[0], # number of calls (should be the same for all entries, so just take the first) - round(np.mean(self.time_percent), 5), # mean percentage of time across calls - round(np.mean(np.array(self.total_time) / np.array(self.n_calls)), 5) # mean time per call + self.name, # calliper name + round(np.mean(self.total_time), 5), # mean total time across calls + round(np.min(self.total_time), 5), # min total time across calls + round(np.max(self.total_time), 5), # max total time across calls + round(np.mean(self.self_time), 5), # mean self time across calls + round(np.min(self.self_time), 5), # min self time across calls + round(np.max(self.self_time), 5), # max self time across calls + round(np.mean(self.cumul_time), 5), # mean cumulative time across calls + self.n_calls[0], # number of calls (should be the same for all entries, so just take the first) + round(np.mean(self.time_percent), 5), # mean percentage of time across calls + round(np.mean(np.array(self.total_time) / np.array(self.n_calls)), 5) # mean time per call ] @classmethod def labels(self): - return ["Routine", "Total Mean(s)", "Total Min(s)", "Total Max(s)","Self Mean(s)", "Self Min(s)", "Self Max(s)", "Cumul time(s)", + return ["Routine", "Total Mean(s)", "Total Min(s)", "Total Max(s)", "Self Mean(s)", "Self Min(s)", "Self Max(s)", "Cumul time(s)", "Max no. calls", "% time", "Time per call(s)"] @@ -332,7 +332,6 @@ def aggregate(self, vernier_data_list=None, internal_consistency=True): self.data[calliper].thread.extend(vernier_data.data[calliper].thread) - class VernierDataCollation(): """ Class to hold an collation of VernierData instances. From 7e2a3fdf98b2c91fbfd4432247f9a9bdc3ddefa9 Mon Sep 17 00:00:00 2001 From: MetBenjaminWent Date: Tue, 7 Apr 2026 16:39:04 +0100 Subject: [PATCH 7/7] Update with output from tool --- post-processing/tests/test_cli_tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/post-processing/tests/test_cli_tools.py b/post-processing/tests/test_cli_tools.py index 55c62eb8..dbf486d4 100644 --- a/post-processing/tests/test_cli_tools.py +++ b/post-processing/tests/test_cli_tools.py @@ -23,9 +23,9 @@ def setUp(self): self.test_data_dir = Path(__file__).parent / 'data' # pylint: disable=line-too-long self.test_data_kgo = ( - '| Routine | Total time (s) | Self (s) | Cumul time (s) | Max no. calls | % time | Time per call (s) |\n' + - '| __test_app__ | 6.3465 | 2.5855 | 4.0895 | 1 | 40.9825 | 6.3465 |\n' + - '| some_process | 2.573 | 2.516 | 3.8075 | 2 | 39.277 | 1.2865 |\n' + '| Routine | Total Mean(s) | Total Min(s) | Total Max(s) | Self Mean(s) | Self Min(s) | Self Max(s) | Cumul time(s) | Max no. calls | % time | Time per call(s) |\n' + + '| __test_app__ | 6.3465 | 5.854 | 6.839 | 2.5855 | 2.583 | 2.588 | 4.0895 | 1 | 40.9825 | 6.3465 |\n' + + '| some_process | 2.573 | 2.077 | 3.069 | 2.516 | 2.023 | 3.009 | 3.8075 | 2 | 39.277 | 1.2865 |\n' ) return super().setUp()