Update fetch results (#779)

* update rbfe_results.tar.gz for new settings schema

* re-enable test_gather tests

* devscript for updating rbfe_results.tar.gz

* cli: gather: rework raw gather to work off protocol_result dict

this allows us to later deduplicate and remove the unit_results key from results dicts

* cli: gather: update expected gather results

current raw results are only one replicate, should probably have multiple repeats to test raw output

* cli: gather: use \t escape code in gather raw test

could instead normalise the whitespace to be more permissive...

* Apply suggestions from code review

Co-authored-by: Mike Henry <11765982+mikemhenry@users.noreply.github.com>

* rerun ci

---------

Co-authored-by: Mike Henry <11765982+mikemhenry@users.noreply.github.com>

devtools/data/fix_rbfe_results.py

@@ -0,0 +1,57 @@
+"""A script to fix up rbfe_results.tar.gz
+
+Useful if Settings are ever changed in a backwards-incompatible way
+
+Will expect "rbfe_results.tar.gz" in this directory, will overwrite this file
+"""
+from gufe.tokenization import JSON_HANDLER
+import glob
+import json
+from openfe.protocols import openmm_rfe
+import os.path
+import tarfile
+
+
+def untar(fn):
+    """extract tarfile called *fn*"""
+    with tarfile.open(fn) as f:
+        f.extractall()
+
+
+def retar(loc, name):
+    """create tar.gz called *name* of directory *loc*"""
+    with tarfile.open(name, mode='w:gz') as f:
+        f.add(loc, arcname=os.path.basename(loc))
+
+
+def replace_settings(fn, new_settings):
+    """replace settings instances in *fn* with *new_settings*"""
+    with open(fn, 'r') as f:
+        data = json.load(f)
+
+    for k in data['protocol_result']['data']:
+        data['protocol_result']['data'][k][0]['inputs']['settings'] = new_settings
+
+    for k in data['unit_results']:
+        data['unit_results'][k]['inputs']['settings'] = new_settings
+
+    with open(fn, 'w') as f:
+        json.dump(data, f, cls=JSON_HANDLER.encoder)
+
+
+def fix_rbfe_results():
+    untar('rbfe_results.tar.gz')
+
+    # generate valid settings as defaults
+    new_settings = openmm_rfe.RelativeHybridTopologyProtocol.default_settings()
+
+    # walk over all result jsons
+    for fn in glob.glob('./results/*json'):
+        # replace instances of settings within with valid settings
+        replace_settings(fn, new_settings)
+
+    retar('results', 'rbfe_results.tar.gz')
+
+
+if __name__ == '__main__':
+    fix_rbfe_results()

openfecli/commands/gather.py

@@ -131,10 +131,11 @@ def _generate_bad_legs_error_message(set_vals, ligpair):
 def _parse_raw_units(results: dict) -> list[tuple]:
     # grab individual unit results from master results dict
     # returns list of (estimate, uncertainty) tuples
-    pus = list(results['unit_results'].values())
+    list_of_pur = list(results['protocol_result']['data'].values())[0]
+
     return [(pu['outputs']['unit_estimate'],
              pu['outputs']['unit_estimate_error'])
-            for pu in pus]
+            for pu in list_of_pur]
 
 
 def _get_ddgs(legs, error_on_missing=True):
@@ -284,7 +285,7 @@ def _write_dg_mle(legs, writer, allow_partial):
     default="dg", show_default=True,
     help=(
         "What data to report. 'dg' gives maximum-likelihood estimate of "
-        "absolute deltaG,  'ddg' gives delta-delta-G, and 'dg-raw' gives "
+        "absolute deltaG,  'ddg' gives delta-delta-G, and 'raw' gives "
         "the raw result of the deltaG for a leg."
     )
 )

openfecli/tests/commands/test_gather.py

@@ -88,71 +88,34 @@ solvent	lig_ejm_46	lig_jmc_28	23.41	0.05
 
 _EXPECTED_RAW = b"""\
 leg	ligand_i	ligand_j	DG(i->j) (kcal/mol)	MBAR uncertainty (kcal/mol)
-complex	lig_ejm_31	lig_ejm_42	-14.77	0.04
-complex	lig_ejm_31	lig_ejm_42	-14.74	0.04
-complex	lig_ejm_31	lig_ejm_42	-14.94	0.04
-solvent	lig_ejm_31	lig_ejm_42	-15.68	0.03
-solvent	lig_ejm_31	lig_ejm_42	-15.69	0.03
-solvent	lig_ejm_31	lig_ejm_42	-15.64	0.03
-complex	lig_ejm_31	lig_ejm_46	-40.56	0.06
-complex	lig_ejm_31	lig_ejm_46	-40.76	0.05
-complex	lig_ejm_31	lig_ejm_46	-40.90	0.04
-solvent	lig_ejm_31	lig_ejm_46	-39.92	0.04
-solvent	lig_ejm_31	lig_ejm_46	-39.94	0.04
-solvent	lig_ejm_31	lig_ejm_46	-39.95	0.04
-complex	lig_ejm_31	lig_ejm_47	-27.68	0.08
-complex	lig_ejm_31	lig_ejm_47	-27.80	0.06
-complex	lig_ejm_31	lig_ejm_47	-27.51	0.07
-solvent	lig_ejm_31	lig_ejm_47	-27.83	0.05
-solvent	lig_ejm_31	lig_ejm_47	-27.84	0.05
-solvent	lig_ejm_31	lig_ejm_47	-27.88	0.05
-complex	lig_ejm_31	lig_ejm_48	-16.15	0.08
-complex	lig_ejm_31	lig_ejm_48	-15.96	0.07
-complex	lig_ejm_31	lig_ejm_48	-16.01	0.08
-solvent	lig_ejm_31	lig_ejm_48	-16.83	0.06
-solvent	lig_ejm_31	lig_ejm_48	-16.65	0.07
-solvent	lig_ejm_31	lig_ejm_48	-16.77	0.06
-complex	lig_ejm_31	lig_ejm_50	-57.31	0.04
-complex	lig_ejm_31	lig_ejm_50	-57.45	0.04
-complex	lig_ejm_31	lig_ejm_50	-57.37	0.04
-solvent	lig_ejm_31	lig_ejm_50	-58.33	0.04
-solvent	lig_ejm_31	lig_ejm_50	-58.42	0.04
-solvent	lig_ejm_31	lig_ejm_50	-58.19	0.04
-complex	lig_ejm_42	lig_ejm_43	-19.24	0.04
-complex	lig_ejm_42	lig_ejm_43	-18.72	0.05
-complex	lig_ejm_42	lig_ejm_43	-18.94	0.04
-solvent	lig_ejm_42	lig_ejm_43	-20.17	0.03
-solvent	lig_ejm_42	lig_ejm_43	-20.28	0.03
-solvent	lig_ejm_42	lig_ejm_43	-20.23	0.03
-complex	lig_ejm_46	lig_jmc_23	17.31	0.02
-complex	lig_ejm_46	lig_jmc_23	17.37	0.02
-complex	lig_ejm_46	lig_jmc_23	17.35	0.02
-solvent	lig_ejm_46	lig_jmc_23	17.20	0.02
-solvent	lig_ejm_46	lig_jmc_23	17.40	0.02
-solvent	lig_ejm_46	lig_jmc_23	17.30	0.02
-complex	lig_ejm_46	lig_jmc_27	15.84	0.03
-complex	lig_ejm_46	lig_jmc_27	15.79	0.03
-complex	lig_ejm_46	lig_jmc_27	15.80	0.03
-solvent	lig_ejm_46	lig_jmc_27	16.16	0.03
-solvent	lig_ejm_46	lig_jmc_27	16.01	0.03
-solvent	lig_ejm_46	lig_jmc_27	16.07	0.03
-complex	lig_ejm_46	lig_jmc_28	23.43	0.04
-complex	lig_ejm_46	lig_jmc_28	23.29	0.04
-complex	lig_ejm_46	lig_jmc_28	23.17	0.04
-solvent	lig_ejm_46	lig_jmc_28	23.67	0.03
-solvent	lig_ejm_46	lig_jmc_28	23.61	0.03
-solvent	lig_ejm_46	lig_jmc_28	23.65	0.03
+complex\tlig_ejm_31\tlig_ejm_42\t-14.9\t0.8
+solvent\tlig_ejm_31\tlig_ejm_42\t-15.7\t0.8
+complex\tlig_ejm_31\tlig_ejm_46\t-40.7\t0.8
+solvent\tlig_ejm_31\tlig_ejm_46\t-39.8\t0.8
+complex\tlig_ejm_31\tlig_ejm_47\t-27.8\t0.8
+solvent\tlig_ejm_31\tlig_ejm_47\t-27.8\t0.8
+complex\tlig_ejm_31\tlig_ejm_48\t-16.2\t0.8
+solvent\tlig_ejm_31\tlig_ejm_48\t-16.8\t0.8
+complex\tlig_ejm_31\tlig_ejm_50\t-57.3\t0.8
+solvent\tlig_ejm_31\tlig_ejm_50\t-58.3\t0.8
+complex\tlig_ejm_42\tlig_ejm_43\t-19.0\t0.8
+solvent\tlig_ejm_42\tlig_ejm_43\t-20.3\t0.8
+complex\tlig_ejm_46\tlig_jmc_23\t17.3\t0.8
+solvent\tlig_ejm_46\tlig_jmc_23\t17.2\t0.8
+complex\tlig_ejm_46\tlig_jmc_27\t15.9\t0.8
+solvent\tlig_ejm_46\tlig_jmc_27\t16.0\t0.8
+complex\tlig_ejm_46\tlig_jmc_28\t23.1\t0.8
+solvent\tlig_ejm_46\tlig_jmc_28\t23.5\t0.8
 """
 
 
-@pytest.mark.xfail
-@pytest.mark.parametrize('report', ["", "dg", "ddg"])
+@pytest.mark.parametrize('report', ["", "dg", "ddg", "raw"])
 def test_gather(results_dir, report):
     expected = {
         "": _EXPECTED_DG,
         "dg": _EXPECTED_DG,
         "ddg": _EXPECTED_DDG,
-        "dg-raw": _EXPECTED_DG_RAW,
+        "raw": _EXPECTED_RAW,
     }[report]
     runner = CliRunner()