Feat_emulators_module

autotest/la_tests.py

@@ -595,167 +595,10 @@ def ends_freyberg_test(tmp_path):
 
 
 
-def ends_run_freyberg_dsi(tmp_d, nst=False, nst_extrap=None, ztz=False, energy=1.0):
-    import pyemu
-    import os
-    import pandas as pd
-    import numpy as np
-    test_d = "ends_master"
-    test_d = setup_tmp(test_d, tmp_d)
-    case = "freyberg6_run_ies"
-    pst_name = os.path.join(test_d, case + ".pst")
-    pst = pyemu.Pst(pst_name)
-    predictions = ["headwater_20171130", "tailwater_20161130", "trgw_0_9_1_20161130"]
-    pst.pestpp_options["predictions"] = predictions
-
-    oe_name = pst_name.replace(".pst", ".0.obs.csv")
-    oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=oe_name).iloc[:100, :]
-
-    ends = pyemu.EnDS(pst=pst, sim_ensemble=oe,verbose=True)
-    t_d = os.path.join(tmp_d, "dsi_template")
-
-    ends.prep_for_dsi(t_d=t_d,
-                      use_ztz=ztz,
-                      apply_normal_score_transform=nst,
-                      nst_extrap=nst_extrap,
-                      energy=energy)
-    # copy exe to dsi_template
-    #shutil.copy2(os.path.join(test_d,"pestpp-ies.exe"),os.path.join(t_d,"pestpp-ies.exe"))
-    filename=os.path.join(t_d,"dsi.0.obs.csv")
-    if os.path.exists(filename):
-        os.remove(filename)
-    pst = pyemu.Pst(os.path.join(t_d,"dsi.pst"))
-    pst.control_data.noptmax = -1
-    pst.pestpp_options["overdue_giveup_fac"] = 100000000
-    pst.write(os.path.join(t_d,"dsi.pst"),version=2)
-    #pyemu.os_utils.run("pestpp-ies dsi.pst",cwd=t_d)
-
-    pvals = pd.read_csv(os.path.join(t_d,"dsi_pars.csv"),index_col=0)
-    pmat = np.load(os.path.join(t_d,"dsi_proj_mat.npy"))
-    ovals = pd.read_csv(os.path.join(t_d,"dsi_pr_mean.csv"),index_col=0)
-
-
-    m_d = t_d.replace("template","master")
-    port = _get_port()
-    pyemu.os_utils.start_workers(t_d, ies_exe_path,"dsi.pst",
-                                 worker_root=tmp_d,
-                                 master_dir=m_d, num_workers=10, port=port,
-                                ppw_function=pyemu.helpers.dsi_pyworker,
-                                ppw_kwargs={"pmat":pmat,"ovals":ovals,"pvals":pvals})
-    #read in the results
-    oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=os.path.join(m_d,"dsi.0.obs.csv"))
-    assert oe.shape[0]==50, f"{50-oe.shape[0]} failed runs"
-    phi_vector = oe.phi_vector.sort_values().values
-    assert phi_vector[0] != phi_vector[1],phi_vector
-
-def ends_freyberg_dsi_test(tmp_path):
-    ends_run_freyberg_dsi(tmp_path)
-
-def ends_freyberg_dsi_nst_test(tmp_path):
-    ends_run_freyberg_dsi(tmp_path,nst=True,nst_extrap=None)
-
-def ends_freyberg_dsi_extrap_test(tmp_path):
-    ends_run_freyberg_dsi(tmp_path,nst=True,nst_extrap='quadratic')
-
-def ends_freyberg_dsi_ztz_test(tmp_path):
-    ends_run_freyberg_dsi(tmp_path,ztz=True)
-
-def ends_freyberg_dsi_svd_test(tmp_path):
-    ends_run_freyberg_dsi(tmp_path,ztz=True,energy=0.999)
-
-
-def plot_freyberg_dsi():
-    import pandas as pd
-    import pyemu
-    import matplotlib.pyplot as plt
-
-    test_d = "ends_master"
-    case = "freyberg6_run_ies"
-    pst_name = os.path.join(test_d, case + ".pst")
-    pst = pyemu.Pst(pst_name)
-    predictions = ["headwater_20171130", "tailwater_20161130", "trgw_0_9_1_20161130"]
-    oe_name = pst_name.replace(".pst", ".0.obs.csv")
-    pr_oe = pd.read_csv(os.path.join(test_d,"freyberg6_run_ies.0.obs.csv"),index_col=0)
-    pt_oe = pd.read_csv(os.path.join(test_d, "freyberg6_run_ies.3.obs.csv"), index_col=0)
-
-    m_d = os.path.join("dsi", "master_dsi")
-    pst = pyemu.Pst(os.path.join(m_d,"dsi.pst"))
-    pr_oe_dsi = pd.read_csv(os.path.join(m_d,"dsi.0.obs.csv"),index_col=0)
-    pt_oe_dsi = pd.read_csv(os.path.join(m_d, "dsi.3.obs.csv"), index_col=0)
-
-    pv = pyemu.ObservationEnsemble(pst=pst,df=pt_oe).phi_vector
-    pv_dsi = pyemu.ObservationEnsemble(pst=pst, df=pt_oe_dsi).phi_vector
-    #print(pt_oe.shape)
-    pt_oe = pt_oe.loc[pv<25, :]
-    pt_oe_dsi = pt_oe_dsi.loc[pv_dsi < 25, :]
-
-    # print(pt_oe.shape)
-    # fig,ax = plt.subplots(1,1,figsize=(5,5))
-    # ax.hist(pv,bins=10,facecolor="b",alpha=0.5,density=True)
-    # ax.hist(pv_dsi, bins=10, facecolor="m", alpha=0.5,density=True)
-    # ax.set_yticks([])
-    # plt.tight_layout()
-    # plt.show()
-
-
-
-    fig,axes = plt.subplots(len(predictions),1,figsize=(10,10))
-    for p,ax in zip(predictions,axes):
-        ax.hist(pr_oe.loc[:,p].values,bins=10,alpha=0.5,facecolor="0.5",density=True,label="prior")
-        ax.hist(pt_oe.loc[:, p].values, bins=10, alpha=0.5, facecolor="b",density=True,label="posterior")
-        ax.hist(pr_oe_dsi.loc[:, p].values, bins=10, facecolor="none",hatch="/",edgecolor="0.5",
-                lw=2.5,density=True,label="dsi prior")
-        ax.hist(pt_oe_dsi.loc[:, p].values, bins=10, facecolor="none",density=True,hatch="/",edgecolor="b",lw=2.5,
-                label="dsi posterior")
-        ax.set_title(p,loc="left")
-        ax.legend(loc="upper right")
-        ax.set_yticks([])
-    plt.tight_layout()
-    plt.savefig("dsi_pred.pdf")
-
-
-def dsi_normscoretransform_test():
-    import numpy as np
-    import pyemu
-    from pyemu.utils.helpers import randrealgen_optimized,normal_score_transform,inverse_normal_score_transform
-    test_d = "ends_master"
-    case = "freyberg6_run_ies"
-    pst_name = os.path.join(test_d, case + ".pst")
-    pst = pyemu.Pst(pst_name)
-
-    oe_name = pst_name.replace(".pst", ".0.obs.csv")
-    oe = pyemu.ObservationEnsemble.from_csv(pst=pst, filename=oe_name).iloc[:100, :]
-
-    nstval = randrealgen_optimized(oe.shape[0], 1e-7, 1e4)
-    window_size=3   
-    if oe.shape[0]>40:
-        window_size=5                    
-    if oe.shape[0]>90:
-        window_size=7
-    if oe.shape[0]>200:
-        window_size=9            
-    for name in oe.columns:
-        print("transforming:",name)
-        sorted_values = oe._df.loc[:,name].sort_values().copy()
-        #if all values are the same, skip
-        if sorted_values.iloc[0] == sorted_values.iloc[-1]:
-            print("all values are the same, skipping")
-            continue
-        sorted_values.loc[:] = pyemu.eds.moving_average_with_endpoints(sorted_values.values, window_size)
-        transformed_values = np.asarray([normal_score_transform(nstval, sorted_values, value)[0] for value in sorted_values])
-        backtransformed_values = np.asarray([inverse_normal_score_transform(nstval, sorted_values, value)[0] for value in transformed_values])
-
-        diff = backtransformed_values-sorted_values
-        assert max(abs(diff))<1e-7, backtransformed_values
-
 
 if __name__ == "__main__":
-    #dsi_normscoretransform_test()
-    #ends_freyberg_test("temp")
-    ends_freyberg_dsi_test("temp")
+    ends_freyberg_test("temp")
     #ends_freyberg_dev()
-    #ends_freyberg_dsi_test("temp")
-    #plot_freyberg_dsi()
     #obscomp_test()
     #alternative_dw()
     #freyberg_verf_test()