added all info

.gitcommit

@@ -0,0 +1,3 @@
+.venv/
+.env
+.claude

.gitignore

@@ -0,0 +1,4 @@
+.env
+.venv/
+__pycache__/
+*.pyc

README.md

@@ -0,0 +1,21 @@
+# README
+
+Reference code for the ontology-guided RIPPER experiments. It runs entirely from
+the bundled Excel dataset (`excels/non_dummy_columns_df_less_classes.xlsx`) — no
+database connection is needed.
+
+## How to run
+### Activate a python env and install deps
+`python -m venv .venv`
+`source ./.venv/Scripts/activate`
+`pip install -r requirements.txt`
+
+### Run all experiments
+`python run_all_experiments.py`
+
+(The Weka JRip baseline additionally needs a Java runtime; it is skipped
+automatically if `java`/`weka.jar` are unavailable.)
+
+### Further info
+- `./results` holds all saved data behind the figures, including the learned rules.
+- the results that were used are in the subfolder `/pooled` (pooled over the 5 folds).

analyze_rule_dynamics.py

@@ -0,0 +1,216 @@
+"""
+Discussion analysis: WHY the un-guided RIPPER crosses the ontology-guided variants
+as label pollution increases.
+
+Read-only w.r.t. the main experiment: this script ONLY reads the already-saved CSVs
+and writes new figures + a regime table + a discussion draft into results/rule_dynamics/.
+It never re-fits a model and never touches results/*.csv.
+
+Inputs (all produced by run_all_experiments.py / the diagnostics):
+    results/summary_runtimeOFF.csv          mean/std per (lambda, method, level), 50 repeats
+    results/metrics_long.csv                per-fold rows (for paired guided-minus-unguided deltas)
+    results/diagnostics/rule_counts.csv     per-(level,run,method,class) rule counts (lambda=4, 10 runs)
+
+Outputs (results/rule_dynamics/):
+    total_rules_vs_pollution.png            rules learned per method
+    rare_class_rules_vs_pollution.png       rules for the rarest class (guided -> 0 at high pollution)
+    train_vs_test_accuracy.png              U-shaped train curve + the overfitting gap
+    invalid_rate_vs_pollution.png           % ontologically-invalid test predictions
+    delta_guided_minus_unguided.png         paired deltas (acc + macro-F1) with 95% CI -> crossovers
+    regime_summary.csv                      tidy mean-per-(method,level) table cited by the text
+"""
+
+from pathlib import Path
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+ROOT = Path(__file__).resolve().parent
+RESULTS = ROOT / "results"
+OUTDIR = RESULTS / "rule_dynamics"
+EXCEL = ROOT / "excels" / "non_dummy_columns_df_less_classes.xlsx"
+
+SUMMARY_CSV = RESULTS / "summary_runtimeOFF.csv"
+METRICS_CSV = RESULTS / "metrics_long.csv"
+RULES_CSV = RESULTS / "diagnostics" / "rule_counts.csv"
+
+# Fixed method order + colours so every figure is consistent.
+METHODS = ["no guidance", "onto MDL", "onto grow", "onto grow and prune"]
+GUIDED = ["onto MDL", "onto grow", "onto grow and prune"]
+COLOURS = {"no guidance": "#d62728", "onto MDL": "#1f77b4",
+           "onto grow": "#2ca02c", "onto grow and prune": "#9467bd"}
+LAM = 4  # rule_counts.csv only covers lambda=4; summary curves are ~identical for lambda=3
+
+
+def mean_ci(values):
+    """Mean and 95% CI half-width (normal approx) over a 1-D array."""
+    v = np.asarray(values, dtype=float)
+    v = v[~np.isnan(v)]
+    n = len(v)
+    if n == 0:
+        return np.nan, np.nan
+    m = v.mean()
+    if n < 2:
+        return m, 0.0
+    return m, 1.96 * v.std(ddof=1) / np.sqrt(n)
+
+
+def rarest_class():
+    """The least frequent action label in the dataset (the 'rare class')."""
+    y = pd.read_excel(EXCEL)["y"]
+    counts = y.value_counts()
+    return counts.index[-1], counts
+
+
+# ── load ──────────────────────────────────────────────────────────────────────
+OUTDIR.mkdir(parents=True, exist_ok=True)
+summary = pd.read_csv(SUMMARY_CSV)
+summary = summary[summary["lambda"] == LAM]
+metrics = pd.read_csv(METRICS_CSV)
+metrics = metrics[(metrics["lambda"] == LAM) & (metrics["runtime"] == "OFF")]
+rules = pd.read_csv(RULES_CSV)
+rules = rules[rules["lambda"] == LAM]
+RARE, CLASS_COUNTS = rarest_class()
+print(f"Rarest class = '{RARE}'  (class frequencies:\n{CLASS_COUNTS.to_string()})\n")
+
+LEVELS = sorted(summary["percentage"].unique())
+
+
+def _line(ax, x, mean, ci, label, colour, ls="-"):
+    ax.plot(x, mean, marker="o", ms=5, lw=2, color=colour, ls=ls, label=label)
+    ax.fill_between(x, np.array(mean) - np.array(ci), np.array(mean) + np.array(ci),
+                    color=colour, alpha=0.15)
+
+
+# ── (a) total rules vs pollution ────────────────────────────────────────────────
+# rule_counts repeats total_rules across the 6 class rows -> one value per (level,run,method)
+per_run = rules.drop_duplicates(["percentage", "run", "method"])
+fig, ax = plt.subplots(figsize=(9, 6))
+rows = []
+for m in METHODS:
+    sub = per_run[per_run["method"] == m]
+    xs, ms, cs = [], [], []
+    for lvl in LEVELS:
+        vals = sub[sub["percentage"] == lvl]["total_rules"].values
+        mm, cc = mean_ci(vals)
+        xs.append(lvl); ms.append(mm); cs.append(cc)
+        rows.append({"method": m, "level": lvl, "total_rules_mean": mm})
+    _line(ax, xs, ms, cs, m, COLOURS[m])
+ax.set_xlabel("% of training labels polluted")
+ax.set_ylabel("Total rules learned (mean ± 95% CI)")
+ax.set_title(f"Rules learned vs. label pollution (λ={LAM}, 10 runs, runtime OFF)", weight="bold")
+ax.set_xticks(LEVELS); ax.legend(title="method"); ax.grid(alpha=.3)
+plt.tight_layout(); plt.savefig(OUTDIR / "total_rules_vs_pollution.png", dpi=150); plt.close()
+total_rules_tbl = pd.DataFrame(rows)
+print("wrote total_rules_vs_pollution.png")
+
+# ── (b) rare-class rules vs pollution ────────────────────────────────────────────
+rare = rules[rules["klass"] == RARE]
+fig, ax = plt.subplots(figsize=(9, 6))
+rare_rows = []
+for m in METHODS:
+    sub = rare[rare["method"] == m]
+    xs, ms, cs = [], [], []
+    for lvl in LEVELS:
+        vals = sub[sub["percentage"] == lvl]["n_rules"].values
+        mm, cc = mean_ci(vals)
+        xs.append(lvl); ms.append(mm); cs.append(cc)
+        rare_rows.append({"method": m, "level": lvl, "rare_rules_mean": mm})
+    _line(ax, xs, ms, cs, m, COLOURS[m])
+ax.set_xlabel("% of training labels polluted")
+ax.set_ylabel(f"Rules for rarest class '{RARE}' (mean ± 95% CI)")
+ax.set_title(f"Rare-class rules vs. label pollution (λ={LAM}, 10 runs, runtime OFF)", weight="bold")
+ax.set_xticks(LEVELS); ax.legend(title="method"); ax.grid(alpha=.3)
+plt.tight_layout(); plt.savefig(OUTDIR / "rare_class_rules_vs_pollution.png", dpi=150); plt.close()
+rare_rules_tbl = pd.DataFrame(rare_rows)
+print("wrote rare_class_rules_vs_pollution.png")
+
+# ── (c) train vs test accuracy (50-repeat means from the summary) ─────────────────
+fig, (axL, axR) = plt.subplots(1, 2, figsize=(14, 6), sharex=True)
+for m in METHODS:
+    s = summary[summary["method"] == m].sort_values("percentage")
+    axL.plot(s["percentage"], s["train_accuracy_mean"], marker="o", ms=5, lw=2,
+             color=COLOURS[m], label=m)
+    axL.fill_between(s["percentage"], s["train_accuracy_mean"] - s["train_accuracy_std"],
+                     s["train_accuracy_mean"] + s["train_accuracy_std"], color=COLOURS[m], alpha=.12)
+    axR.plot(s["percentage"], s["test_accuracy_mean"], marker="o", ms=5, lw=2,
+             color=COLOURS[m], label=m)
+    axR.fill_between(s["percentage"], s["test_accuracy_mean"] - s["test_accuracy_std"],
+                     s["test_accuracy_mean"] + s["test_accuracy_std"], color=COLOURS[m], alpha=.12)
+axL.set_title("Train accuracy (note the U-shape)", weight="bold")
+axR.set_title("Test accuracy", weight="bold")
+for ax in (axL, axR):
+    ax.set_xlabel("% of training labels polluted"); ax.set_ylabel("accuracy")
+    ax.set_xticks(LEVELS); ax.grid(alpha=.3)
+axR.legend(title="method")
+fig.suptitle(f"Train vs. test accuracy (λ={LAM}, 50 repeats, runtime OFF)", weight="bold")
+plt.tight_layout(); plt.savefig(OUTDIR / "train_vs_test_accuracy.png", dpi=150); plt.close()
+print("wrote train_vs_test_accuracy.png")
+
+# ── (d) invalid_rate vs pollution ────────────────────────────────────────────────
+fig, ax = plt.subplots(figsize=(9, 6))
+for m in METHODS:
+    s = summary[summary["method"] == m].sort_values("percentage")
+    ax.plot(s["percentage"], s["invalid_rate_mean"], marker="o", ms=5, lw=2,
+            color=COLOURS[m], label=m)
+    ax.fill_between(s["percentage"], s["invalid_rate_mean"] - s["invalid_rate_std"],
+                    s["invalid_rate_mean"] + s["invalid_rate_std"], color=COLOURS[m], alpha=.12)
+ax.set_xlabel("% of training labels polluted")
+ax.set_ylabel("% test predictions ontologically INVALID")
+ax.set_title(f"Invalid-prediction rate vs. pollution (λ={LAM}, 50 repeats, runtime OFF)", weight="bold")
+ax.set_xticks(LEVELS); ax.legend(title="method"); ax.grid(alpha=.3)
+plt.tight_layout(); plt.savefig(OUTDIR / "invalid_rate_vs_pollution.png", dpi=150); plt.close()
+print("wrote invalid_rate_vs_pollution.png")
+
+# ── (e) paired guided-minus-unguided deltas (acc + macro-F1) with 95% CI ──────────
+# Common-random-numbers: every method shares each fold's data/pollution/seed, matched by run.
+def delta_table(metric):
+    base = metrics[metrics["method"] == "no guidance"].set_index(["percentage", "run"])[metric]
+    out = {}
+    for m in GUIDED:
+        gm = metrics[metrics["method"] == m].set_index(["percentage", "run"])[metric]
+        d = (gm - base).dropna()
+        out[m] = d
+    return out
+
+fig, (axA, axB) = plt.subplots(1, 2, figsize=(14, 6), sharex=True)
+delta_rows = []
+for metric, ax, lab in [("test_accuracy", axA, "Δ test accuracy"),
+                        ("test_f1_macro", axB, "Δ test macro-F1")]:
+    dt = delta_table(metric)
+    for m in GUIDED:
+        d = dt[m]
+        xs, ms, cs = [], [], []
+        for lvl in LEVELS:
+            vals = d.xs(lvl, level="percentage").values if lvl in d.index.get_level_values(0) else []
+            mm, cc = mean_ci(vals)
+            xs.append(lvl); ms.append(mm); cs.append(cc)
+            delta_rows.append({"metric": metric, "guided_method": m, "level": lvl,
+                               "delta_mean": mm, "delta_ci95": cc})
+        _line(ax, xs, ms, cs, m, COLOURS[m])
+    ax.axhline(0, color="k", lw=1, ls="--")
+    ax.set_xlabel("% of training labels polluted"); ax.set_ylabel(lab)
+    ax.set_title(f"{lab}  (guided − no guidance)", weight="bold")
+    ax.set_xticks(LEVELS); ax.grid(alpha=.3)
+axA.legend(title="guided method")
+fig.suptitle(f"Paired guided − unguided (λ={LAM}, 50 repeats, runtime OFF). "
+             f"Above 0 = guidance helps.", weight="bold")
+plt.tight_layout(); plt.savefig(OUTDIR / "delta_guided_minus_unguided.png", dpi=150); plt.close()
+pd.DataFrame(delta_rows).to_csv(OUTDIR / "delta_table.csv", index=False)
+print("wrote delta_guided_minus_unguided.png + delta_table.csv")
+
+# ── regime_summary.csv ────────────────────────────────────────────────────────────
+reg = summary[["method", "percentage", "train_accuracy_mean", "test_accuracy_mean",
+               "test_f1_macro_mean", "test_f1_weighted_mean", "invalid_rate_mean",
+               "abstain_rate_mean"]].copy()
+reg = reg.merge(total_rules_tbl.rename(columns={"level": "percentage"}),
+                on=["method", "percentage"], how="left")
+reg = reg.merge(rare_rules_tbl.rename(columns={"level": "percentage"}),
+                on=["method", "percentage"], how="left")
+reg = reg.sort_values(["method", "percentage"]).round(4)
+reg.to_csv(OUTDIR / "regime_summary.csv", index=False)
+print("wrote regime_summary.csv")
+print("\nAll rule-dynamics outputs in", OUTDIR)

diagnostics_extreme_and_combined.py

@@ -0,0 +1,113 @@
+"""Scratch: (1) extreme-lambda effect on onto-MDL vs onto-grow, (2) combined grow+prune+MDL
+variant vs all others. ONE repeat (5 folds) pooled per point — quick look, not a full sweep."""
+import numpy as np, pandas as pd
+from collections import defaultdict
+from sklearn.model_selection import RepeatedStratifiedKFold
+from sklearn.metrics import f1_score
+import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt
+from pathlib import Path
+
+from run_all_experiments import (load_clean_dummies, make_fold_dfs, ontology_invalid_rate,
+                                 N_FOLDS, SPLIT_SEED)
+from oud_maak_verv_dataset import pollute_train_labels_my_version
+from evaluate_ruleset import evaluate_ruleset
+from RIPPER_no_onto_guid import run_RIPPER_without_guidance
+from RIPPER_onto_guid import run_RIPPER_with_guidance
+from RIPPER_onto_guid_grow import run_RIPPER_with_guidance_in_grow
+from RIPPER_onto_guid_grow_and_prune import run_RIPPER_with_guidance_in_grow_and_prune
+from RIPPER_onto_guid_grow_prune_mdl import run_RIPPER_with_guidance_in_grow_prune_mdl
+
+X, y, all_classes = load_clean_dummies()
+SPLITS = list(enumerate(RepeatedStratifiedKFold(n_splits=N_FOLDS, n_repeats=1,
+                                                random_state=SPLIT_SEED).split(X, y), start=1))
+FIGDIR = Path("results/figures/diagnostics"); FIGDIR.mkdir(parents=True, exist_ok=True)
+RARE = "Nieuwe TFM Coupler"
+
+
+def pooled_run(fit_fn, kw, level):
+    """5 folds polluted at `level`; pool OOF preds -> one macro-F1/acc. Runtime OFF."""
+    yt_all, yp_all, frames, totals = [], [], [], []
+    rc = defaultdict(list)
+    for run, (tri, tei) in SPLITS:
+        tr, te = make_fold_dfs(X, y, tri, tei)
+        tr, _ = pollute_train_labels_my_version(tr, level, run)
+        np.random.seed(run)
+        rs, sv = fit_fn(tr.copy(), **kw)
+        for c, rules in rs.items():
+            rc[c].append(len(rules))
+        totals.append(sum(len(v) for v in rs.values()))
+        res, _ = evaluate_ruleset(rs, te.copy(), sv, False)
+        yt_all.append(res["y"].astype(str)); yp_all.append(res["y_pred"].astype(str))
+        frames.append(res)
+    yt = pd.concat(yt_all).values; yp = pd.concat(yp_all).values
+    return {"f1_macro": f1_score(yt, yp, average="macro", zero_division=0),
+            "accuracy": float((yt == yp).mean()),
+            "invalid_rate": ontology_invalid_rate(pd.concat(frames)),
+            "total_rules": float(np.mean(totals)),
+            "rare_rules": float(np.mean(rc.get(RARE, [0]))),
+            "rule_counts": {c: float(np.mean(v)) for c, v in rc.items()}}
+
+
+# ============ EXPERIMENT 1: extreme lambda, MDL vs grow ============
+print("=" * 92)
+print("EXPERIMENT 1 — extreme lambda for onto-MDL and onto-grow (5-fold pooled, runtime OFF)")
+print("=" * 92)
+E1_LEVELS = [0, 50, 100]
+E1_LAMBDAS = [4, 100, 100000]
+E1_METHODS = [("onto MDL", run_RIPPER_with_guidance),
+              ("onto grow", run_RIPPER_with_guidance_in_grow)]
+print(f"{'method':>10} {'lambda':>8} {'poll':>5} {'macroF1':>8} {'acc':>7} "
+      f"{'rules':>6} {'TFMrul':>7} {'invalid%':>9}")
+e1 = []
+for name, fn in E1_METHODS:
+    for lam in E1_LAMBDAS:
+        for lvl in E1_LEVELS:
+            r = pooled_run(fn, {"lambda_val": lam}, lvl)
+            e1.append({"method": name, "lambda": lam, "level": lvl, **r})
+            print(f"{name:>10} {lam:>8} {lvl:>5} {r['f1_macro']:>8.3f} {r['accuracy']:>7.3f} "
+                  f"{r['total_rules']:>6.1f} {r['rare_rules']:>7.1f} {r['invalid_rate']:>8.1%}")
+    print("-" * 92)
+
+# ============ EXPERIMENT 2: combined variant vs all, lambda=4, one run, full sweep ============
+print("\n" + "=" * 92)
+print("EXPERIMENT 2 — combined grow+prune+MDL vs all variants (lambda=4, 5-fold pooled, runtime OFF)")
+print("=" * 92)
+E2_LEVELS = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
+E2_METHODS = [("no guidance",      run_RIPPER_without_guidance,                {}),
+              ("onto MDL",         run_RIPPER_with_guidance,                   {"lambda_val": 4}),
+              ("onto grow",        run_RIPPER_with_guidance_in_grow,           {"lambda_val": 4}),
+              ("onto grow+prune",  run_RIPPER_with_guidance_in_grow_and_prune, {"lambda_val": 4}),
+              ("grow+prune+MDL",   run_RIPPER_with_guidance_in_grow_prune_mdl, {"lambda_val": 4})]
+e2 = []
+for name, fn, kw in E2_METHODS:
+    for lvl in E2_LEVELS:
+        r = pooled_run(fn, kw, lvl)
+        e2.append({"method": name, "level": lvl, **r})
+e2df = pd.DataFrame(e2)
+
+for metric, lab in [("f1_macro", "MACRO-F1"), ("accuracy", "ACCURACY"),
+                    ("total_rules", "TOTAL RULES"), ("rare_rules", f"'{RARE}' RULES")]:
+    print(f"\n{lab} vs pollution (rows=method):")
+    piv = e2df.pivot(index="method", columns="level", values=metric).reindex([m[0] for m in E2_METHODS])
+    fmt = "{:>6.3f}" if metric in ("f1_macro", "accuracy") else "{:>6.1f}"
+    hdr = "".join(f"{l:>7}" for l in E2_LEVELS)
+    print(f"{'method':>16}{hdr}")
+    for m in piv.index:
+        print(f"{m:>16}" + "".join(fmt.format(v) for v in piv.loc[m].values))
+
+# figures
+for metric, lab in [("f1_macro", "Pooled macro-F1"), ("accuracy", "Pooled accuracy"),
+                    ("total_rules", "Total rules learned")]:
+    plt.figure(figsize=(9, 6))
+    for name, *_ in E2_METHODS:
+        s = e2df[e2df["method"] == name]
+        plt.plot(s["level"], s[metric], marker="o", label=name)
+    plt.xlabel("% of training labels polluted"); plt.ylabel(lab)
+    plt.title(f"{lab} vs pollution — incl. combined grow+prune+MDL (λ=4, 1 repeat/5 folds, runtime OFF)")
+    plt.legend(); plt.grid(alpha=.3); plt.tight_layout()
+    out = FIGDIR / f"combined_variant_{metric}.png"; plt.savefig(out, dpi=140); plt.close()
+    print(f"\nwrote {out}")
+
+pd.DataFrame(e1).to_csv("results/diagnostics/extreme_lambda.csv", index=False)
+e2df.drop(columns=["rule_counts"]).to_csv("results/diagnostics/combined_variant.csv", index=False)
+print("\nwrote results/diagnostics/extreme_lambda.csv and combined_variant.csv")

evaluate_ruleset.py

@@ -0,0 +1,129 @@
+"""
+Evaluate a learned RIPPER ruleset against a dataset.
+
+Ruleset format:  {class_label: [[cond1, cond2, ...], [cond3, ...], ...], ...}
+Conditions:      'col_name'  → col == True
+                 '!col_name' → col == False (negated condition)
+
+The last entry in sorted_values is the default class — predicted when no rule fires.
+"""
+
+import pandas as pd
+
+# Sentinel returned by runtime ontology guidance when no ontologically-valid action exists
+# for the row's asset. It is never a true label, so it always counts as WRONG in accuracy/F1,
+# and it is a plain string (not None) so sklearn's f1_score never chokes on mixed None/str
+# labels. Tracked separately via the abstain_rate metric in run_all_experiments.
+ABSTAIN = "Geen advies mogelijk."
+
+
+def _row_matches_condition(row: pd.Series, cond: str) -> bool:
+    """Return True if the row satisfies one condition."""
+    if cond.startswith('!'):
+        return row[cond[1:]] == False
+    return row[cond] == True
+
+
+def _row_matches_rule(row: pd.Series, rule: list) -> bool:
+    """Return True if the row satisfies ALL conditions in a rule (AND logic)."""
+    return all(_row_matches_condition(row, cond) for cond in rule)
+
+
+def _predict_one(row: pd.Series, rule_set: dict, sorted_values: list) -> str:
+    """
+    Predict the class for a single row.
+
+    Checks each class in sorted_values order (rarest first, same as training).
+    The first class whose rules fire wins. If nothing fires, return the default
+    class (sorted_values[-1], the most common one).
+    """
+    for class_label in sorted_values[:-1]:
+        for rule in rule_set.get(class_label, []):
+            if rule and _row_matches_rule(row, rule):
+                return class_label
+    return sorted_values[-1]  # default class: no rule fired
+
+from ontology_guidance.get_onto_guidance_actions_per_asset import get_ontology_info_actions_per_asset
+
+
+def _row_asset(row: pd.Series) -> str | None:
+    """Return the asset value of a one-hot row: the assetType_* column that is True.
+
+    Mirrors the column convention used in build_invalid_row_mask (assetType_<value>).
+    Returns None if no assetType column is set.
+    """
+    for col in row.index:
+        if col.startswith('assetType_') and row[col] == True:
+            return col.replace('assetType_', '')
+    return None
+
+
+def _predict_one_runtime_onto_guidance(row: pd.Series, rule_set: dict, sorted_values: list) -> str:
+    """
+    Predict the class for a single row, with inference-time ontology guidance.
+
+    Checks each class in sorted_values order (rarest first, same as training).
+    The first class whose rules fire wins. Then it checks whether that action is
+    ontologically valid for the row's asset. If not, it falls back to the most
+    common action that IS valid for the asset.
+    """
+    asset = _row_asset(row)
+    valid_actions = get_ontology_info_actions_per_asset(asset) if asset is not None else []
+    valid_norm = {a.strip().lower() for a in valid_actions}
+
+    def _most_common_valid() -> str:
+        # walk most-common-first; return the first action valid for this asset
+        for item in reversed(sorted_values):
+            if item.strip().lower() in valid_norm:
+                return item
+        return ABSTAIN
+
+    for class_label in sorted_values[:-1]:
+        for rule in rule_set.get(class_label, []):
+            if rule and _row_matches_rule(row, rule):
+                if valid_norm and class_label.strip().lower() not in valid_norm:
+                    return _most_common_valid()
+                return class_label
+
+    # no rule fired -> default class, but prefer one valid for the asset
+    if valid_norm:
+        return _most_common_valid()
+    return ABSTAIN
+
+
+def evaluate_ruleset(
+    rule_set: dict,
+    data: pd.DataFrame,
+    sorted_values: list,
+    runtime_ontology_guidance_bool: bool
+) -> tuple[pd.DataFrame, float]:
+    """
+    Evaluate a ruleset against labelled data.
+
+    Args:
+        rule_set:      {class_label: [[cond, ...], ...], ...}
+        data:          DataFrame with feature columns plus a 'y' column (true labels)
+        sorted_values: class labels sorted by frequency ascending;
+                       sorted_values[-1] is the default (most common) class
+        runtime_ontology_guidance_bool: if true will evaluate using inference time ontology guidance.
+
+    Returns:
+        result_df: copy of data with an extra 'y_pred' column
+        accuracy:  fraction of rows where y_pred == y  (0.0 – 1.0)
+    """
+    result_df = data.copy()
+
+    if not runtime_ontology_guidance_bool:
+        result_df['y_pred'] = result_df.apply(
+            lambda row: _predict_one(row, rule_set, sorted_values),
+            axis=1,
+        )
+    else:
+        result_df['y_pred'] = result_df.apply(
+            lambda row: _predict_one_runtime_onto_guidance(row, rule_set, sorted_values),
+            axis=1,
+        )
+
+    accuracy = (result_df['y_pred'] == result_df['y']).mean()
+    
+    return result_df, accuracy

final_compare_accuracies_no_vervuiling.py

@@ -0,0 +1,76 @@
+"""
+Baseline helpers used by run_all_experiments' "comparing to prof" experiment:
+a scikit-learn wrapper around wittgenstein RIPPER, and ARFF writing + the Weka
+JAR path for the Weka JRip baseline.
+"""
+
+import os
+import re
+from pathlib import Path
+
+import numpy as np
+import wittgenstein as lw
+from sklearn.base import BaseEstimator, ClassifierMixin
+
+# Path to weka.jar (override with the WEKA_JAR env var). If missing, Weka is skipped.
+WEKA_JAR = os.environ.get("WEKA_JAR", str(Path(__file__).parent / "weka.jar"))
+
+
+# ── Wittgenstein RIPPER (one-vs-rest) ─────────────────────────────────────────
+class _SklearnRIPPER(BaseEstimator, ClassifierMixin):
+    """sklearn-compatible wrapper around wittgenstein RIPPER.
+
+    Wittgenstein's RIPPER is binary, so callers wrap it in OneVsRestClassifier.
+    pos_class=1 tells it which label is "positive" once sklearn binarizes y.
+    """
+
+    def __init__(self, random_state=42, verbosity=0):
+        self.random_state = random_state
+        self.verbosity = verbosity
+
+    def fit(self, X, y):
+        self._ripper = lw.RIPPER(random_state=self.random_state, verbosity=self.verbosity)
+        self._ripper.fit(X, y, pos_class=1)
+        return self
+
+    def predict(self, X):
+        return self._ripper.predict(X)
+
+    def predict_proba(self, X):
+        preds = np.array(self._ripper.predict(X), dtype=int)
+        return np.column_stack([1 - preds, preds])
+
+
+# ── Weka JRip ARFF I/O ────────────────────────────────────────────────────────
+def _sanitize_attr_name(s):
+    """ARFF attribute names are safer without special chars. Map to [A-Za-z0-9_]."""
+    return re.sub(r"[^A-Za-z0-9_]", "_", s)
+
+
+def _arff_quote_value(s):
+    """Quote a nominal value for ARFF if it contains whitespace or special chars."""
+    s = str(s)
+    if not s or any(c in s for c in " ,{}'\"%\t\n"):
+        return "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'"
+    return s
+
+
+def _write_arff(df, path, all_classes):
+    """Write a one-hot DataFrame (bool features + 'y') to an ARFF file.
+
+    all_classes is the full class list from the whole dataset, declared in BOTH
+    train and test ARFFs so Weka sees identical @attribute headers even if a
+    rare class is absent from one fold.
+    """
+    feature_cols = [c for c in df.columns if c != "y"]
+    class_decl = ",".join(_arff_quote_value(c) for c in all_classes)
+    with open(path, "w", encoding="utf-8", newline="\n") as f:
+        f.write("@relation jrip_compare\n\n")
+        for col in feature_cols:
+            f.write(f"@attribute {_sanitize_attr_name(col)} {{False,True}}\n")
+        f.write(f"@attribute y {{{class_decl}}}\n\n")
+        f.write("@data\n")
+        for _, row in df.iterrows():
+            vals = ["True" if bool(row[c]) else "False" for c in feature_cols]
+            vals.append(_arff_quote_value(row["y"]))
+            f.write(",".join(vals) + "\n")

helper_functions/build_invalid_row_mask.py

@@ -0,0 +1,34 @@
+"""Per-action ontology mask: True for rows whose assetType is invalid for the action."""
+
+import pandas as pd
+
+from ontology_guidance.get_onto_guidance_assets_per_action import get_ontology_info_assets_per_action
+
+
+def build_invalid_row_mask(action_type, data):
+    """Return a boolean Series over data.index, True where the row's assetType
+    is NOT in the valid set for action_type. Returns None when no penalty
+    applies (unknown action, no assetType columns, or every asset is valid).
+
+    get_ontology_info_assets_per_action returns None for an action that is unknown
+    to the ontology, which we treat as "no guidance signal" and skip."""
+    valid_assets = get_ontology_info_assets_per_action(action_type)
+    if valid_assets is None:
+        return None  # action unknown to the ontology -> no guidance signal
+
+    asset_cols = [col for col in data.columns if col.startswith('assetType')]
+    if not asset_cols:
+        return None
+
+    # strip both sides: some asset columns carry a leading space (e.g.
+    # 'assetType_ Bambu Lab P1S Combo') that the dict keys don't, which would
+    # otherwise mis-flag a valid asset as a violation.
+    valid_norm = {a.strip() for a in valid_assets}
+    invalid_cols = [
+        col for col in asset_cols
+        if col.replace('assetType_', '').strip() not in valid_norm
+    ]
+    if not invalid_cols:
+        return None
+
+    return data[invalid_cols].any(axis=1)

ontology_guidance/get_onto_guidance_actions_per_asset.py

@@ -0,0 +1,38 @@
+"""
+Static actions-per-asset lookup (the inverse of assets-per-action).
+
+Served from a hardcoded dict that is the inverse of ASSETS_PER_ACTION in
+get_onto_guidance_assets_per_action. If the ontology changes, update
+ASSETS_PER_ACTION (the single source of truth); this module inverts it
+automatically at import time.
+"""
+
+# --- invert ASSETS_PER_ACTION (single source of truth) ---
+try:
+    from ontology_guidance.get_onto_guidance_assets_per_action import ASSETS_PER_ACTION
+except ImportError:  # when run as a script from inside the ontology_guidance/ dir
+    from get_onto_guidance_assets_per_action import ASSETS_PER_ACTION
+
+
+def _normalize(name: str) -> str:
+    """Normalize an asset/action string for matching (strip + lowercase)."""
+    return name.strip().lower()
+
+
+# normalized asset name -> list of valid action names (as they appear in ASSETS_PER_ACTION)
+ACTIONS_PER_ASSET: dict[str, list[str]] = {}
+for _action, _assets in ASSETS_PER_ACTION.items():
+    for _asset in _assets:
+        _key = _normalize(_asset)
+        ACTIONS_PER_ASSET.setdefault(_key, [])
+        if _action not in ACTIONS_PER_ASSET[_key]:
+            ACTIONS_PER_ASSET[_key].append(_action)
+
+
+def get_ontology_info_actions_per_asset(asset_search_name: str) -> list[str]:
+    """Return the actions valid for an asset name.
+
+    Case-insensitive / whitespace-insensitive match against the inverted
+    ASSETS_PER_ACTION mapping. Returns [] if the asset is unknown.
+    """
+    return ACTIONS_PER_ASSET.get(_normalize(asset_search_name), [])

ontology_guidance/get_onto_guidance_assets_per_action.py

@@ -0,0 +1,30 @@
+"""
+Static assets-per-action lookup: the valid asset types for each action, used as
+the ontology's single source of truth. Case-insensitive prefix match on the
+action name.
+"""
+
+ASSETS_PER_ACTION = {'Feeder schoongemaakt': ["BAMBU LAB P1S", "BAMBU LAB X1 CARBON", "Bambu Lab P1S Combo", "UM2+ Connect"],
+ 'Nieuwe nozzle': ["BAMBU LAB P1S", "BAMBU LAB X1 CARBON", "UM2+ Connect"],
+ 'Vervangen Bambu Lab Hotend met geharde staal nozzle': ["BAMBU LAB P1S", "BAMBU LAB X1 CARBON", "Bambu Lab P1S Combo"],
+ 'Nieuwe TFM Coupler': ["BAMBU LAB P1S", "UM2+ Connect"],
+ 'Assen gesmeerd': ["Build Chamber", "BAMBU LAB P1S", "BAMBU LAB X1 CARBON", "UM2+ Connect"],
+ 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': ["BAMBU LAB P1S", "BAMBU LAB X1 CARBON", "Bambu Lab P1S Combo", "BigRep Studio G2"],
+ }
+
+
+def get_ontology_info_assets_per_action(action_search_name: str) -> list[str] | None:
+    """Return the valid asset types for an action name, or None if the action is
+    unknown to the ontology.
+
+    Case-insensitive prefix match on the action name. None (not []) signals
+    "unknown action" so callers can stay silent; build_invalid_row_mask treats
+    None as "no guidance signal".
+    """
+    needle = action_search_name.lower().strip()
+    for key, assets in ASSETS_PER_ACTION.items():
+        # case-insensitive on BOTH sides: keys here are Title-case, needle is lowered,
+        # so comparing raw would never match (this silently disabled all guidance).
+        if key.lower().startswith(needle):
+            return assets
+    return None

ontology_guidance/ontology_guidance.py

@@ -0,0 +1,68 @@
+"""
+Ontology-guided rule learning: compute violations for MDL-Bayes prior placement.
+
+Placement 3 (compute_ruleset_dl): penalty on description length
+- min total_DL ≡ max P(theory) · P(data | theory)
+- Adding λ · viol to the theory DL is equivalent to placing an ontology-informed prior
+- MDL ↔ Bayes correspondence (Rissanen, Grünwald 2007)
+
+Violation: when a rule for action type covers data with an invalid asset type.
+Use get_ontology_info_assets_per_action() to determine valid (action, asset) pairs.
+"""
+
+import pandas as pd
+
+from helper_functions.build_invalid_row_mask import build_invalid_row_mask
+
+
+def compute_rule_viol(rule, action_type, data, invalid_row_mask=None):
+    """
+    Count ontology violations for a single rule (list of conjuncts).
+    Returns the number of rows the rule covers whose assetType is invalid
+    for action_type. invalid_row_mask can be precomputed on a superset
+    (e.g. all_data) — it's sliced to data.index here.
+    """
+    if not rule:
+        return 0  # grow always starts non-empty; treat empty rule as 0 violations
+
+    if invalid_row_mask is None:
+        invalid_row_mask = build_invalid_row_mask(action_type, data)
+        if invalid_row_mask is None:
+            return 0
+    else:
+        # Align to the active subset — the mask may have been built on a superset.
+        invalid_row_mask = invalid_row_mask.loc[data.index]
+
+    mask = pd.Series(True, index=data.index)
+    for cond in rule:
+        if cond.startswith('!'):
+            mask &= data[cond[1:]] == False
+        else:
+            mask &= data[cond] == True
+    return int((mask & invalid_row_mask).sum())
+
+
+def compute_ontology_viol(rules, action_type, data):
+    """
+    Count ontology violations: rows covered by rules that have invalid (asset, action) pairs.
+
+    Vectorized: builds one boolean mask per rule, intersects with an
+    "invalid-asset" row mask precomputed once for the action type.
+    """
+    invalid_row_mask = build_invalid_row_mask(action_type, data)
+    if invalid_row_mask is None:
+        return 0
+
+    covered = pd.Series(False, index=data.index)
+    for rule in rules:
+        if not rule:
+            continue
+        mask = pd.Series(True, index=data.index)
+        for cond in rule:
+            if cond.startswith('!'):
+                mask &= data[cond[1:]] == False
+            else:
+                mask &= data[cond] == True
+        covered |= mask
+
+    return int((covered & invalid_row_mask).sum())

oud_maak_verv_dataset.py

@@ -0,0 +1,70 @@
+"""Label pollution for the contamination sweep.
+
+`pollute_train_labels_my_version` flips a fraction of training labels to actions
+that are ontologically INVALID for the row's asset, using the static
+actions-per-asset lookup below.
+"""
+
+import numpy as np
+import pandas as pd
+
+
+# Valid actions per asset type (ontology lookup). Used to decide which labels are
+# "invalid" for an asset when injecting contamination.
+ACTIONS_PER_ASSET = {
+    'Bambu Lab P1S Combo': ["Nieuwe feeder", "Feeder schoongemaakt", "Vervangen Bambu Lab Compleet hotend met geharde staal nozzle", "Vervangen Bambu Lab Hotend met geharde staal nozzle"],
+    'BAMBU LAB P1S': ['Nieuwe nozzle', 'Nieuwe TFM Coupler', 'Assen gesmeerd', 'Feeder gerepareerd', 'Nieuwe feeder', 'Feeder schoongemaakt', 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle', 'Vervangen Bambu Lab Hotend met geharde staal nozzle', 'Bambuplate vervangen'],
+    'BAMBU LAB X1 CARBON': ['Nieuwe nozzle', 'Assen gesmeerd', 'Feeder gerepareerd', 'Nieuwe Bowden tube', 'Feeder schoongemaakt', 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle', 'Vervangen Bambu Lab Hotend met geharde staal nozzle', 'Buildplate buildchamber vervangen', 'Vervangen ventilator'],
+    'BigRep Studio G2': ['Vervangen Bambu Lab Compleet hotend met geharde staal nozzle'],
+    'UM2+ Connect': ["Nieuwe nozzle", "Nieuwe TFM Coupler", "Assen gesmeerd", "Feeder gerepareerd", "Nieuwe feeder", "Nieuwe bandjes", "Nieuw Olsson blok", "Nieuwe Steelcoupler", "Naar leverancier gebracht", "Retour van leverancier", "Nieuwe heater", "Nieuwe Bowden tube", "Nieuw heaterblock", "Heater cables vervangen", "Nieuwe UM fan", "Reparatie uitgevoerd", "Bandjes UM gesteld", "Feeder schoongemaakt", "UM Sliding Block assembly 616 service", "Assen gesteld", "Bandjes gesteld", "Vervangen ventilator"],
+}
+
+
+def get_ontology_info_actions_per_asset(asset_search_name: str) -> list[str]:
+    """Return the actions that are ontologically valid for an asset name."""
+    return ACTIONS_PER_ASSET[asset_search_name.strip()]
+
+
+def _row_asset(row):
+    """Asset value of a one-hot row: the assetType_* column that is True (or None)."""
+    for col in row.index:
+        if col.startswith("assetType_") and row[col] == True:
+            return col.replace("assetType_", "")
+    return None
+
+
+def pollute_train_labels_my_version(train_df, percentage, seed):
+    train_df = train_df.copy()  # don't mutate caller's df
+    if percentage == 0:
+        return train_df, pd.Index([])
+
+    rng = np.random.default_rng(seed)
+    alle_assets = ['UM2+ Connect', 'BAMBU LAB X1 CARBON', 'BAMBU LAB P1S',
+                   'Bambu Lab P1S Combo', 'BigRep Studio G2']
+    dataset_actions = set(train_df['y'].unique())  # only actions that occur in the data (no always-impossible actions)
+
+    invalid_actions_per_asset = {}
+    for asset in alle_assets:
+        valid = set(get_ontology_info_actions_per_asset(asset))
+        # sorted(), NOT list(): set-difference iteration order is randomized per
+        # process (PYTHONHASHSEED), so list(...) made rng.choice pick a different
+        # invalid label every run even with a fixed seed -> non-reproducible
+        # pollution that swamped the guidance signal. sorted() pins the order.
+        invalid_actions_per_asset[asset] = sorted(dataset_actions - valid)
+
+    # only sample from rows whose asset has ≥1 invalid action available
+    eligible_mask = train_df.apply(
+        lambda r: len(invalid_actions_per_asset.get(_row_asset(r), [])) > 0,
+        axis=1
+    )
+    eligible_df = train_df[eligible_mask]
+
+    n_to_contaminate = int(len(eligible_df) * percentage / 100)
+    index_list = eligible_df.sample(n=n_to_contaminate, random_state=seed).index
+
+    for idx in index_list:
+        asset_type = _row_asset(train_df.loc[idx])
+        invalid_actions = invalid_actions_per_asset[asset_type]
+        train_df.at[idx, 'y'] = str(rng.choice(invalid_actions))
+
+    return train_df, index_list

results/diagnostics/combined_variant.csv

@@ -0,0 +1,56 @@
+method,level,f1_macro,accuracy,invalid_rate,total_rules,rare_rules
+no guidance,0,0.30962263012390256,0.6078767123287672,0.010273972602739725,4.2,1.2
+no guidance,10,0.3126831590500485,0.613013698630137,0.005136986301369863,3.8,1.0
+no guidance,20,0.37078815658022063,0.6335616438356164,0.003424657534246575,4.4,1.0
+no guidance,30,0.3180640159307459,0.6164383561643836,0.032534246575342464,4.0,1.0
+no guidance,40,0.2613806857477286,0.5702054794520548,0.1267123287671233,4.4,2.2
+no guidance,50,0.24313514076648168,0.4743150684931507,0.2791095890410959,5.0,1.4
+no guidance,60,0.18703028787948417,0.386986301369863,0.4280821917808219,5.8,1.2
+no guidance,70,0.06822070387131386,0.04280821917808219,0.8715753424657534,3.8,1.2
+no guidance,80,0.04659857824643377,0.03767123287671233,0.8715753424657534,4.0,1.2
+no guidance,90,0.04760665628079876,0.03767123287671233,0.8698630136986302,3.8,1.0
+no guidance,100,0.039857189224277834,0.0547945205479452,0.8698630136986302,4.8,1.0
+onto MDL,0,0.30962263012390256,0.6078767123287672,0.010273972602739725,4.2,1.2
+onto MDL,10,0.30963289231837193,0.6095890410958904,0.005136986301369863,4.0,1.0
+onto MDL,20,0.3349774783347925,0.6352739726027398,0.0017123287671232876,4.2,1.0
+onto MDL,30,0.32199956969757415,0.6335616438356164,0.0017123287671232876,3.2,0.4
+onto MDL,40,0.2079659029148431,0.559931506849315,0.00684931506849315,2.0,0.0
+onto MDL,50,0.2624251409965696,0.5376712328767124,0.17294520547945205,3.0,0.0
+onto MDL,60,0.20707008774593386,0.4537671232876712,0.3270547945205479,2.8,0.0
+onto MDL,70,0.10641253264813279,0.1780821917808219,0.6678082191780822,1.2,0.0
+onto MDL,80,0.06939100239555175,0.10273972602739725,0.7294520547945206,0.8,0.0
+onto MDL,90,0.06969176336115006,0.10273972602739725,0.7294520547945206,1.2,0.0
+onto MDL,100,0.058247725225011183,0.07705479452054795,0.7294520547945206,2.0,0.0
+onto grow,0,0.32484102800527287,0.6061643835616438,0.010273972602739725,4.8,1.2
+onto grow,10,0.3072810577008187,0.5976027397260274,0.0136986301369863,5.4,1.0
+onto grow,20,0.3883334298188416,0.6404109589041096,0.003424657534246575,5.2,1.0
+onto grow,30,0.3549201908209279,0.6215753424657534,0.003424657534246575,4.2,0.6
+onto grow,40,0.2794539816841854,0.5753424657534246,0.02226027397260274,3.0,0.8
+onto grow,50,0.3001460150612125,0.6164383561643836,0.04794520547945205,3.4,0.2
+onto grow,60,0.20706914632768694,0.4486301369863014,0.3253424657534247,3.0,0.0
+onto grow,70,0.10664931166169911,0.1832191780821918,0.6678082191780822,1.2,0.0
+onto grow,80,0.06939100239555175,0.10273972602739725,0.7294520547945206,0.8,0.0
+onto grow,90,0.06969176336115006,0.10273972602739725,0.7294520547945206,1.6,0.0
+onto grow,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.6,0.0
+onto grow+prune,0,0.32484102800527287,0.6061643835616438,0.010273972602739725,4.8,1.2
+onto grow+prune,10,0.3072810577008187,0.5976027397260274,0.0136986301369863,5.4,1.0
+onto grow+prune,20,0.3883334298188416,0.6404109589041096,0.003424657534246575,5.2,1.0
+onto grow+prune,30,0.3549201908209279,0.6215753424657534,0.003424657534246575,4.2,0.6
+onto grow+prune,40,0.2794539816841854,0.5753424657534246,0.02226027397260274,3.0,0.8
+onto grow+prune,50,0.3001460150612125,0.6164383561643836,0.04794520547945205,3.4,0.2
+onto grow+prune,60,0.20706914632768694,0.4486301369863014,0.3253424657534247,3.0,0.0
+onto grow+prune,70,0.10664931166169911,0.1832191780821918,0.6678082191780822,1.2,0.0
+onto grow+prune,80,0.06939100239555175,0.10273972602739725,0.7294520547945206,0.8,0.0
+onto grow+prune,90,0.06969176336115006,0.10273972602739725,0.7294520547945206,1.6,0.0
+onto grow+prune,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.6,0.0
+grow+prune+MDL,0,0.32484102800527287,0.6061643835616438,0.010273972602739725,4.8,1.2
+grow+prune+MDL,10,0.3072810577008187,0.5976027397260274,0.0136986301369863,5.4,1.0
+grow+prune+MDL,20,0.3886291209794814,0.6404109589041096,0.0017123287671232876,5.2,1.0
+grow+prune+MDL,30,0.34196885627902146,0.6198630136986302,0.0017123287671232876,4.0,0.4
+grow+prune+MDL,40,0.2602871282960079,0.577054794520548,0.02054794520547945,2.6,0.4
+grow+prune+MDL,50,0.3001460150612125,0.6164383561643836,0.04794520547945205,3.4,0.2
+grow+prune+MDL,60,0.20706914632768694,0.4486301369863014,0.3253424657534247,3.0,0.0
+grow+prune+MDL,70,0.10664931166169911,0.1832191780821918,0.6678082191780822,1.2,0.0
+grow+prune+MDL,80,0.06939100239555175,0.10273972602739725,0.7294520547945206,0.8,0.0
+grow+prune+MDL,90,0.06969176336115006,0.10273972602739725,0.7294520547945206,1.4,0.0
+grow+prune+MDL,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.4,0.0

results/diagnostics/extreme_lambda.csv

@@ -0,0 +1,19 @@
+method,lambda,level,f1_macro,accuracy,invalid_rate,total_rules,rare_rules,rule_counts
+onto MDL,4,0,0.30962263012390256,0.6078767123287672,0.010273972602739725,4.2,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.4, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.6, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto MDL,4,50,0.2624251409965696,0.5376712328767124,0.17294520547945205,3.0,0.0,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto MDL,4,100,0.058247725225011183,0.07705479452054795,0.7294520547945206,2.0,0.0,"{'Assen gesmeerd': 0.8, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 0.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"
+onto MDL,100,0,0.30962263012390256,0.6078767123287672,0.010273972602739725,4.2,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.4, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.6, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto MDL,100,50,0.2624251409965696,0.5376712328767124,0.17294520547945205,3.0,0.0,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto MDL,100,100,0.058247725225011183,0.07705479452054795,0.7294520547945206,2.0,0.0,"{'Assen gesmeerd': 0.8, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 0.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"
+onto MDL,100000,0,0.30962263012390256,0.6078767123287672,0.010273972602739725,4.2,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.4, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.6, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto MDL,100000,50,0.2624251409965696,0.5376712328767124,0.17294520547945205,3.0,0.0,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto MDL,100000,100,0.058247725225011183,0.07705479452054795,0.7294520547945206,2.0,0.0,"{'Assen gesmeerd': 0.8, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 0.6, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"
+onto grow,4,0,0.32484102800527287,0.6061643835616438,0.010273972602739725,4.8,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 1.0, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.6, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto grow,4,50,0.3001460150612125,0.6164383561643836,0.04794520547945205,3.4,0.2,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.8, 'Nieuwe TFM Coupler': 0.2, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto grow,4,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.6,0.0,"{'Assen gesmeerd': 1.0, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 1.0, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"
+onto grow,100,0,0.30272435245092205,0.6078767123287672,0.010273972602739725,4.8,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.8, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.8, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto grow,100,50,0.29273423146810695,0.6147260273972602,0.05650684931506849,3.2,0.0,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.8, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto grow,100,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.4,0.0,"{'Assen gesmeerd': 0.8, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 1.0, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"
+onto grow,100000,0,0.30272435245092205,0.6078767123287672,0.010273972602739725,4.8,1.2,"{'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.8, 'Assen gesmeerd': 0.4, 'Feeder schoongemaakt': 0.8, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 1.6, 'Nieuwe TFM Coupler': 1.2, 'Nieuwe nozzle': 0.0}"
+onto grow,100000,50,0.29273423146810695,0.6147260273972602,0.05650684931506849,3.2,0.0,"{'Assen gesmeerd': 0.2, 'Feeder schoongemaakt': 1.8, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.2, 'Nieuwe nozzle': 1.0}"
+onto grow,100000,100,0.07024175945917434,0.0821917808219178,0.7294520547945206,2.4,0.0,"{'Assen gesmeerd': 0.8, 'Nieuwe nozzle': 0.6, 'Feeder schoongemaakt': 1.0, 'Nieuwe TFM Coupler': 0.0, 'Vervangen Bambu Lab Compleet hotend met geharde staal nozzle': 0.0, 'Vervangen Bambu Lab Hotend met geharde staal nozzle': 0.0}"

results/diagnostics/lambda_sweep.csv

@@ -0,0 +1,141 @@
+method,percentage,run,lambda,total_rules,theory_dl,exception_bits,viol_count,viol_penalty,test_f1_macro
+onto MDL,30,1,0,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,1,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,4,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,16,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,256,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,4096,4,32.8,1554.9,0,0,0.3584755646583604
+onto MDL,30,1,100000,4,32.8,1554.9,0,0,0.3584755646583604
+onto grow,30,1,0,4,32.8,1554.9,0,0,0.3584755646583604
+onto grow,30,1,1,5,36.7,1548.3,0,0,0.35564602704621406
+onto grow,30,1,4,5,36.7,1548.3,0,0,0.35564602704621406
+onto grow,30,1,16,4,30.7,1558.9,2,32,0.31506962072050154
+onto grow,30,1,256,4,30.7,1558.9,2,512,0.31506962072050154
+onto grow,30,1,4096,4,30.7,1558.9,2,8192,0.31506962072050154
+onto grow,30,1,100000,4,30.7,1558.9,2,200000,0.31506962072050154
+onto MDL,30,2,0,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,1,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,4,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,16,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,256,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,4096,2,18.8,1585.6,0,0,0.25115995115995116
+onto MDL,30,2,100000,2,18.8,1585.6,0,0,0.25115995115995116
+onto grow,30,2,0,2,18.8,1585.6,0,0,0.25115995115995116
+onto grow,30,2,1,3,24.4,1574.4,2,2,0.3051442474800139
+onto grow,30,2,4,4,32.2,1569.6,0,0,0.35449106637561645
+onto grow,30,2,16,4,32.2,1569.6,0,0,0.35449106637561645
+onto grow,30,2,256,4,32.2,1569.6,0,0,0.35449106637561645
+onto grow,30,2,4096,4,32.2,1569.6,0,0,0.35449106637561645
+onto grow,30,2,100000,4,32.2,1569.6,0,0,0.35449106637561645
+onto MDL,30,3,0,6,56.1,1536.1,58,0,0.264654640764724
+onto MDL,30,3,1,4,37.1,1556.7,1,1,0.3069881648264415
+onto MDL,30,3,4,3,29.7,1557.8,0,0,0.28566066066066065
+onto MDL,30,3,16,4,37.5,1549.6,0,0,0.284297052154195
+onto MDL,30,3,256,3,29.7,1563.9,0,0,0.2063131313131313
+onto MDL,30,3,4096,3,29.7,1563.9,0,0,0.2063131313131313
+onto MDL,30,3,100000,3,29.7,1563.9,0,0,0.2063131313131313
+onto grow,30,3,0,6,56.1,1536.1,58,0,0.264654640764724
+onto grow,30,3,1,6,52.7,1535.3,1,1,0.3774246257642842
+onto grow,30,3,4,6,52.7,1535.3,1,4,0.3774246257642842
+onto grow,30,3,16,4,39.2,1557.7,0,0,0.28162037037037035
+onto grow,30,3,256,4,39.2,1557.7,0,0,0.28162037037037035
+onto grow,30,3,4096,4,39.2,1557.7,0,0,0.28162037037037035
+onto grow,30,3,100000,4,39.2,1557.7,0,0,0.28162037037037035
+onto MDL,30,4,0,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,1,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,4,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,16,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,256,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,4096,4,39.2,1562.7,0,0,0.3914504024310916
+onto MDL,30,4,100000,4,39.2,1562.7,0,0,0.3914504024310916
+onto grow,30,4,0,4,39.2,1562.7,0,0,0.3914504024310916
+onto grow,30,4,1,4,33.9,1566.0,0,0,0.39830624498816714
+onto grow,30,4,4,3,28.3,1576.1,0,0,0.31149301825993553
+onto grow,30,4,16,3,25.1,1576.1,0,0,0.31149301825993553
+onto grow,30,4,256,3,25.1,1576.1,0,0,0.31149301825993553
+onto grow,30,4,4096,3,25.1,1576.1,0,0,0.31149301825993553
+onto grow,30,4,100000,3,25.1,1576.1,0,0,0.31149301825993553
+onto MDL,30,5,0,4,34.3,1573.5,16,0,0.25187998124706984
+onto MDL,30,5,1,3,26.5,1581.5,0,0,0.2524074074074074
+onto MDL,30,5,4,3,26.5,1581.5,0,0,0.2524074074074074
+onto MDL,30,5,16,3,26.5,1581.5,0,0,0.2524074074074074
+onto MDL,30,5,256,3,26.5,1581.5,0,0,0.2524074074074074
+onto MDL,30,5,4096,3,26.5,1581.5,0,0,0.2524074074074074
+onto MDL,30,5,100000,3,26.5,1581.5,0,0,0.2524074074074074
+onto grow,30,5,0,4,34.3,1573.5,16,0,0.25187998124706984
+onto grow,30,5,1,3,26.5,1581.5,0,0,0.2524074074074074
+onto grow,30,5,4,3,26.5,1581.5,0,0,0.2524074074074074
+onto grow,30,5,16,2,18.8,1590.2,0,0,0.19825494205862304
+onto grow,30,5,256,2,18.8,1590.2,0,0,0.19825494205862304
+onto grow,30,5,4096,2,18.8,1590.2,0,0,0.19825494205862304
+onto grow,30,5,100000,2,18.8,1590.2,0,0,0.19825494205862304
+onto MDL,50,1,0,5,38.5,1540.1,69,0,0.27342995169082124
+onto MDL,50,1,1,4,34.3,1598.9,2,2,0.2897486946762309
+onto MDL,50,1,4,3,23.3,1609.1,0,0,0.24027940220922675
+onto MDL,50,1,16,3,23.3,1609.1,0,0,0.24027940220922675
+onto MDL,50,1,256,3,23.3,1609.1,0,0,0.24027940220922675
+onto MDL,50,1,4096,3,23.3,1609.1,0,0,0.24027940220922675
+onto MDL,50,1,100000,3,23.3,1609.1,0,0,0.24027940220922675
+onto grow,50,1,0,5,38.5,1540.1,69,0,0.27342995169082124
+onto grow,50,1,1,4,32.8,1599.5,0,0,0.27518661216712853
+onto grow,50,1,4,4,32.8,1599.5,0,0,0.27518661216712853
+onto grow,50,1,16,3,25.1,1621.3,0,0,0.24922360248447206
+onto grow,50,1,256,3,25.1,1621.3,0,0,0.24922360248447206
+onto grow,50,1,4096,3,25.1,1621.3,0,0,0.24922360248447206
+onto grow,50,1,100000,3,25.1,1621.3,0,0,0.24922360248447206
+onto MDL,50,2,0,3,27.6,1520.8,64,0,0.18121057620028105
+onto MDL,50,2,1,3,30.0,1556.0,0,0,0.22781469176818014
+onto MDL,50,2,4,2,18.8,1561.2,0,0,0.23402255639097747
+onto MDL,50,2,16,2,18.8,1561.2,0,0,0.23402255639097747
+onto MDL,50,2,256,2,18.8,1561.2,0,0,0.23402255639097747
+onto MDL,50,2,4096,2,18.8,1561.2,0,0,0.23402255639097747
+onto MDL,50,2,100000,2,18.8,1561.2,0,0,0.23402255639097747
+onto grow,50,2,0,3,27.6,1520.8,64,0,0.18121057620028105
+onto grow,50,2,1,2,18.8,1561.2,0,0,0.23402255639097747
+onto grow,50,2,4,2,17.3,1561.2,0,0,0.23402255639097747
+onto grow,50,2,16,2,17.3,1561.2,0,0,0.23402255639097747
+onto grow,50,2,256,2,17.3,1561.2,0,0,0.23402255639097747
+onto grow,50,2,4096,2,17.3,1561.2,0,0,0.23402255639097747
+onto grow,50,2,100000,2,17.3,1561.2,0,0,0.23402255639097747
+onto MDL,50,3,0,8,68.6,1476.4,215,0,0.13616600671714402
+onto MDL,50,3,1,6,54.8,1541.3,0,0,0.28809351933144733
+onto MDL,50,3,4,4,35.4,1546.9,0,0,0.3181476794922173
+onto MDL,50,3,16,4,35.4,1546.9,0,0,0.3181476794922173
+onto MDL,50,3,256,4,35.4,1546.9,0,0,0.3181476794922173
+onto MDL,50,3,4096,4,35.4,1546.9,0,0,0.3181476794922173
+onto MDL,50,3,100000,4,35.4,1546.9,0,0,0.3181476794922173
+onto grow,50,3,0,8,68.6,1476.4,215,0,0.13616600671714402
+onto grow,50,3,1,6,53.1,1541.3,0,0,0.28809351933144733
+onto grow,50,3,4,4,35.4,1546.9,0,0,0.3181476794922173
+onto grow,50,3,16,4,35.4,1546.9,0,0,0.3181476794922173
+onto grow,50,3,256,4,35.4,1546.9,0,0,0.3181476794922173
+onto grow,50,3,4096,4,35.4,1546.9,0,0,0.3181476794922173
+onto grow,50,3,100000,4,35.4,1546.9,0,0,0.3181476794922173
+onto MDL,50,4,0,6,53.6,1501.7,53,0,0.32713898567557104
+onto MDL,50,4,1,4,36.3,1541.0,0,0,0.3390746369469773
+onto MDL,50,4,4,4,39.0,1556.0,0,0,0.2846620205110771
+onto MDL,50,4,16,4,39.0,1556.0,0,0,0.2846620205110771
+onto MDL,50,4,256,4,39.0,1556.0,0,0,0.2846620205110771
+onto MDL,50,4,4096,4,39.0,1556.0,0,0,0.2846620205110771
+onto MDL,50,4,100000,4,39.0,1556.0,0,0,0.2846620205110771
+onto grow,50,4,0,6,53.6,1501.7,53,0,0.32713898567557104
+onto grow,50,4,1,5,48.5,1544.6,0,0,0.3390746369469773
+onto grow,50,4,4,5,44.1,1546.3,0,0,0.3390746369469773
+onto grow,50,4,16,5,44.1,1546.3,0,0,0.3390746369469773
+onto grow,50,4,256,5,44.1,1546.3,0,0,0.3390746369469773
+onto grow,50,4,4096,5,44.1,1546.3,0,0,0.3390746369469773
+onto grow,50,4,100000,5,44.1,1546.3,0,0,0.3390746369469773
+onto MDL,50,5,0,3,22.3,1527.4,64,0,0.19813218390804596
+onto MDL,50,5,1,2,16.6,1576.0,0,0,0.25964012041598244
+onto MDL,50,5,4,2,20.5,1596.1,0,0,0.18455387205387205
+onto MDL,50,5,16,2,20.5,1596.1,0,0,0.18455387205387205
+onto MDL,50,5,256,2,20.5,1596.1,0,0,0.18455387205387205
+onto MDL,50,5,4096,2,20.5,1596.1,0,0,0.18455387205387205
+onto MDL,50,5,100000,2,20.5,1596.1,0,0,0.18455387205387205
+onto grow,50,5,0,3,22.3,1527.4,64,0,0.19813218390804596
+onto grow,50,5,1,2,16.6,1576.0,0,0,0.25964012041598244
+onto grow,50,5,4,2,16.6,1576.0,0,0,0.25964012041598244
+onto grow,50,5,16,2,16.6,1576.0,0,0,0.25964012041598244
+onto grow,50,5,256,2,16.6,1576.0,0,0,0.25964012041598244
+onto grow,50,5,4096,2,16.6,1576.0,0,0,0.25964012041598244
+onto grow,50,5,100000,2,16.6,1576.0,0,0,0.25964012041598244

results/normality_tests.csv

@@ -0,0 +1,67 @@
+lam,pct,method,n,nonzero,ties_at_zero,shapiro_W,shapiro_p,normal,mean_delta,median_delta,wins,losses,wilcoxon_p,ttest_p
+3,0,onto MDL,50,1,49,0.12542840513280018,1.1274100233477779e-15,False,-0.0001295870836951507,0.0,0,1,0.31731050786291415,0.3222234059506754
+3,0,onto grow,50,36,14,0.9190109812662899,0.0021720275461818454,False,0.0017398493082464639,0.0,23,13,0.16204065786582023,0.3064195169870864
+3,0,onto grow and prune,50,36,14,0.9190109812662899,0.0021720275461818454,False,0.0017398493082464639,0.0,23,13,0.16204065786582023,0.3064195169870864
+3,10,onto MDL,50,14,36,0.6046078608424392,2.281258540806125e-10,False,-4.648061705179174e-05,0.0,6,8,0.5097974849608639,0.8937091813412903
+3,10,onto grow,50,46,4,0.9450382872196427,0.021420245465764674,False,-0.001981272188089278,0.0,24,22,0.5515541606343191,0.294827576774398
+3,10,onto grow and prune,50,48,2,0.9548408286557913,0.05410176146097083,True,-0.001475346302953623,0.0,24,24,0.7043224179898399,0.45615678289225964
+3,20,onto MDL,50,21,29,0.5282914635779512,1.9237950861397012e-11,False,-0.0011627377240614268,0.0,9,12,0.2172414463719159,0.08241031963892155
+3,20,onto grow,50,50,0,0.9455995132356043,0.02256862974186193,False,0.0011052334826056687,0.0018002362071088407,33,17,0.1329605561585705,0.4494359767076001
+3,20,onto grow and prune,50,49,1,0.9336799173017933,0.007635465713314887,False,0.0012721598668214297,0.0015531522288093347,34,15,0.07417386882006018,0.3764497321252334
+3,30,onto MDL,50,49,1,0.9694954741430384,0.2207537163839054,True,-0.004510154331653753,-0.0034386488718411157,20,29,0.03901168331216344,0.035066460143719266
+3,30,onto grow,50,50,0,0.9437749963631448,0.019052415571394768,False,0.011086076261276645,0.013257718956048575,34,16,0.00011903464815610221,0.0004571417837678272
+3,30,onto grow and prune,50,50,0,0.945828295512618,0.02305497967812218,False,0.010532830084721724,0.013257718956048575,33,17,0.00032407658280853013,0.0010112674839723873
+3,40,onto MDL,50,50,0,0.9731007517652605,0.30803960233894495,True,-0.012740973116078856,-0.010866608884789852,19,31,0.004178445711550793,0.0020335620534605303
+3,40,onto grow,50,50,0,0.9804710626289203,0.5719950824935351,True,0.026772516292561013,0.029129509946984294,44,6,1.9326371614170057e-09,4.5077417505455057e-10
+3,40,onto grow and prune,50,50,0,0.9831606348616115,0.6905693564242854,True,0.026738017881094117,0.029129509946984294,44,6,1.7568968502246207e-09,2.9731128269997035e-10
+3,50,onto MDL,50,50,0,0.9692017031175331,0.21473759952855842,True,0.013072220439863398,0.013154279432938051,31,19,0.00035222154756553437,9.391967129976528e-05
+3,50,onto grow,50,50,0,0.9933150297563677,0.9932010193784759,True,0.04411626697394975,0.04448053746303671,48,2,2.4868995751603507e-14,1.6494587247945278e-17
+3,50,onto grow and prune,50,50,0,0.9940090376793136,0.9965483065858859,True,0.042831734234216635,0.042684988055378556,48,2,2.4868995751603507e-14,6.700186398197848e-17
+3,60,onto MDL,50,50,0,0.926557320389235,0.004103364789925923,False,0.01041534392072354,0.011861401178130643,33,17,0.0014846473444478647,0.0015145861069850945
+3,60,onto grow,50,50,0,0.9514444539152882,0.039121888798718314,False,0.02099398097940716,0.018239149301192248,40,10,1.0490046520317264e-07,3.1959996451376927e-07
+3,60,onto grow and prune,50,50,0,0.945722586672719,0.022828920717122392,False,0.020407352766542406,0.016243578438240397,41,9,1.2141019567479816e-07,4.6628079318952006e-07
+3,70,onto MDL,50,50,0,0.9544136252863921,0.05193227594959449,True,0.011918814318185911,0.011015101933676295,40,10,0.0009871220775430345,0.0007638129060927107
+3,70,onto grow,50,50,0,0.9710988899160806,0.25639530092572443,True,0.01566546337181986,0.015195100409059498,39,11,0.00015581408102782746,0.00011265265206218932
+3,70,onto grow and prune,50,50,0,0.9723853871548077,0.2886306852963083,True,0.01559352343650226,0.015195100409059498,39,11,0.00015581408102782746,0.00011471289932271423
+3,80,onto MDL,50,50,0,0.9373924852806654,0.01063847117421645,False,0.009688529343792673,0.009546862427253,42,8,1.7496796758109667e-06,2.2210543545303727e-06
+3,80,onto grow,50,50,0,0.9576600805929033,0.07093747489867304,True,0.006517362999950538,0.007481497040786402,36,14,0.007380741603101981,0.013679997970944538
+3,80,onto grow and prune,50,50,0,0.9576797372466886,0.07107190790203703,True,0.0065162937983039675,0.007481497040786402,36,14,0.007380741603101981,0.013695704898264802
+3,90,onto MDL,50,50,0,0.9557928084267029,0.05927570420857092,True,0.0030821898275293826,0.005506781219287101,30,20,0.07885790741956278,0.1360444734107177
+3,90,onto grow,50,50,0,0.9349569815101737,0.00855297352369549,False,0.010181047095130139,0.012602066258826903,41,9,5.6804637029728156e-06,1.7813192173008554e-06
+3,90,onto grow and prune,50,50,0,0.9349825843296661,0.008572510115664149,False,0.010179981219149653,0.012575419359314752,41,9,5.6804637029728156e-06,1.7843226569567727e-06
+3,100,onto MDL,50,50,0,0.9304102253177837,0.0057271981601623246,False,-0.01102700273894665,-0.008625531891566951,12,38,1.1287300338835848e-07,2.61022674802467e-07
+3,100,onto grow,50,50,0,0.9167341243399015,0.0018004421617156957,False,0.0103395665697723,0.015003651208749798,40,10,2.6885878247284722e-05,1.977726248972888e-05
+3,100,onto grow and prune,50,50,0,0.9167341243399015,0.0018004421617156957,False,0.0103395665697723,0.015003651208749798,40,10,2.6885878247284722e-05,1.977726248972888e-05
+4,0,onto MDL,50,4,46,0.3114113271749194,6.009678302856076e-14,False,-0.00043811785463704567,0.0,1,3,0.4652088184521418,0.3231124071257798
+4,0,onto grow,50,41,9,0.9342382063953327,0.008023155217605295,False,0.0023516377291416136,0.0013064327815036358,28,13,0.061139832684513486,0.19443497786391611
+4,0,onto grow and prune,50,41,9,0.9342382063953327,0.008023155217605295,False,0.0023516377291416136,0.0013064327815036358,28,13,0.061139832684513486,0.19443497786391611
+4,10,onto MDL,50,14,36,0.6046078608424392,2.281258540806125e-10,False,-4.648061705179174e-05,0.0,6,8,0.5097974849608639,0.8937091813412903
+4,10,onto grow,50,47,3,0.9750486855380904,0.36651375942182585,True,-0.0035794931171793066,-0.0017510276534334346,19,28,0.08454696118090776,0.10082664734213999
+4,10,onto grow and prune,50,48,2,0.975295614280263,0.374525672276769,True,-0.0029274719133446448,-0.0024056202155169903,20,28,0.17907537170252064,0.19710240726771086
+4,20,onto MDL,50,25,25,0.5649380456304249,6.078505843571378e-11,False,-0.0028480708687979284,0.0,10,15,0.05108666110389699,0.01577335634728748
+4,20,onto grow,50,50,0,0.970088522745854,0.23337393373979182,True,0.0022991908032037258,0.0017653081129824222,30,20,0.10737600575610173,0.17237490700974653
+4,20,onto grow and prune,50,49,1,0.9619591335267971,0.10739430306125297,True,0.002464307340831073,0.0017171061964621481,31,18,0.0808543667009045,0.1403651310023357
+4,30,onto MDL,50,49,1,0.9955838420860846,0.9995986807582445,True,-0.0075901048152034875,-0.006827260209891356,15,34,0.003505585565059998,0.0023458099258936668
+4,30,onto grow,50,50,0,0.9442743846452957,0.019954056762947512,False,0.007931583424974721,0.007536904976736591,33,17,0.0017779629774352657,0.006656357336094241
+4,30,onto grow and prune,50,50,0,0.954967546121522,0.05476293915034592,True,0.007585019714868717,0.007536904976736591,31,19,0.004912360514200742,0.012785338260257167
+4,40,onto MDL,50,50,0,0.9853114344964572,0.7849115068116961,True,-0.013763186584818309,-0.011395775102262529,18,32,0.0010251160635235124,0.0005228166870663649
+4,40,onto grow,50,50,0,0.9806733987846118,0.5806930923150632,True,0.023495313603546935,0.024502399772517114,42,8,1.8673260271384606e-07,4.996333594850335e-08
+4,40,onto grow and prune,50,50,0,0.976644659989844,0.4206885534285245,True,0.023596832681638914,0.025239433060219058,42,8,1.0490046520317264e-07,2.1690100774112125e-08
+4,50,onto MDL,50,50,0,0.9683349344664781,0.19786263086445077,True,0.012377517572104053,0.01355281567401366,32,18,0.0002857006608838475,0.000195250913248652
+4,50,onto grow,50,50,0,0.9837788569747057,0.7181090795699334,True,0.04038307981851763,0.0443494965160955,48,2,9.769962616701378e-14,1.4763468981946096e-15
+4,50,onto grow and prune,50,50,0,0.9866570456403779,0.839838376622123,True,0.04002551265029013,0.042620351295797096,49,1,4.440892098500626e-14,1.1378238340990511e-15
+4,60,onto MDL,50,50,0,0.8859208679328525,0.00017007779518874529,False,0.011745808696861594,0.011863062404650779,35,15,0.0006715891126169993,0.0009754014355390173
+4,60,onto grow,50,50,0,0.9361403699393891,0.009506797642009982,False,0.0196379880678945,0.01804328758525095,37,13,1.1326234101716182e-06,2.5985704465769914e-06
+4,60,onto grow and prune,50,50,0,0.9247240491345063,0.0035086177751395696,False,0.017810165320359315,0.01545803605998726,37,13,4.268415494834699e-06,7.497444267506702e-06
+4,70,onto MDL,50,50,0,0.9555218321986979,0.05775352200217879,True,0.013313863339226016,0.012591439198086612,41,9,0.00021178724761483636,0.0002593627266312863
+4,70,onto grow,50,50,0,0.9736629249823842,0.3240611850081543,True,0.0153133189977645,0.015681912276117652,38,12,0.0001778852638985029,0.00019575378140047122
+4,70,onto grow and prune,50,50,0,0.9745039728212311,0.3493197556650465,True,0.015236213226084554,0.015681912276117652,38,12,0.0001858580190070569,0.00020093807436410708
+4,80,onto MDL,50,50,0,0.9384197499414436,0.011672184646874428,False,0.009538181984263065,0.009377850146964344,42,8,1.7496796758109667e-06,2.660665925772433e-06
+4,80,onto grow,50,50,0,0.9577614783625203,0.07163371445306561,True,0.006730964676699718,0.007589397112769354,36,14,0.006525205201144857,0.0112936671537753
+4,80,onto grow and prune,50,50,0,0.9577797877279594,0.07176017018485106,True,0.006729895475053149,0.007589397112769354,36,14,0.006525205201144857,0.01130688995101116
+4,90,onto MDL,50,50,0,0.915875523655408,0.0016783079468313173,False,0.009496456742616873,0.012883386359368247,39,11,1.4422642710343325e-05,5.977539287821946e-06
+4,90,onto grow,50,50,0,0.9434048121215772,0.018411528792881594,False,0.010334362648039557,0.012575419359314752,41,9,5.6804637029728156e-06,3.191439749443588e-06
+4,90,onto grow and prune,50,50,0,0.9434048121215772,0.018411528792881594,False,0.010334362648039557,0.012575419359314752,41,9,5.6804637029728156e-06,3.191439749443588e-06
+4,100,onto MDL,50,50,0,0.9480225222406085,0.02831181579857028,False,0.0062296988223423215,0.0070558094027418986,36,14,0.0021224348550905603,0.002394956527138198
+4,100,onto grow,50,50,0,0.9161674385901732,0.0017188199546184133,False,0.010478214840442153,0.015003651208749798,40,10,2.4281497463363166e-05,1.7476241983591324e-05
+4,100,onto grow and prune,50,50,0,0.9161674385901732,0.0017188199546184133,False,0.010478214840442153,0.015003651208749798,40,10,2.4281497463363166e-05,1.7476241983591324e-05

results/paired_tests.csv

@@ -0,0 +1,265 @@
+basis,lambda,percentage,runtime,method,n,mean_delta,median_delta,wins,losses,ties,wilcoxon_p,ttest_p
+per_fold,3,0,OFF,onto MDL,250,-5.377026074700519e-05,0.0,0,1,249,0.31731050786291415,0.3182813015158007
+per_fold,3,0,OFF,onto grow,250,0.0014722579049995653,0.0,28,31,191,0.25439281515853485,0.35182053868080887
+per_fold,3,0,OFF,onto grow and prune,250,0.0014722579049995653,0.0,28,31,191,0.25439281515853485,0.35182053868080887
+per_fold,3,0,ON,onto MDL,250,-5.377026074700519e-05,0.0,0,1,249,0.31731050786291415,0.3182813015158007
+per_fold,3,0,ON,onto grow,250,0.0015976415779585494,0.0,29,30,191,0.15810563123058052,0.27524464443595575
+per_fold,3,0,ON,onto grow and prune,250,0.0015976415779585494,0.0,29,30,191,0.15810563123058052,0.27524464443595575
+per_fold,3,10,OFF,onto MDL,250,-0.00011817501608798876,0.0,9,7,234,1.0,0.7643387501861244
+per_fold,3,10,OFF,onto grow,250,0.0005461347750003443,0.0,49,53,148,0.9960047910509009,0.7619466549547059
+per_fold,3,10,OFF,onto grow and prune,250,0.0008842532551416229,0.0,50,55,145,0.9172516262147151,0.6309376798356222
+per_fold,3,10,ON,onto MDL,250,0.00010296881637513055,0.0,7,6,237,0.700703412251682,0.7293777663553282
+per_fold,3,10,ON,onto grow,250,-0.00018457358078985853,0.0,49,50,151,0.8015815072022382,0.9041642466373225
+per_fold,3,10,ON,onto grow and prune,250,0.0001519069279441645,0.0,50,51,149,0.7132024028534253,0.9232969218319514
+per_fold,3,20,OFF,onto MDL,250,-0.0010566370329412992,0.0,11,13,226,0.265156633625956,0.08876729589283774
+per_fold,3,20,OFF,onto grow,250,0.0017598692243790748,0.0,73,60,117,0.24698168837758294,0.22987684360592034
+per_fold,3,20,OFF,onto grow and prune,250,0.00196817447022305,0.0,73,57,120,0.151985220026965,0.17680667467976305
+per_fold,3,20,ON,onto MDL,250,-0.0010554045880557985,0.0,9,12,229,0.1592244176320513,0.05794150649579572
+per_fold,3,20,ON,onto grow,250,0.0014691327436173486,0.0,68,62,120,0.4022048515312292,0.30951367523450585
+per_fold,3,20,ON,onto grow and prune,250,0.0016911694897708165,0.0,69,60,121,0.2687334545394233,0.238366447633107
+per_fold,3,30,OFF,onto MDL,250,-0.005664888504343146,0.0,71,63,116,0.16683345520561388,0.023110068337650166
+per_fold,3,30,OFF,onto grow,250,0.010182953756340794,0.0,123,75,52,4.301355236018728e-05,0.0002773462673864709
+per_fold,3,30,OFF,onto grow and prune,250,0.009487620080070038,0.0,121,74,55,0.00012059103735653603,0.000771232553284156
+per_fold,3,30,ON,onto MDL,250,-0.006183727539638348,0.0,56,65,129,0.026203272310949776,0.007281913849795919
+per_fold,3,30,ON,onto grow,250,0.009546698314792589,0.0,113,70,67,7.218313452241703e-05,0.0001779483356205263
+per_fold,3,30,ON,onto grow and prune,250,0.00886059280108122,0.0,111,69,70,0.00017299994437420027,0.0005743708898756874
+per_fold,3,40,OFF,onto MDL,250,-0.016055599998873984,-0.011934553745498871,90,156,4,4.1004690756374255e-05,3.096360844988355e-05
+per_fold,3,40,OFF,onto grow,250,0.020093312677394976,0.013651720683815455,165,82,3,3.721426821103375e-07,2.916555750025675e-07
+per_fold,3,40,OFF,onto grow and prune,250,0.019755806459966697,0.012033842584525792,162,85,3,6.767678338570368e-07,4.896156868059634e-07
+per_fold,3,40,ON,onto MDL,250,-0.020550580214514766,-0.0149329069355741,79,143,28,2.1587563102832287e-09,3.68193724825138e-10
+per_fold,3,40,ON,onto grow,250,0.014406451231823198,0.006539052228941744,134,80,36,8.636086407167024e-06,7.252418319416685e-06
+per_fold,3,40,ON,onto grow and prune,250,0.014239981133055095,0.006424287566229536,134,80,36,1.0540791592287125e-05,8.79657066588424e-06
+per_fold,3,50,OFF,onto MDL,250,0.0203790955632715,0.014824452666449953,148,102,0,1.9667374182824533e-05,5.359949467367103e-07
+per_fold,3,50,OFF,onto grow,250,0.04642625022783899,0.03857310982415327,203,47,0,6.12249169806947e-26,1.8037890140665754e-26
+per_fold,3,50,OFF,onto grow and prune,250,0.04528049615986987,0.03841346263194309,201,49,0,4.230650046307905e-25,1.6298818625020672e-25
+per_fold,3,50,ON,onto MDL,250,-0.027468627988017837,-0.017792988506870944,14,131,105,4.602423723947967e-21,2.2418407977114048e-27
+per_fold,3,50,ON,onto grow,250,-0.004098749616288456,0.0,67,83,100,0.07361572661925511,0.055246020091606514
+per_fold,3,50,ON,onto grow and prune,250,-0.005215410435933353,0.0,64,84,102,0.024986507643352654,0.01383273850385689
+per_fold,3,60,OFF,onto MDL,250,0.00938899334420429,0.017953201363127863,172,78,0,1.326276489141964e-07,0.00691500396622882
+per_fold,3,60,OFF,onto grow,250,0.01874836708535652,0.020979604831153496,179,71,0,3.831284868422745e-10,3.654626855731488e-07
+per_fold,3,60,OFF,onto grow and prune,250,0.018693252747892346,0.020979604831153496,180,70,0,1.1485380696677046e-10,4.008460207084489e-07
+per_fold,3,60,ON,onto MDL,250,-0.012393413284649544,0.0,17,66,167,9.44333959533226e-12,4.664918358474387e-12
+per_fold,3,60,ON,onto grow,250,-0.007966313007286072,0.0,47,77,126,1.676769061307449e-05,3.244504180046305e-05
+per_fold,3,60,ON,onto grow and prune,250,-0.008358306206212994,0.0,44,79,127,3.331360520443226e-06,1.2336253086111745e-05
+per_fold,3,70,OFF,onto MDL,250,0.0033730495477487113,0.0110042130415645,159,91,0,0.0110730245293221,0.20356708759301984
+per_fold,3,70,OFF,onto grow,250,0.004133008541803416,0.009488123361480347,159,91,0,0.011901631269592261,0.15221706493457082
+per_fold,3,70,OFF,onto grow and prune,250,0.004062506842707492,0.009488123361480347,159,91,0,0.011990411437532517,0.15784264294075984
+per_fold,3,70,ON,onto MDL,250,-0.005146973618393978,0.0,14,43,193,3.1920143229406396e-05,2.868232525155808e-05
+per_fold,3,70,ON,onto grow,250,-0.005518402039647002,0.0,23,54,173,3.782735548704649e-05,0.0008005573617968768
+per_fold,3,70,ON,onto grow and prune,250,-0.005598501903205511,0.0,23,54,173,2.5914808809268105e-05,0.0006572327076919223
+per_fold,3,80,OFF,onto MDL,250,-0.0028084500407277706,0.00012698412698414902,125,124,1,0.18896199557537752,0.0546449300335976
+per_fold,3,80,OFF,onto grow,250,-0.005428679243860594,-0.0016992902930402995,119,130,1,0.02975685973352608,0.006113265226975204
+per_fold,3,80,OFF,onto grow and prune,250,-0.0054296482361086565,-0.0016992902930402995,119,130,1,0.02975685973352608,0.006106346868899847
+per_fold,3,80,ON,onto MDL,250,-0.007313678346788829,0.0,22,61,167,1.9609627139231502e-07,8.928367250242921e-08
+per_fold,3,80,ON,onto grow,250,-0.010057418825469545,0.0,30,74,146,2.3488911676923338e-08,1.5043225405928071e-07
+per_fold,3,80,ON,onto grow and prune,250,-0.010065058894111458,0.0,29,74,147,2.1915715513994258e-08,1.4698912173709539e-07
+per_fold,3,90,OFF,onto MDL,250,-0.012346413492049443,-0.010134090728171848,92,155,3,3.5057725414091343e-11,1.0058030504588916e-13
+per_fold,3,90,OFF,onto grow,250,-0.006500702145745862,-0.004357761200902498,112,138,0,0.0002712732952366113,0.00012734475497469224
+per_fold,3,90,OFF,onto grow and prune,250,-0.006501671137993925,-0.004357761200902498,112,138,0,0.0002712732952366113,0.00012716338806370632
+per_fold,3,90,ON,onto MDL,250,-0.011377369588566659,0.0,18,79,153,1.734245188016631e-12,1.4803571184857002e-13
+per_fold,3,90,ON,onto grow,250,-0.010751911085641573,0.0,33,87,130,2.9573320001916945e-10,1.1609770820183683e-09
+per_fold,3,90,ON,onto grow and prune,250,-0.01075955115428349,0.0,32,87,131,2.7198079339808187e-10,1.127251764760443e-09
+per_fold,3,100,OFF,onto MDL,250,-0.02477708089483852,-0.018997129417895352,33,205,12,6.373058261349205e-34,6.841382652737208e-42
+per_fold,3,100,OFF,onto grow,250,-0.009537540803420261,-0.005790333033854151,100,150,0,4.3236923546420384e-07,5.7564148427455634e-09
+per_fold,3,100,OFF,onto grow and prune,250,-0.009537540803420261,-0.005790333033854151,100,150,0,4.3236923546420384e-07,5.7564148427455634e-09
+per_fold,3,100,ON,onto MDL,250,-0.018459929999151567,0.0,18,110,122,5.755317316821637e-19,4.8863466483653075e-22
+per_fold,3,100,ON,onto grow,250,-0.018898216628406152,0.0,22,113,115,8.343033485104265e-19,6.025318799435321e-22
+per_fold,3,100,ON,onto grow and prune,250,-0.018898216628406152,0.0,22,113,115,8.343033485104265e-19,6.025318799435321e-22
+per_fold,4,0,OFF,onto MDL,250,-0.00029131363490910567,0.0,1,3,246,0.4652088184521418,0.3853981469561415
+per_fold,4,0,OFF,onto grow,250,0.0015895224855168822,0.0,36,33,181,0.13576545380935756,0.3476851773173459
+per_fold,4,0,OFF,onto grow and prune,250,0.0015895224855168822,0.0,36,33,181,0.13576545380935756,0.3476851773173459
+per_fold,4,0,ON,onto MDL,250,-0.00029131363490910567,0.0,1,3,246,0.4652088184521418,0.3853981469561415
+per_fold,4,0,ON,onto grow,250,0.0017264600778698212,0.0,37,32,181,0.06958383193457679,0.26550053901169735
+per_fold,4,0,ON,onto grow and prune,250,0.0017264600778698212,0.0,37,32,181,0.06958383193457679,0.26550053901169735
+per_fold,4,10,OFF,onto MDL,250,-0.00011817501608798876,0.0,9,7,234,1.0,0.7643387501861244
+per_fold,4,10,OFF,onto grow,250,-0.0008804402794625979,0.0,47,65,138,0.3054556538913088,0.6533838260156347
+per_fold,4,10,OFF,onto grow and prune,250,-0.00038836981006982194,0.0,50,66,134,0.4132400813833613,0.8462500053835113
+per_fold,4,10,ON,onto MDL,250,0.00010296881637513055,0.0,7,6,237,0.700703412251682,0.7293777663553282
+per_fold,4,10,ON,onto grow,250,-0.0012174834524704998,0.0,49,61,140,0.41640306757969525,0.47939354521301514
+per_fold,4,10,ON,onto grow and prune,250,-0.0007282046442529001,0.0,52,61,137,0.5637596735947419,0.6807387260476376
+per_fold,4,20,OFF,onto MDL,250,-0.0024748774445604933,0.0,13,18,219,0.03430851926047363,0.007251500898002692
+per_fold,4,20,OFF,onto grow,250,0.0028272617699814182,0.0,77,64,109,0.1826644769831699,0.07886279552906036
+per_fold,4,20,OFF,onto grow and prune,250,0.003036077213476018,0.0,77,61,112,0.11358553433051558,0.05793014201383156
+per_fold,4,20,ON,onto MDL,250,-0.00248752116340762,0.0,10,17,223,0.014263819097464155,0.004875529471891427
+per_fold,4,20,ON,onto grow,250,0.0025839287863061227,0.0,74,64,112,0.21968857046487666,0.10537863794296395
+per_fold,4,20,ON,onto grow and prune,250,0.0028064757301102155,0.0,75,62,113,0.14313479336326607,0.07668832442022663
+per_fold,4,30,OFF,onto MDL,250,-0.008956975916196493,0.0,68,75,107,0.006131691552475097,0.0006383412507081468
+per_fold,4,30,OFF,onto grow,250,0.0069781777959364265,0.0,114,84,52,0.00403332731778627,0.01086545236295419
+per_fold,4,30,OFF,onto grow and prune,250,0.00636934489157347,0.0,115,81,54,0.0054260235335302,0.02552845649191158
+per_fold,4,30,ON,onto MDL,250,-0.00941977588704175,0.0,55,77,118,0.00042345043940878797,0.00010302082834916343
+per_fold,4,30,ON,onto grow,250,0.006282390812232728,0.0,104,80,66,0.008665924415176426,0.013519576868082233
+per_fold,4,30,ON,onto grow and prune,250,0.005639700955961672,0.0,104,77,69,0.011748904311739179,0.033038389555950835
+per_fold,4,40,OFF,onto MDL,250,-0.01617558416583157,-0.012426900584795356,86,161,3,1.8084385306996194e-05,2.251561389299542e-05
+per_fold,4,40,OFF,onto grow,250,0.017006443819950246,0.010683108885136045,158,89,3,2.2335568507036568e-05,1.530313750013389e-05
+per_fold,4,40,OFF,onto grow and prune,250,0.016813683855065972,0.010556114174535225,156,91,3,2.379558054003367e-05,1.7536496832681237e-05
+per_fold,4,40,ON,onto MDL,250,-0.020992451131433536,-0.01785559238453155,76,147,27,5.021021011423548e-10,1.423835805485597e-10
+per_fold,4,40,ON,onto grow,250,0.010850511488715341,0.003918925691749495,132,84,34,0.0007312515389441183,0.0009925627583199737
+per_fold,4,40,ON,onto grow and prune,250,0.010825514665631598,0.0036865249953433876,133,83,34,0.0005873386461694267,0.0008842687413714624
+per_fold,4,50,OFF,onto MDL,250,0.02046152095167934,0.014824452666449953,146,104,0,2.237183768301841e-05,5.149932196327111e-07
+per_fold,4,50,OFF,onto grow,250,0.04285981614721873,0.037751036818423844,197,53,0,5.087456624863223e-22,1.2393603476328574e-22
+per_fold,4,50,OFF,onto grow and prune,250,0.0424818689185778,0.037777507251384956,196,54,0,1.2707950103393253e-21,2.2181778035319043e-22
+per_fold,4,50,ON,onto MDL,250,-0.027733019496774602,-0.0195081777819425,15,134,101,4.290372071589858e-21,2.465628251373006e-27
+per_fold,4,50,ON,onto grow,250,-0.007827222809978784,0.0,64,92,94,0.001203847394359639,0.0008793817751635617
+per_fold,4,50,ON,onto grow and prune,250,-0.008290694315908403,0.0,60,92,98,0.0005148993145339456,0.0003939076570916405
+per_fold,4,60,OFF,onto MDL,250,0.011649152821565492,0.01835233800546055,175,75,0,1.6965665389727883e-08,0.0009958255533948498
+per_fold,4,60,OFF,onto grow,250,0.0169745240765574,0.021162247659270453,175,75,0,8.626730468242548e-09,1.086596820188632e-05
+per_fold,4,60,OFF,onto grow and prune,250,0.01589412690361924,0.020979604831153496,176,74,0,6.794621980320727e-09,3.240831318152777e-05
+per_fold,4,60,ON,onto MDL,250,-0.012263005761138571,0.0,18,66,166,1.8062505777499126e-11,9.285651915029584e-12
+per_fold,4,60,ON,onto grow,250,-0.007877038789641761,0.0,49,80,121,2.3861693039189053e-05,5.4143299619933315e-05
+per_fold,4,60,ON,onto grow and prune,250,-0.009057231247444156,0.0,42,86,122,3.875534510857909e-07,2.8797606706245638e-06
+per_fold,4,70,OFF,onto MDL,250,0.004091935922945117,0.011261928036121655,161,89,0,0.005989880237175888,0.12779400804201999
+per_fold,4,70,OFF,onto grow,250,0.003820556641680585,0.008803202062925848,158,92,0,0.015640888583882774,0.19517402104883044
+per_fold,4,70,OFF,onto grow and prune,250,0.0037489244516285894,0.008803202062925848,158,92,0,0.015716140704062086,0.20212441819060817
+per_fold,4,70,ON,onto MDL,250,-0.005263755227589381,0.0,13,43,194,1.717381290479516e-05,1.6946171950312563e-05
+per_fold,4,70,ON,onto grow,250,-0.005859720999299742,0.0,25,56,169,2.7562432289827024e-05,0.000695408482180429
+per_fold,4,70,ON,onto grow and prune,250,-0.005949248472285861,0.0,25,56,169,1.8119201194313228e-05,0.000562781112373203
+per_fold,4,80,OFF,onto MDL,250,-0.0022692314901096157,0.00040225392482574895,127,122,1,0.30437146280166005,0.12351735982711469
+per_fold,4,80,OFF,onto grow,250,-0.005217451842390732,-0.0016992902930402995,119,130,1,0.033957831807217106,0.009651317946920902
+per_fold,4,80,OFF,onto grow and prune,250,-0.005218420834638795,-0.0016992902930402995,119,130,1,0.033957831807217106,0.009640858797832635
+per_fold,4,80,ON,onto MDL,250,-0.007097509521587746,0.0,25,59,166,7.239778968956144e-07,2.3678825103770522e-07
+per_fold,4,80,ON,onto grow,250,-0.009747328825160334,0.0,36,76,138,1.1881443356453175e-07,3.6098020115029555e-07
+per_fold,4,80,ON,onto grow and prune,250,-0.009754968893802249,0.0,35,76,139,1.0739346881276082e-07,3.529885890482226e-07
+per_fold,4,90,OFF,onto MDL,250,-0.007080874563258173,-0.004185561666266054,113,137,0,0.00012018593658468034,6.264180452907702e-06
+per_fold,4,90,OFF,onto grow,250,-0.006299348301672776,-0.004328896376951302,113,137,0,0.0004447829334105335,0.00019944513163818914
+per_fold,4,90,OFF,onto grow and prune,250,-0.006299348301672776,-0.004328896376951302,113,137,0,0.0004447829334105335,0.00019944513163818914
+per_fold,4,90,ON,onto MDL,250,-0.011377369588566659,0.0,18,79,153,1.734245188016631e-12,1.4803571184857002e-13
+per_fold,4,90,ON,onto grow,250,-0.010581337336338308,0.0,32,86,132,7.829990226198197e-10,3.08209203806091e-09
+per_fold,4,90,ON,onto grow and prune,250,-0.010581337336338308,0.0,32,86,132,7.829990226198197e-10,3.08209203806091e-09
+per_fold,4,100,OFF,onto MDL,250,-0.012722906689013306,-0.00863274392686155,77,172,1,4.0758972448456836e-13,1.81605308485326e-15
+per_fold,4,100,OFF,onto grow,250,-0.009355033526292,-0.005790333033854151,101,149,0,7.755854061295561e-07,1.4290353954208176e-08
+per_fold,4,100,OFF,onto grow and prune,250,-0.009355033526292,-0.005790333033854151,101,149,0,7.755854061295561e-07,1.4290353954208176e-08
+per_fold,4,100,ON,onto MDL,250,-0.018459929999151567,0.0,18,110,122,5.755317316821637e-19,4.8863466483653075e-22
+per_fold,4,100,ON,onto grow,250,-0.018716243839290506,0.0,23,114,113,2.8142005476846898e-18,2.747653332243502e-21
+per_fold,4,100,ON,onto grow and prune,250,-0.018716243839290506,0.0,23,114,113,2.8142005476846898e-18,2.747653332243502e-21
+per_repeat_pooled,3,0,OFF,onto MDL,50,-0.0001295870836951507,0.0,0,1,49,0.31731050786291415,0.3222234059506754
+per_repeat_pooled,3,0,OFF,onto grow,50,0.0017398493082464639,0.0,23,13,14,0.16204065786582023,0.3064195169870864
+per_repeat_pooled,3,0,OFF,onto grow and prune,50,0.0017398493082464639,0.0,23,13,14,0.16204065786582023,0.3064195169870864
+per_repeat_pooled,3,0,ON,onto MDL,50,-0.00013053613053612967,0.0,0,1,49,0.31731050786291415,0.3222234059506754
+per_repeat_pooled,3,0,ON,onto grow,50,0.0018358054031406268,0.0,23,13,14,0.12752725562664105,0.2689164177914008
+per_repeat_pooled,3,0,ON,onto grow and prune,50,0.0018358054031406268,0.0,23,13,14,0.12752725562664105,0.2689164177914008
+per_repeat_pooled,3,10,OFF,onto MDL,50,-4.648061705179174e-05,0.0,6,8,36,0.5097974849608639,0.8937091813412903
+per_repeat_pooled,3,10,OFF,onto grow,50,-0.001981272188089278,0.0,24,22,4,0.5515541606343191,0.294827576774398
+per_repeat_pooled,3,10,OFF,onto grow and prune,50,-0.001475346302953623,0.0,24,24,2,0.7043224179898399,0.45615678289225964
+per_repeat_pooled,3,10,ON,onto MDL,50,5.6997989391277534e-05,0.0,5,7,38,0.6948866023724733,0.8441342604654636
+per_repeat_pooled,3,10,ON,onto grow,50,-0.002165639718553969,0.0,24,22,4,0.5442768914173914,0.2299780035573213
+per_repeat_pooled,3,10,ON,onto grow and prune,50,-0.0016586507371843739,0.0,24,23,3,0.7111026853091279,0.38165812007082583
+per_repeat_pooled,3,20,OFF,onto MDL,50,-0.0011627377240614268,0.0,9,12,29,0.2172414463719159,0.08241031963892155
+per_repeat_pooled,3,20,OFF,onto grow,50,0.0011052334826056687,0.0018002362071088407,33,17,0,0.1329605561585705,0.4494359767076001
+per_repeat_pooled,3,20,OFF,onto grow and prune,50,0.0012721598668214297,0.0015531522288093347,34,15,1,0.07417386882006018,0.3764497321252334
+per_repeat_pooled,3,20,ON,onto MDL,50,-0.0012341987835500212,0.0,8,11,31,0.09895736546518985,0.052595290604664974
+per_repeat_pooled,3,20,ON,onto grow,50,0.0007938685303164683,0.0016211236240571691,32,17,1,0.17770392319862494,0.581556728208207
+per_repeat_pooled,3,20,ON,onto grow and prune,50,0.0009697237462768693,0.0021016203914129283,33,16,1,0.09970668891391905,0.4938591617328542
+per_repeat_pooled,3,30,OFF,onto MDL,50,-0.004510154331653753,-0.0034386488718411157,20,29,1,0.03901168331216344,0.035066460143719266
+per_repeat_pooled,3,30,OFF,onto grow,50,0.011086076261276645,0.013257718956048575,34,16,0,0.00011903464815610221,0.0004571417837678272
+per_repeat_pooled,3,30,OFF,onto grow and prune,50,0.010532830084721724,0.013257718956048575,33,17,0,0.00032407658280853013,0.0010112674839723873
+per_repeat_pooled,3,30,ON,onto MDL,50,-0.006624068419697783,-0.005758280742865718,17,32,1,0.0036191049075792858,0.0016382733530787565
+per_repeat_pooled,3,30,ON,onto grow,50,0.008916837768849212,0.009681625762427537,32,18,0,0.0005294876867694143,0.0015062927686738533
+per_repeat_pooled,3,30,ON,onto grow and prune,50,0.008372503038698222,0.009681625762427537,32,18,0,0.0011471783973160399,0.0034720292588614815
+per_repeat_pooled,3,40,OFF,onto MDL,50,-0.012740973116078856,-0.010866608884789852,19,31,0,0.004178445711550793,0.0020335620534605303
+per_repeat_pooled,3,40,OFF,onto grow,50,0.026772516292561013,0.029129509946984294,44,6,0,1.9326371614170057e-09,4.5077417505455057e-10
+per_repeat_pooled,3,40,OFF,onto grow and prune,50,0.026738017881094117,0.029129509946984294,44,6,0,1.7568968502246207e-09,2.9731128269997035e-10
+per_repeat_pooled,3,40,ON,onto MDL,50,-0.021041660376192265,-0.0206539026541413,12,38,0,2.649707155910619e-07,2.1372872645675535e-07
+per_repeat_pooled,3,40,ON,onto grow,50,0.017705906171148034,0.017668546536830976,40,10,0,1.4543591948523726e-06,4.7227049760085636e-07
+per_repeat_pooled,3,40,ON,onto grow and prune,50,0.017748815120925607,0.017668546536830976,42,8,0,8.784112139892386e-07,2.1087954583240856e-07
+per_repeat_pooled,3,50,OFF,onto MDL,50,0.013072220439863398,0.013154279432938051,31,19,0,0.00035222154756553437,9.391967129976528e-05
+per_repeat_pooled,3,50,OFF,onto grow,50,0.04411626697394975,0.04448053746303671,48,2,0,2.4868995751603507e-14,1.6494587247945278e-17
+per_repeat_pooled,3,50,OFF,onto grow and prune,50,0.042831734234216635,0.042684988055378556,48,2,0,2.4868995751603507e-14,6.700186398197848e-17
+per_repeat_pooled,3,50,ON,onto MDL,50,-0.03906587178872413,-0.0374773247555511,0,50,0,1.7763568394002505e-15,9.614183717770261e-23
+per_repeat_pooled,3,50,ON,onto grow,50,-0.010569633344208311,-0.012855928390513999,14,36,0,0.0001858580190070569,0.0001012642003421509
+per_repeat_pooled,3,50,ON,onto grow and prune,50,-0.011798286263998162,-0.012855928390513999,13,37,0,4.6469730756726335e-05,2.6124685856270336e-05
+per_repeat_pooled,3,60,OFF,onto MDL,50,0.01041534392072354,0.011861401178130643,33,17,0,0.0014846473444478647,0.0015145861069850945
+per_repeat_pooled,3,60,OFF,onto grow,50,0.02099398097940716,0.018239149301192248,40,10,0,1.0490046520317264e-07,3.1959996451376927e-07
+per_repeat_pooled,3,60,OFF,onto grow and prune,50,0.020407352766542406,0.016243578438240397,41,9,0,1.2141019567479816e-07,4.6628079318952006e-07
+per_repeat_pooled,3,60,ON,onto MDL,50,-0.019928895094649072,-0.017117249285513142,3,40,7,2.1094341485655406e-08,2.0411736259796905e-10
+per_repeat_pooled,3,60,ON,onto grow,50,-0.012939848906750248,-0.013355553055874714,16,34,0,3.128163219123792e-05,1.4578170352227298e-05
+per_repeat_pooled,3,60,ON,onto grow and prune,50,-0.01416551004261128,-0.013389357539849794,15,35,0,7.946967700789287e-06,5.363316426507715e-06
+per_repeat_pooled,3,70,OFF,onto MDL,50,0.011918814318185911,0.011015101933676295,40,10,0,0.0009871220775430345,0.0007638129060927107
+per_repeat_pooled,3,70,OFF,onto grow,50,0.01566546337181986,0.015195100409059498,39,11,0,0.00015581408102782746,0.00011265265206218932
+per_repeat_pooled,3,70,OFF,onto grow and prune,50,0.01559352343650226,0.015195100409059498,39,11,0,0.00015581408102782746,0.00011471289932271423
+per_repeat_pooled,3,70,ON,onto MDL,50,-0.006354529639228133,-0.0031621610259552346,11,26,13,0.006769241117075083,0.005841161832638964
+per_repeat_pooled,3,70,ON,onto grow,50,-0.0054829043349067606,0.0,18,23,9,0.04527758677441353,0.02026796481850363
+per_repeat_pooled,3,70,ON,onto grow and prune,50,-0.005566063616472638,0.0,18,23,9,0.038747514649022465,0.017958757648131308
+per_repeat_pooled,3,80,OFF,onto MDL,50,0.009688529343792673,0.009546862427253,42,8,0,1.7496796758109667e-06,2.2210543545303727e-06
+per_repeat_pooled,3,80,OFF,onto grow,50,0.006517362999950538,0.007481497040786402,36,14,0,0.007380741603101981,0.013679997970944538
+per_repeat_pooled,3,80,OFF,onto grow and prune,50,0.0065162937983039675,0.007481497040786402,36,14,0,0.007380741603101981,0.013695704898264802
+per_repeat_pooled,3,80,ON,onto MDL,50,-0.002173045421855756,0.0,24,18,8,0.8073691126849867,0.3977452933509351
+per_repeat_pooled,3,80,ON,onto grow,50,-0.005463184013381753,0.0,21,24,5,0.12065304607118647,0.09172508946996338
+per_repeat_pooled,3,80,ON,onto grow and prune,50,-0.005470450932592677,0.0,21,24,5,0.12065304607118647,0.0912636198911035
+per_repeat_pooled,3,90,OFF,onto MDL,50,0.0030821898275293826,0.005506781219287101,30,20,0,0.07885790741956278,0.1360444734107177
+per_repeat_pooled,3,90,OFF,onto grow,50,0.010181047095130139,0.012602066258826903,41,9,0,5.6804637029728156e-06,1.7813192173008554e-06
+per_repeat_pooled,3,90,OFF,onto grow and prune,50,0.010179981219149653,0.012575419359314752,41,9,0,5.6804637029728156e-06,1.7843226569567727e-06
+per_repeat_pooled,3,90,ON,onto MDL,50,-0.0007637896954383555,0.0,23,23,4,0.8655266250286244,0.7440750617730048
+per_repeat_pooled,3,90,ON,onto grow,50,-8.631097610218186e-05,7.662338976430205e-05,25,23,2,0.9509297401292157,0.971270678122887
+per_repeat_pooled,3,90,ON,onto grow and prune,50,-9.349057189494969e-05,7.662338976430205e-05,25,23,2,0.9509297401292157,0.9689079574957785
+per_repeat_pooled,3,100,OFF,onto MDL,50,-0.01102700273894665,-0.008625531891566951,12,38,0,1.1287300338835848e-07,2.61022674802467e-07
+per_repeat_pooled,3,100,OFF,onto grow,50,0.0103395665697723,0.015003651208749798,40,10,0,2.6885878247284722e-05,1.977726248972888e-05
+per_repeat_pooled,3,100,OFF,onto grow and prune,50,0.0103395665697723,0.015003651208749798,40,10,0,2.6885878247284722e-05,1.977726248972888e-05
+per_repeat_pooled,3,100,ON,onto MDL,50,-0.001966443059106569,0.001122678411655309,26,24,0,0.766732668436843,0.4462195675403087
+per_repeat_pooled,3,100,ON,onto grow,50,-0.0026205072558879187,0.0022102385142370035,30,20,0,0.81837027658068,0.33846607901166437
+per_repeat_pooled,3,100,ON,onto grow and prune,50,-0.0026205072558879187,0.0022102385142370035,30,20,0,0.81837027658068,0.33846607901166437
+per_repeat_pooled,4,0,OFF,onto MDL,50,-0.00043811785463704567,0.0,1,3,46,0.4652088184521418,0.3231124071257798
+per_repeat_pooled,4,0,OFF,onto grow,50,0.0023516377291416136,0.0013064327815036358,28,13,9,0.061139832684513486,0.19443497786391611
+per_repeat_pooled,4,0,OFF,onto grow and prune,50,0.0023516377291416136,0.0013064327815036358,28,13,9,0.061139832684513486,0.19443497786391611
+per_repeat_pooled,4,0,ON,onto MDL,50,-0.0004393074658260876,0.0,1,3,46,0.4652088184521418,0.3220688694631895
+per_repeat_pooled,4,0,ON,onto grow,50,0.002442668663671088,0.0010365383648485993,28,13,9,0.04390205117798322,0.16363532308434922
+per_repeat_pooled,4,0,ON,onto grow and prune,50,0.002442668663671088,0.0010365383648485993,28,13,9,0.04390205117798322,0.16363532308434922
+per_repeat_pooled,4,10,OFF,onto MDL,50,-4.648061705179174e-05,0.0,6,8,36,0.5097974849608639,0.8937091813412903
+per_repeat_pooled,4,10,OFF,onto grow,50,-0.0035794931171793066,-0.0017510276534334346,19,28,3,0.08454696118090776,0.10082664734213999
+per_repeat_pooled,4,10,OFF,onto grow and prune,50,-0.0029274719133446448,-0.0024056202155169903,20,28,2,0.17907537170252064,0.19710240726771086
+per_repeat_pooled,4,10,ON,onto MDL,50,5.6997989391277534e-05,0.0,5,7,38,0.6948866023724733,0.8441342604654636
+per_repeat_pooled,4,10,ON,onto grow,50,-0.0035739141028983024,-0.00015990044074459808,22,25,3,0.12755097006124186,0.0864059882029709
+per_repeat_pooled,4,10,ON,onto grow and prune,50,-0.0029213754497903442,-0.0006608265008562209,21,27,2,0.2108255907095561,0.179776073544265
+per_repeat_pooled,4,20,OFF,onto MDL,50,-0.0028480708687979284,0.0,10,15,25,0.05108666110389699,0.01577335634728748
+per_repeat_pooled,4,20,OFF,onto grow,50,0.0022991908032037258,0.0017653081129824222,30,20,0,0.10737600575610173,0.17237490700974653
+per_repeat_pooled,4,20,OFF,onto grow and prune,50,0.002464307340831073,0.0017171061964621481,31,18,1,0.0808543667009045,0.1403651310023357
+per_repeat_pooled,4,20,ON,onto MDL,50,-0.002933191353788994,0.0,9,14,27,0.024404130322722078,0.01207724290696222
+per_repeat_pooled,4,20,ON,onto grow,50,0.002037370480408911,0.0015827950022109794,29,20,1,0.14780413771427342,0.2221431765485318
+per_repeat_pooled,4,20,ON,onto grow and prune,50,0.0022117436806042613,0.00158825691930356,31,18,1,0.10600021484120394,0.18124938959263057
+per_repeat_pooled,4,30,OFF,onto MDL,50,-0.0075901048152034875,-0.006827260209891356,15,34,1,0.003505585565059998,0.0023458099258936668
+per_repeat_pooled,4,30,OFF,onto grow,50,0.007931583424974721,0.007536904976736591,33,17,0,0.0017779629774352657,0.006656357336094241
+per_repeat_pooled,4,30,OFF,onto grow and prune,50,0.007585019714868717,0.007536904976736591,31,19,0,0.004912360514200742,0.012785338260257167
+per_repeat_pooled,4,30,ON,onto MDL,50,-0.009670760777199914,-0.009660247476395462,14,35,1,0.0002373604558953415,8.732250769013851e-05
+per_repeat_pooled,4,30,ON,onto grow,50,0.005722912498417829,0.005286620795034003,30,20,0,0.020665866669398625,0.02755204467321064
+per_repeat_pooled,4,30,ON,onto grow and prune,50,0.005365972128828142,0.0060400709114719064,29,21,0,0.04610038786489312,0.04887770890282603
+per_repeat_pooled,4,40,OFF,onto MDL,50,-0.013763186584818309,-0.011395775102262529,18,32,0,0.0010251160635235124,0.0005228166870663649
+per_repeat_pooled,4,40,OFF,onto grow,50,0.023495313603546935,0.024502399772517114,42,8,0,1.8673260271384606e-07,4.996333594850335e-08
+per_repeat_pooled,4,40,OFF,onto grow and prune,50,0.023596832681638914,0.025239433060219058,42,8,0,1.0490046520317264e-07,2.1690100774112125e-08
+per_repeat_pooled,4,40,ON,onto MDL,50,-0.02224307632392385,-0.02111503971164118,8,42,0,1.6183291151605772e-08,1.3674274958938072e-08
+per_repeat_pooled,4,40,ON,onto grow,50,0.01416368969337627,0.016009668256592724,37,13,0,0.00017022784864551,0.00010337199724995956
+per_repeat_pooled,4,40,ON,onto grow and prune,50,0.014337834321380609,0.016009668256592724,38,12,0,5.374574670646837e-05,3.9617565099519896e-05
+per_repeat_pooled,4,50,OFF,onto MDL,50,0.012377517572104053,0.01355281567401366,32,18,0,0.0002857006608838475,0.000195250913248652
+per_repeat_pooled,4,50,OFF,onto grow,50,0.04038307981851763,0.0443494965160955,48,2,0,9.769962616701378e-14,1.4763468981946096e-15
+per_repeat_pooled,4,50,OFF,onto grow and prune,50,0.04002551265029013,0.042620351295797096,49,1,0,4.440892098500626e-14,1.1378238340990511e-15
+per_repeat_pooled,4,50,ON,onto MDL,50,-0.03978679833733936,-0.03814591281663321,0,50,0,1.7763568394002505e-15,1.0846949996207964e-22
+per_repeat_pooled,4,50,ON,onto grow,50,-0.014657494179773294,-0.01641433567029099,11,39,0,8.237842195057965e-07,3.152378867320467e-07
+per_repeat_pooled,4,50,ON,onto grow and prune,50,-0.015145292311337735,-0.01623608143775826,11,39,0,6.354784893858323e-07,3.2737840940662485e-07
+per_repeat_pooled,4,60,OFF,onto MDL,50,0.011745808696861594,0.011863062404650779,35,15,0,0.0006715891126169993,0.0009754014355390173
+per_repeat_pooled,4,60,OFF,onto grow,50,0.0196379880678945,0.01804328758525095,37,13,0,1.1326234101716182e-06,2.5985704465769914e-06
+per_repeat_pooled,4,60,OFF,onto grow and prune,50,0.017810165320359315,0.01545803605998726,37,13,0,4.268415494834699e-06,7.497444267506702e-06
+per_repeat_pooled,4,60,ON,onto MDL,50,-0.020077367641210788,-0.017117249285513142,2,41,7,1.5944495794831735e-08,1.3142362707826297e-10
+per_repeat_pooled,4,60,ON,onto grow,50,-0.012749242095400772,-0.012792439579858872,14,36,0,3.457260659445183e-05,2.6427810858222546e-05
+per_repeat_pooled,4,60,ON,onto grow and prune,50,-0.015067387101004668,-0.014497442592809023,13,37,0,1.2841054282120012e-06,1.3544728652986158e-06
+per_repeat_pooled,4,70,OFF,onto MDL,50,0.013313863339226016,0.012591439198086612,41,9,0,0.00021178724761483636,0.0002593627266312863
+per_repeat_pooled,4,70,OFF,onto grow,50,0.0153133189977645,0.015681912276117652,38,12,0,0.0001778852638985029,0.00019575378140047122
+per_repeat_pooled,4,70,OFF,onto grow and prune,50,0.015236213226084554,0.015681912276117652,38,12,0,0.0001858580190070569,0.00020093807436410708
+per_repeat_pooled,4,70,ON,onto MDL,50,-0.006073480133468789,-0.0011681213082958553,11,25,14,0.009108481108181287,0.008156227441419257
+per_repeat_pooled,4,70,ON,onto grow,50,-0.0058567602172097345,-0.00023143763454575028,19,25,6,0.03671092160627901,0.0111310860798823
+per_repeat_pooled,4,70,ON,onto grow and prune,50,-0.005952959215036997,-0.00023143763454575028,19,25,6,0.033672230122832755,0.009654840224200753
+per_repeat_pooled,4,80,OFF,onto MDL,50,0.009538181984263065,0.009377850146964344,42,8,0,1.7496796758109667e-06,2.660665925772433e-06
+per_repeat_pooled,4,80,OFF,onto grow,50,0.006730964676699718,0.007589397112769354,36,14,0,0.006525205201144857,0.0112936671537753
+per_repeat_pooled,4,80,OFF,onto grow and prune,50,0.006729895475053149,0.007589397112769354,36,14,0,0.006525205201144857,0.01130688995101116
+per_repeat_pooled,4,80,ON,onto MDL,50,-0.002867905530588548,0.0,23,19,8,0.5865038395019753,0.2565402601457114
+per_repeat_pooled,4,80,ON,onto grow,50,-0.005148352989005766,-0.001161377108059955,21,26,3,0.13292505619274184,0.10434810201374345
+per_repeat_pooled,4,80,ON,onto grow and prune,50,-0.005155619908216693,-0.001161377108059955,21,26,3,0.13292505619274184,0.1038266824425522
+per_repeat_pooled,4,90,OFF,onto MDL,50,0.009496456742616873,0.012883386359368247,39,11,0,1.4422642710343325e-05,5.977539287821946e-06
+per_repeat_pooled,4,90,OFF,onto grow,50,0.010334362648039557,0.012575419359314752,41,9,0,5.6804637029728156e-06,3.191439749443588e-06
+per_repeat_pooled,4,90,OFF,onto grow and prune,50,0.010334362648039557,0.012575419359314752,41,9,0,5.6804637029728156e-06,3.191439749443588e-06
+per_repeat_pooled,4,90,ON,onto MDL,50,-0.0007637896954383555,0.0,23,23,4,0.8655266250286244,0.7440750617730048
+per_repeat_pooled,4,90,ON,onto grow,50,3.6565935425496577e-05,0.00012279848894559964,25,23,2,0.9509297401292157,0.9882015476438673
+per_repeat_pooled,4,90,ON,onto grow and prune,50,3.6565935425496577e-05,0.00012279848894559964,25,23,2,0.9509297401292157,0.9882015476438673
+per_repeat_pooled,4,100,OFF,onto MDL,50,0.0062296988223423215,0.0070558094027418986,36,14,0,0.0021224348550905603,0.002394956527138198
+per_repeat_pooled,4,100,OFF,onto grow,50,0.010478214840442153,0.015003651208749798,40,10,0,2.4281497463363166e-05,1.7476241983591324e-05
+per_repeat_pooled,4,100,OFF,onto grow and prune,50,0.010478214840442153,0.015003651208749798,40,10,0,2.4281497463363166e-05,1.7476241983591324e-05
+per_repeat_pooled,4,100,ON,onto MDL,50,-0.001966443059106569,0.001122678411655309,26,24,0,0.766732668436843,0.4462195675403087
+per_repeat_pooled,4,100,ON,onto grow,50,-0.0024861640386462745,0.0024680645956450947,30,20,0,0.893447016970665,0.3640369764663699
+per_repeat_pooled,4,100,ON,onto grow and prune,50,-0.0024861640386462745,0.0024680645956450947,30,20,0,0.893447016970665,0.3640369764663699

results/prof_comparison_summary.csv

@@ -0,0 +1,7 @@
+method,test_accuracy_mean,test_accuracy_std,test_f1_macro_mean,test_f1_macro_std,test_f1_weighted_mean,test_f1_weighted_std
+WEKA,0.636510757441792,0.041192597298334965,0.3252385671913087,0.0488031486755389,0.5630876431664139,0.048982235502031256
+Wittgenstein,0.5747524314765694,0.055499601558162776,0.38384237070337945,0.04185736320171567,0.5640112597448842,0.04434493411054236
+no guidance,0.6168897730621868,0.04753046746687279,0.3111571765455101,0.05465433344289358,0.5430189181937055,0.05771583771498859
+onto MDL,0.617026525198939,0.04759994049887874,0.31086586291060103,0.05426243203391457,0.5429549709632112,0.05767007097597026
+onto grow,0.6164096669613911,0.0494181191173883,0.31274669903102703,0.05598721953629108,0.5429788780348631,0.05865102339282895
+onto grow and prune,0.6164096669613911,0.0494181191173883,0.31274669903102703,0.05598721953629108,0.5429788780348631,0.05865102339282895

results/rule_dynamics/delta_table.csv

@@ -0,0 +1,67 @@
+metric,guided_method,level,delta_mean,delta_ci95
+test_accuracy,onto MDL,0,0.00013675213675213627,0.0003284931515746177
+test_accuracy,onto MDL,10,0.0006501620984379588,0.001695393845525678
+test_accuracy,onto MDL,20,-0.00099145299145299,0.0011110267506833523
+test_accuracy,onto MDL,30,0.0035885646920129667,0.0038228009016886694
+test_accuracy,onto MDL,40,0.015577954612437373,0.00690595971929375
+test_accuracy,onto MDL,50,0.10398997936928973,0.02162163454904151
+test_accuracy,onto MDL,60,0.04777541998231653,0.0168082933931708
+test_accuracy,onto MDL,70,0.06685027998821104,0.011325346905880138
+test_accuracy,onto MDL,80,0.06634070144414975,0.0035661707680036507
+test_accuracy,onto MDL,90,0.06232331270262306,0.004171773486667871
+test_accuracy,onto MDL,100,0.0444606542882405,0.005164168374252942
+test_accuracy,onto grow,0,-0.0004801061007957559,0.0029771111552491
+test_accuracy,onto grow,10,0.00016858237547892884,0.002685844144991781
+test_accuracy,onto grow,20,-0.00033333333333333305,0.0025633683494340016
+test_accuracy,onto grow,30,0.005960506926024165,0.004416357259988527
+test_accuracy,onto grow,40,0.022264367816091958,0.006509793693508296
+test_accuracy,onto grow,50,0.11536133215443561,0.020796058731825724
+test_accuracy,onto grow,60,0.05312319481284999,0.017315367588694053
+test_accuracy,onto grow,70,0.06905658709106988,0.012625449202345507
+test_accuracy,onto grow,80,0.062329796640141495,0.006574610485743653
+test_accuracy,onto grow,90,0.06345181255526085,0.00462849425439062
+test_accuracy,onto grow,100,0.057297376952549384,0.004796934688684249
+test_accuracy,onto grow and prune,0,-0.0004801061007957559,0.0029771111552491
+test_accuracy,onto grow and prune,10,0.0008885941644562355,0.0029293061589177696
+test_accuracy,onto grow and prune,20,-0.00023106395520188582,0.002580179283602735
+test_accuracy,onto grow and prune,30,0.005448275862068965,0.004537850237665943
+test_accuracy,onto grow and prune,40,0.022060123784261716,0.006544648433022054
+test_accuracy,onto grow and prune,50,0.11556616563513115,0.02070444322946091
+test_accuracy,onto grow and prune,60,0.05346831712348954,0.01719885767983797
+test_accuracy,onto grow and prune,70,0.06902239905688184,0.012612437132315853
+test_accuracy,onto grow and prune,80,0.062329796640141495,0.006574610485743653
+test_accuracy,onto grow and prune,90,0.06345181255526085,0.00462849425439062
+test_accuracy,onto grow and prune,100,0.057297376952549384,0.004796934688684249
+test_f1_macro,onto MDL,0,-0.00029131363490910567,0.0006566522813460928
+test_f1_macro,onto MDL,10,-0.00011817501608798876,0.0007717772602733174
+test_f1_macro,onto MDL,20,-0.0024748774445604933,0.0017916859508128806
+test_f1_macro,onto MDL,30,-0.008956975916196493,0.005075818842752148
+test_f1_macro,onto MDL,40,-0.01617558416583157,0.007338186637399785
+test_f1_macro,onto MDL,50,0.02046152095167934,0.007778508386274938
+test_f1_macro,onto MDL,60,0.011649152821565492,0.006853998144977997
+test_f1_macro,onto MDL,70,0.004091935922945117,0.005248986959110748
+test_f1_macro,onto MDL,80,-0.0022692314901096157,0.002878004504902562
+test_f1_macro,onto MDL,90,-0.007080874563258173,0.0030064733165295577
+test_f1_macro,onto MDL,100,-0.012722906689013306,0.0029352588855109172
+test_f1_macro,onto grow,0,0.0015895224855168822,0.003311271377479235
+test_f1_macro,onto grow,10,-0.0008804402794625979,0.003838145035352017
+test_f1_macro,onto grow,20,0.0028272617699814182,0.0031403950418272403
+test_f1_macro,onto grow,30,0.0069781777959364265,0.00532959443554973
+test_f1_macro,onto grow,40,0.017006443819950246,0.007556164119071725
+test_f1_macro,onto grow,50,0.04285981614721873,0.0077601279999537235
+test_f1_macro,onto grow,60,0.0169745240765574,0.007408635973099277
+test_f1_macro,onto grow,70,0.003820556641680585,0.005765027612978602
+test_f1_macro,onto grow,80,-0.005217451842390732,0.003920749135125546
+test_f1_macro,onto grow,90,-0.006299348301672776,0.0032698991174653245
+test_f1_macro,onto grow,100,-0.009355033526292,0.0031267990776208472
+test_f1_macro,onto grow and prune,0,0.0015895224855168822,0.003311271377479235
+test_f1_macro,onto grow and prune,10,-0.00038836981006982194,0.00392156405769381
+test_f1_macro,onto grow and prune,20,0.003036077213476018,0.0031237073453517445
+test_f1_macro,onto grow and prune,30,0.00636934489157347,0.005556272508168221
+test_f1_macro,onto grow and prune,40,0.016813683855065972,0.007524926044459912
+test_f1_macro,onto grow and prune,50,0.0424818689185778,0.007747898629601222
+test_f1_macro,onto grow and prune,60,0.01589412690361924,0.007358850349214752
+test_f1_macro,onto grow and prune,70,0.0037489244516285894,0.005745497744096606
+test_f1_macro,onto grow and prune,80,-0.005218420834638795,0.0039209038483859655
+test_f1_macro,onto grow and prune,90,-0.006299348301672776,0.0032698991174653245
+test_f1_macro,onto grow and prune,100,-0.009355033526292,0.0031267990776208472

results/rule_dynamics/regime_summary.csv

@@ -0,0 +1,45 @@
+method,percentage,train_accuracy_mean,test_accuracy_mean,test_f1_macro_mean,test_f1_weighted_mean,invalid_rate_mean,abstain_rate_mean,total_rules_mean,rare_rules_mean
+no guidance,0,0.6389,0.6169,0.3112,0.543,0.0053,0.0,4.2,0.5
+no guidance,10,0.5842,0.6163,0.3103,0.5429,0.0057,0.0,4.2,0.1
+no guidance,20,0.5333,0.6241,0.3151,0.5516,0.0047,0.0,4.3,0.1
+no guidance,30,0.4716,0.606,0.296,0.5309,0.0307,0.0,4.1,0.1
+no guidance,40,0.418,0.5667,0.2551,0.4948,0.1114,0.0,4.3,0.1
+no guidance,50,0.3759,0.4619,0.223,0.4242,0.2736,0.0,4.6,0.3
+no guidance,60,0.3803,0.3415,0.1768,0.3511,0.4882,0.0,5.1,1.1
+no guidance,70,0.4215,0.0826,0.082,0.0768,0.8286,0.0,3.7,0.9
+no guidance,80,0.4696,0.0563,0.0695,0.051,0.8614,0.0,3.9,1.4
+no guidance,90,0.5171,0.0504,0.0642,0.0436,0.8688,0.0,3.3,0.9
+no guidance,100,0.5657,0.051,0.0597,0.0412,0.8694,0.0,4.6,0.6
+onto MDL,0,0.639,0.617,0.3109,0.543,0.0053,0.0,4.2,0.5
+onto MDL,10,0.5844,0.617,0.3102,0.5431,0.0058,0.0,4.3,0.1
+onto MDL,20,0.5327,0.6231,0.3126,0.5491,0.0038,0.0,4.2,0.0
+onto MDL,30,0.4632,0.6095,0.287,0.5202,0.0052,0.0,3.2,0.0
+onto MDL,40,0.3832,0.5823,0.2389,0.4686,0.0097,0.0,2.2,0.0
+onto MDL,50,0.3454,0.5659,0.2434,0.4745,0.0822,0.0,2.5,0.0
+onto MDL,60,0.3241,0.3892,0.1884,0.3553,0.3894,0.0,2.6,0.0
+onto MDL,70,0.3377,0.1495,0.0861,0.0814,0.6936,0.0,1.3,0.0
+onto MDL,80,0.369,0.1227,0.0672,0.0492,0.7227,0.0,1.0,0.0
+onto MDL,90,0.4014,0.1127,0.0571,0.0377,0.7292,0.0,1.0,0.0
+onto MDL,100,0.4626,0.0955,0.0469,0.0305,0.7608,0.0,2.0,0.0
+onto grow,0,0.6386,0.6164,0.3127,0.543,0.0058,0.0,4.6,0.9
+onto grow,10,0.5857,0.6165,0.3094,0.5433,0.0057,0.0,4.9,0.6
+onto grow,20,0.5324,0.6237,0.3179,0.5527,0.0043,0.0,4.7,0.1
+onto grow,30,0.4661,0.6119,0.3029,0.534,0.0053,0.0,3.8,0.1
+onto grow,40,0.3919,0.5889,0.2721,0.499,0.0095,0.0,3.4,0.1
+onto grow,50,0.349,0.5773,0.2658,0.4956,0.0656,0.0,3.4,0.0
+onto grow,60,0.3239,0.3946,0.1938,0.3606,0.3795,0.0,2.9,0.0
+onto grow,70,0.3374,0.1517,0.0858,0.0829,0.6894,0.0,1.3,0.0
+onto grow,80,0.3695,0.1187,0.0643,0.0436,0.7275,0.0,1.2,0.0
+onto grow,90,0.4017,0.1139,0.0579,0.0392,0.7281,0.0,1.5,0.0
+onto grow,100,0.4375,0.1083,0.0503,0.0332,0.7289,0.0,2.3,0.0
+onto grow and prune,0,0.6386,0.6164,0.3127,0.543,0.0058,0.0,4.6,0.9
+onto grow and prune,10,0.586,0.6172,0.3099,0.5439,0.0057,0.0,4.9,0.6
+onto grow and prune,20,0.5325,0.6238,0.3181,0.5529,0.0042,0.0,4.7,0.1
+onto grow and prune,30,0.4659,0.6114,0.3023,0.5333,0.0051,0.0,3.8,0.1
+onto grow and prune,40,0.3917,0.5887,0.2719,0.4986,0.0095,0.0,3.4,0.1
+onto grow and prune,50,0.349,0.5775,0.2655,0.4955,0.0655,0.0,3.4,0.0
+onto grow and prune,60,0.3236,0.3949,0.1927,0.3601,0.3796,0.0,2.9,0.0
+onto grow and prune,70,0.3373,0.1517,0.0857,0.0829,0.6895,0.0,1.3,0.0
+onto grow and prune,80,0.3694,0.1187,0.0643,0.0436,0.7275,0.0,1.2,0.0
+onto grow and prune,90,0.4017,0.1139,0.0579,0.0392,0.7281,0.0,1.5,0.0
+onto grow and prune,100,0.4375,0.1083,0.0503,0.0332,0.7289,0.0,2.3,0.0

results/runtime_ON_results.csv

@@ -0,0 +1,89 @@
+lambda,method,percentage,train_accuracy_mean,train_accuracy_std,test_accuracy_mean,test_accuracy_std,test_f1_macro_mean,test_f1_macro_std,test_f1_weighted_mean,test_f1_weighted_std,invalid_rate_mean,invalid_rate_std,abstain_rate_mean,abstain_rate_std
+3,no guidance,0,0.6416355899632131,0.03522836353740601,0.6202449160035367,0.04382797065605943,0.31648679007665537,0.047830793615791144,0.5483825662123751,0.05019058910420566,0.0,0.0,0.0,0.0
+3,no guidance,10,0.5860526180933033,0.030689407446778293,0.6195938697318008,0.04072829566249178,0.3154378204256887,0.045637196802629086,0.5477903478179468,0.04569374117691377,0.0,0.0,0.0,0.0
+3,no guidance,20,0.533495598382108,0.022154829968964007,0.6254694960212202,0.037578436191611324,0.31705409937555606,0.04406512054476819,0.5532915369386701,0.04293869206298684,0.0,0.0,0.0,0.0
+3,no guidance,30,0.4629803071066454,0.021864963820981773,0.6080825228411435,0.037784212126295756,0.2997424813838507,0.05072079347396136,0.529033985454581,0.049166154804330654,0.0,0.0,0.0,0.0
+3,no guidance,40,0.3820721279672029,0.019829941681020327,0.5763689949896846,0.032242385724369424,0.26937059985190986,0.05359489994508302,0.48953729621401915,0.049156433428863534,0.0,0.0,0.0,0.0
+3,no guidance,50,0.3452226065630776,0.019806842884168615,0.6115075154730327,0.042420477267015644,0.2936274635103366,0.0549940112979839,0.522040541682896,0.05561992416428116,0.0,0.0,0.0,0.0
+3,no guidance,60,0.29156695766760005,0.014640226861077238,0.6250374300029473,0.02704513609184367,0.29008651104624944,0.03823291539381069,0.5258715646205067,0.03016252242395345,0.0,0.0,0.0,0.0
+3,no guidance,70,0.21966935705265467,0.036789746460666864,0.5732740937223696,0.12734822967102538,0.2703828250296886,0.044545663894698946,0.4674436427469209,0.13179459044031488,0.0,0.0,0.0,0.0
+3,no guidance,80,0.1022696791668954,0.015005280911434386,0.2702973769525493,0.047020004943243654,0.19657869424216348,0.037217956245266845,0.15451902244271878,0.05620937195928732,0.0,0.0,0.0,0.0
+3,no guidance,90,0.07970076319112718,0.009923872068670996,0.2614582964927792,0.027594781611901063,0.1890282499430488,0.031112244400970154,0.14330756322137467,0.020290045401146834,0.0,0.0,0.0,0.0
+3,no guidance,100,0.061592854920477995,0.00739956873817343,0.25937400530503973,0.02862651622411579,0.18793793714348456,0.03212344706462108,0.14113428895613306,0.020082535627232997,0.0,0.0,0.0,0.0
+3,onto MDL,0,0.6416784165156756,0.03526192273769776,0.6203132920719128,0.04383369146800163,0.3164330198159084,0.04780491610317273,0.5483971485292831,0.05019411107414464,0.0,0.0,0.0,0.0
+3,onto MDL,10,0.5864551327806146,0.030534750400422993,0.6203804892425582,0.040408000658168845,0.31554078924206386,0.04555195883826585,0.5481900893679033,0.04569407901173049,0.0,0.0,0.0,0.0
+3,onto MDL,20,0.5334436574607881,0.02270690649948424,0.6247866195107574,0.038234713031479406,0.31599869478750026,0.04447851791994689,0.5522494297557939,0.04381951296811285,0.0,0.0,0.0,0.0
+3,onto MDL,30,0.46589260418382467,0.021896941749517493,0.6126982021809608,0.036888207692351675,0.2935587538442123,0.051805137780017364,0.5265838634391895,0.04967405590368263,0.0,0.0,0.0,0.0
+3,onto MDL,40,0.38833014879481687,0.024232460316353878,0.5869899793692896,0.0371900100611999,0.24882001963739508,0.05140779108390321,0.47834294183770903,0.05022775402791386,0.0,0.0,0.0,0.0
+3,onto MDL,50,0.342294862643899,0.01722776912395273,0.6105800176834659,0.037509832562848726,0.2661588355223188,0.043537764761902166,0.5020250758216019,0.04485498081545355,0.0,0.0,0.0,0.0
+3,onto MDL,60,0.29193484507403133,0.012760941345843648,0.6278464485705865,0.024733001163159615,0.27769309776159984,0.02668705643093563,0.5207281292101698,0.022173773126696446,0.0,0.0,0.0,0.0
+3,onto MDL,70,0.22049115100935224,0.03651783239823368,0.5760503978779841,0.12651989315719592,0.26523585141129463,0.04127345338016712,0.4667266360394275,0.13122504918536435,0.0,0.0,0.0,0.0
+3,onto MDL,80,0.10232917879170551,0.01581444324066397,0.27406896551724136,0.04758024627430639,0.18926501589537467,0.033240817481135256,0.15188038519990982,0.057859520277356274,0.0,0.0,0.0,0.0
+3,onto MDL,90,0.07853663134391181,0.011897287923733419,0.2634789272030651,0.03084841563544836,0.17765088035448212,0.029056913101229397,0.1369552234976835,0.028949007076493897,0.0,0.0,0.0,0.0
+3,onto MDL,100,0.059135287981112346,0.009920592979846838,0.2590315355142941,0.03450031296270427,0.169478007144333,0.0313112031926618,0.12995245362155974,0.03219245681073592,0.0,0.0,0.0,0.0
+3,onto grow,0,0.6410364757773751,0.03637835245434941,0.6193171234895373,0.045318434283672104,0.3180844316546139,0.049768205162373166,0.548357473547466,0.05121128968288741,0.0,0.0,0.0,0.0
+3,onto grow,10,0.5878612163472977,0.028958626194103016,0.6198924255820807,0.038148696184831195,0.31525324684489886,0.044056089250194934,0.5485181469235751,0.04388848298539027,0.0,0.0,0.0,0.0
+3,onto grow,20,0.5332292318673475,0.02117463177147613,0.6248638373121133,0.035434000225441596,0.31852323211917344,0.04654413184766141,0.5537867318786226,0.04307895919321017,0.0,0.0,0.0,0.0
+3,onto grow,30,0.4687077728362524,0.02097831365421446,0.6145918066607722,0.03832274853955769,0.3092891796986433,0.05059695549613844,0.5399000085159913,0.047948723004910584,0.0,0.0,0.0,0.0
+3,onto grow,40,0.3958813667892897,0.02445144377892253,0.5941844974948423,0.04130265892864718,0.2837770510837331,0.058299398210205446,0.5099019472034011,0.0570902905223578,0.0,0.0,0.0,0.0
+3,onto grow,50,0.3472690202968575,0.017823890232091865,0.614211317418214,0.03918113903245469,0.28952871389404816,0.052143758343239796,0.5227048682397888,0.052274541744383965,0.0,0.0,0.0,0.0
+3,onto grow,60,0.29268872051099026,0.013488039670775146,0.6282914824638963,0.026785303197667944,0.2821201980389633,0.03166914541338366,0.5244760393952667,0.02742770470622777,0.0,0.0,0.0,0.0
+3,onto grow,70,0.22060209740295392,0.0359183827675597,0.5766710875331565,0.12544391862807253,0.2648644229900416,0.04154161559065117,0.4674929733764606,0.13056807323310152,0.0,0.0,0.0,0.0
+3,onto grow,80,0.10159331246911542,0.014585150145819072,0.2704002357795461,0.03767588708839537,0.18652127541669394,0.029666693278352384,0.14644993340218984,0.04044005231932188,0.0,0.0,0.0,0.0
+3,onto grow,90,0.07923022017240429,0.01242241012420311,0.26447067491895077,0.03436747531435578,0.1782763388574072,0.030837950391919167,0.13844090985973118,0.036551111938989175,0.0,0.0,0.0,0.0
+3,onto grow,100,0.05932365160416548,0.009701794814317426,0.25896374889478335,0.03429355020994543,0.16903972051507837,0.030130172585092728,0.1297953537813481,0.03154816843207273,0.0,0.0,0.0,0.0
+3,onto grow and prune,0,0.6410364757773751,0.03637835245434941,0.6193171234895373,0.045318434283672104,0.3180844316546139,0.049768205162373166,0.548357473547466,0.05121128968288741,0.0,0.0,0.0,0.0
+3,onto grow and prune,10,0.5879637255440253,0.028959537382632967,0.6200654288240495,0.037930726773910343,0.3155897273536329,0.04398453956549824,0.548805376841976,0.04372000067597367,0.0,0.0,0.0,0.0
+3,onto grow and prune,20,0.5334260692911656,0.021245217554784337,0.6250002947244327,0.03558152334051094,0.31874526886532684,0.046464020543759,0.5540919560343378,0.04317272179823049,0.0,0.0,0.0,0.0
+3,onto grow and prune,30,0.46854497703105835,0.021064323155547567,0.6140450928381963,0.0384576854831864,0.30860307418493194,0.05103440787716521,0.5390611397592715,0.04846812953899637,0.0,0.0,0.0,0.0
+3,onto grow and prune,40,0.39583848533099064,0.02435074784930772,0.593910993221338,0.041267331863056464,0.28361058098496494,0.05791109116242714,0.509561900441332,0.05658104637494532,0.0,0.0,0.0,0.0
+3,onto grow and prune,50,0.3472518164680905,0.01781014465257841,0.6137318007662835,0.03896405700295163,0.28841205307440326,0.05200502092941704,0.5215987449350924,0.051961497041071704,0.0,0.0,0.0,0.0
+3,onto grow and prune,60,0.2925687512582587,0.013582561090973227,0.6283940465664604,0.02653165448766286,0.2817282048400364,0.031659487874336416,0.5243335503083496,0.027392233275640776,0.0,0.0,0.0,0.0
+3,onto grow and prune,70,0.22055927085049137,0.03590174532850135,0.5766368994989685,0.12542513378518488,0.2647843231264831,0.04150666992795722,0.46741221966563173,0.13053009006690205,0.0,0.0,0.0,0.0
+3,onto grow and prune,80,0.10159331246911542,0.014585150145819072,0.2704002357795461,0.03767588708839537,0.18651363534805201,0.02967481049565779,0.14644438196922208,0.040442596067665135,0.0,0.0,0.0,0.0
+3,onto grow and prune,90,0.07923022017240429,0.01242241012420311,0.26447067491895077,0.03436747531435578,0.1782686987887653,0.030843708962030217,0.1384353584267634,0.0365527050824387,0.0,0.0,0.0,0.0
+3,onto grow and prune,100,0.05932365160416548,0.009701794814317426,0.25896374889478335,0.03429355020994543,0.16903972051507837,0.030130172585092728,0.1297953537813481,0.03154816843207273,0.0,0.0,0.0,0.0
+4,no guidance,0,0.6416355899632131,0.03522836353740601,0.6202449160035367,0.04382797065605943,0.31648679007665537,0.047830793615791144,0.5483825662123751,0.05019058910420566,0.0,0.0,0.0,0.0
+4,no guidance,10,0.5860526180933033,0.030689407446778293,0.6195938697318008,0.04072829566249178,0.3154378204256887,0.045637196802629086,0.5477903478179468,0.04569374117691377,0.0,0.0,0.0,0.0
+4,no guidance,20,0.533495598382108,0.022154829968964007,0.6254694960212202,0.037578436191611324,0.31705409937555606,0.04406512054476819,0.5532915369386701,0.04293869206298684,0.0,0.0,0.0,0.0
+4,no guidance,30,0.4629803071066454,0.021864963820981773,0.6080825228411435,0.037784212126295756,0.2997424813838507,0.05072079347396136,0.529033985454581,0.049166154804330654,0.0,0.0,0.0,0.0
+4,no guidance,40,0.3820721279672029,0.019829941681020327,0.5763689949896846,0.032242385724369424,0.26937059985190986,0.05359489994508302,0.48953729621401915,0.049156433428863534,0.0,0.0,0.0,0.0
+4,no guidance,50,0.3452226065630776,0.019806842884168615,0.6115075154730327,0.042420477267015644,0.2936274635103366,0.0549940112979839,0.522040541682896,0.05561992416428116,0.0,0.0,0.0,0.0
+4,no guidance,60,0.29156695766760005,0.014640226861077238,0.6250374300029473,0.02704513609184367,0.29008651104624944,0.03823291539381069,0.5258715646205067,0.03016252242395345,0.0,0.0,0.0,0.0
+4,no guidance,70,0.21966935705265467,0.036789746460666864,0.5732740937223696,0.12734822967102538,0.2703828250296886,0.044545663894698946,0.4674436427469209,0.13179459044031488,0.0,0.0,0.0,0.0
+4,no guidance,80,0.1022696791668954,0.015005280911434386,0.2702973769525493,0.047020004943243654,0.19657869424216348,0.037217956245266845,0.15451902244271878,0.05620937195928732,0.0,0.0,0.0,0.0
+4,no guidance,90,0.07970076319112718,0.009923872068670996,0.2614582964927792,0.027594781611901063,0.1890282499430488,0.031112244400970154,0.14330756322137467,0.020290045401146834,0.0,0.0,0.0,0.0
+4,no guidance,100,0.061592854920477995,0.00739956873817343,0.25937400530503973,0.02862651622411579,0.18793793714348456,0.03212344706462108,0.14113428895613306,0.020082535627232997,0.0,0.0,0.0,0.0
+4,onto MDL,0,0.6417897655520781,0.035319077367878454,0.6203816681402888,0.04389280896055804,0.3161954764417463,0.047415375489819306,0.5483186189818808,0.050144821603763885,0.0,0.0,0.0,0.0
+4,onto MDL,10,0.5864551327806146,0.030534750400422993,0.6203804892425582,0.040408000658168845,0.31554078924206386,0.04555195883826585,0.5481900893679033,0.04569407901173049,0.0,0.0,0.0,0.0
+4,onto MDL,20,0.5333838467029045,0.022669897798048483,0.6245461243737105,0.03821735973049909,0.31456657821214845,0.04514153335715511,0.5511019302853832,0.04431312495883235,0.0,0.0,0.0,0.0
+4,onto MDL,30,0.4651476051904317,0.022105915105478706,0.6118420277040967,0.036708786828294586,0.29032270549680894,0.05136407363697021,0.523621673933712,0.04954568624794779,0.0,0.0,0.0,0.0
+4,onto MDL,40,0.38825328062373027,0.02390193957933183,0.5876401414677276,0.03721963683396316,0.24837814872047634,0.05069298101139089,0.47814458327595516,0.04952667351561828,0.0,0.0,0.0,0.0
+4,onto MDL,50,0.34252641885832463,0.016796857792062247,0.6107500736811081,0.037334648749119966,0.265894444013562,0.04286816068219353,0.5019292646495326,0.04420189884897579,0.0,0.0,0.0,0.0
+4,onto MDL,60,0.29196910631600137,0.012701107057970246,0.6279148246389626,0.024650732130363145,0.27782350528511085,0.026754449413685346,0.5208955755569263,0.022043129276544345,0.0,0.0,0.0,0.0
+4,onto MDL,70,0.2204313219495232,0.03649911424968663,0.5760159151193635,0.12651592963119881,0.2651190698020992,0.041257979561008955,0.46657505458877346,0.13119610023141737,0.0,0.0,0.0,0.0
+4,onto MDL,80,0.1025688427679862,0.015520572169012084,0.2741715296198054,0.04744615558894501,0.18948118472057576,0.03318120067079072,0.15221370281701285,0.05768711209876869,0.0,0.0,0.0,0.0
+4,onto MDL,90,0.07853663134391181,0.011897287923733419,0.2634789272030651,0.03084841563544836,0.17765088035448212,0.029056913101229397,0.1369552234976835,0.028949007076493897,0.0,0.0,0.0,0.0
+4,onto MDL,100,0.059135287981112346,0.009920592979846838,0.2590315355142941,0.03450031296270427,0.169478007144333,0.0313112031926618,0.12995245362155974,0.03219245681073592,0.0,0.0,0.0,0.0
+4,onto grow,0,0.6415413715477956,0.03615825494348182,0.6199015620394931,0.0458644958308257,0.31821325015452523,0.0496700901908477,0.5485133052434606,0.051465720527921706,0.0,0.0,0.0,0.0
+4,onto grow,10,0.5876379692161277,0.029871787970287073,0.619763041556145,0.04013728202239521,0.3142203369732183,0.045032529385457126,0.5478653053665367,0.0452276814511622,0.0,0.0,0.0,0.0
+4,onto grow,20,0.5333148300664361,0.020950438434347244,0.6250335985853227,0.03543763829261366,0.31963802816186215,0.04650413321236954,0.5544435607305006,0.04307679008172777,0.0,0.0,0.0,0.0
+4,onto grow,30,0.46810005673603106,0.021391641003611007,0.6140067786619511,0.038426422887923875,0.30602487219608343,0.050926053468274245,0.5371342695731842,0.04844545026909421,0.0,0.0,0.0,0.0
+4,onto grow,40,0.39594895587400936,0.02407175977033457,0.5936050692602416,0.04175833034098426,0.2802211113406252,0.05828978183434533,0.5069111653988005,0.05748968005337593,0.0,0.0,0.0,0.0
+4,onto grow,50,0.34640407035267845,0.018284997130791996,0.6134585912172119,0.039125042107716446,0.28580024070035787,0.05102630736885677,0.5191542582469333,0.051897210657702225,0.0,0.0,0.0,0.0
+4,onto grow,60,0.2927488607038928,0.013447703807508296,0.6282575891541409,0.027075408079966907,0.2822094722566077,0.03275409009023839,0.5247057871748746,0.028528452928324757,0.0,0.0,0.0,0.0
+4,onto grow,70,0.22067913029154998,0.03593948641976812,0.5766366047745358,0.1256395112031592,0.26452310403038887,0.04187578846442189,0.46730258293560445,0.13090876219900102,0.0,0.0,0.0,0.0
+4,onto grow,80,0.1015419389081059,0.014502624761719422,0.2706407309165929,0.03766863461497464,0.18683136541700315,0.02970038438140867,0.14666732265773189,0.040474393838516386,0.0,0.0,0.0,0.0
+4,onto grow,90,0.07922165486191177,0.012412768675234722,0.2645048629531388,0.03435513722707721,0.1784469126067105,0.030923751611384404,0.1385148531473243,0.036568336488156,0.0,0.0,0.0,0.0
+4,onto grow,100,0.05929795567268796,0.009666796430792912,0.2589979369289714,0.034320908584517106,0.16922169330419404,0.030278178492267434,0.12989626081082653,0.03162237426013998,0.0,0.0,0.0,0.0
+4,onto grow and prune,0,0.6415413715477956,0.03615825494348182,0.6199015620394931,0.0458644958308257,0.31821325015452523,0.0496700901908477,0.5485133052434606,0.051465720527921706,0.0,0.0,0.0,0.0
+4,onto grow and prune,10,0.5879032193122129,0.02956409172616981,0.6204830533451223,0.03892598255360576,0.31470961578143586,0.04492021138209578,0.548478399522393,0.044806580582625516,0.0,0.0,0.0,0.0
+4,onto grow and prune,20,0.5335031021797617,0.021010152591749286,0.6251700559976422,0.03559273634396928,0.3198605751056663,0.04642104900108402,0.554749205675206,0.04316740037919196,0.0,0.0,0.0,0.0
+4,onto grow and prune,30,0.46797143066307945,0.021268096377601298,0.6134603595638078,0.03861741489909737,0.30538218233981235,0.051348063509491906,0.5363384354100146,0.04893230507508172,0.0,0.0,0.0,0.0
+4,onto grow and prune,40,0.3958889437947254,0.023959422723401576,0.5934692012967876,0.04170513977423367,0.28019611451754145,0.057559101034039356,0.5067120751124533,0.05672038894808037,0.0,0.0,0.0,0.0
+4,onto grow and prune,50,0.3463612071963249,0.01830556721567114,0.6133899204244032,0.03928393947955579,0.2853367691944282,0.05145325171754241,0.5187695820260907,0.05224736250493652,0.0,0.0,0.0,0.0
+4,onto grow and prune,60,0.2924490199308186,0.013471370704739848,0.6285652814618331,0.02630355876199098,0.28102927979880526,0.0314244179075806,0.5239921805177247,0.027441049665941363,0.0,0.0,0.0,0.0
+4,onto grow and prune,70,0.22063630373908744,0.03592295096058847,0.5766024167403477,0.12562074618616664,0.26443357655740274,0.04183489192016755,0.4672145772175235,0.13086739113621068,0.0,0.0,0.0,0.0
+4,onto grow and prune,80,0.1015419389081059,0.014502624761719422,0.2706407309165929,0.03766863461497464,0.18682372534836125,0.029708572458661636,0.14666177122476412,0.04047696536346658,0.0,0.0,0.0,0.0
+4,onto grow and prune,90,0.07922165486191177,0.012412768675234722,0.2645048629531388,0.03435513722707721,0.1784469126067105,0.030923751611384404,0.1385148531473243,0.036568336488156,0.0,0.0,0.0,0.0
+4,onto grow and prune,100,0.05929795567268796,0.009666796430792912,0.2589979369289714,0.034320908584517106,0.16922169330419404,0.030278178492267434,0.12989626081082653,0.03162237426013998,0.0,0.0,0.0,0.0

results/significance_both.csv

@@ -0,0 +1,67 @@
+lam,pct,method,n,nonzero,shapiro_p,normal,mean_delta,wilcoxon_p,ttest_p,wilcoxon_sig,ttest_sig,agree
+3,0,onto MDL,50,1,1.1274100233477779e-15,False,-0.0001295870836951507,0.31731050786291415,0.3222234059506754,False,False,True
+3,0,onto grow,50,36,0.0021720275461818454,False,0.0017398493082464639,0.16204065786582023,0.3064195169870864,False,False,True
+3,0,onto grow and prune,50,36,0.0021720275461818454,False,0.0017398493082464639,0.16204065786582023,0.3064195169870864,False,False,True
+3,10,onto MDL,50,14,2.281258540806125e-10,False,-4.648061705179174e-05,0.5097974849608639,0.8937091813412903,False,False,True
+3,10,onto grow,50,46,0.021420245465764674,False,-0.001981272188089278,0.5515541606343191,0.294827576774398,False,False,True
+3,10,onto grow and prune,50,48,0.05410176146097083,True,-0.001475346302953623,0.7043224179898399,0.45615678289225964,False,False,True
+3,20,onto MDL,50,21,1.9237950861397012e-11,False,-0.0011627377240614268,0.2172414463719159,0.08241031963892155,False,False,True
+3,20,onto grow,50,50,0.02256862974186193,False,0.0011052334826056687,0.1329605561585705,0.4494359767076001,False,False,True
+3,20,onto grow and prune,50,49,0.007635465713314887,False,0.0012721598668214297,0.07417386882006018,0.3764497321252334,False,False,True
+3,30,onto MDL,50,49,0.2207537163839054,True,-0.004510154331653753,0.03901168331216344,0.035066460143719266,True,True,True
+3,30,onto grow,50,50,0.019052415571394768,False,0.011086076261276645,0.00011903464815610221,0.0004571417837678272,True,True,True
+3,30,onto grow and prune,50,50,0.02305497967812218,False,0.010532830084721724,0.00032407658280853013,0.0010112674839723873,True,True,True
+3,40,onto MDL,50,50,0.30803960233894495,True,-0.012740973116078856,0.004178445711550793,0.0020335620534605303,True,True,True
+3,40,onto grow,50,50,0.5719950824935351,True,0.026772516292561013,1.9326371614170057e-09,4.5077417505455057e-10,True,True,True
+3,40,onto grow and prune,50,50,0.6905693564242854,True,0.026738017881094117,1.7568968502246207e-09,2.9731128269997035e-10,True,True,True
+3,50,onto MDL,50,50,0.21473759952855842,True,0.013072220439863398,0.00035222154756553437,9.391967129976528e-05,True,True,True
+3,50,onto grow,50,50,0.9932010193784759,True,0.04411626697394975,2.4868995751603507e-14,1.6494587247945278e-17,True,True,True
+3,50,onto grow and prune,50,50,0.9965483065858859,True,0.042831734234216635,2.4868995751603507e-14,6.700186398197848e-17,True,True,True
+3,60,onto MDL,50,50,0.004103364789925923,False,0.01041534392072354,0.0014846473444478647,0.0015145861069850945,True,True,True
+3,60,onto grow,50,50,0.039121888798718314,False,0.02099398097940716,1.0490046520317264e-07,3.1959996451376927e-07,True,True,True
+3,60,onto grow and prune,50,50,0.022828920717122392,False,0.020407352766542406,1.2141019567479816e-07,4.6628079318952006e-07,True,True,True
+3,70,onto MDL,50,50,0.05193227594959449,True,0.011918814318185911,0.0009871220775430345,0.0007638129060927107,True,True,True
+3,70,onto grow,50,50,0.25639530092572443,True,0.01566546337181986,0.00015581408102782746,0.00011265265206218932,True,True,True
+3,70,onto grow and prune,50,50,0.2886306852963083,True,0.01559352343650226,0.00015581408102782746,0.00011471289932271423,True,True,True
+3,80,onto MDL,50,50,0.01063847117421645,False,0.009688529343792673,1.7496796758109667e-06,2.2210543545303727e-06,True,True,True
+3,80,onto grow,50,50,0.07093747489867304,True,0.006517362999950538,0.007380741603101981,0.013679997970944538,True,True,True
+3,80,onto grow and prune,50,50,0.07107190790203703,True,0.0065162937983039675,0.007380741603101981,0.013695704898264802,True,True,True
+3,90,onto MDL,50,50,0.05927570420857092,True,0.0030821898275293826,0.07885790741956278,0.1360444734107177,False,False,True
+3,90,onto grow,50,50,0.00855297352369549,False,0.010181047095130139,5.6804637029728156e-06,1.7813192173008554e-06,True,True,True
+3,90,onto grow and prune,50,50,0.008572510115664149,False,0.010179981219149653,5.6804637029728156e-06,1.7843226569567727e-06,True,True,True
+3,100,onto MDL,50,50,0.0057271981601623246,False,-0.01102700273894665,1.1287300338835848e-07,2.61022674802467e-07,True,True,True
+3,100,onto grow,50,50,0.0018004421617156957,False,0.0103395665697723,2.6885878247284722e-05,1.977726248972888e-05,True,True,True
+3,100,onto grow and prune,50,50,0.0018004421617156957,False,0.0103395665697723,2.6885878247284722e-05,1.977726248972888e-05,True,True,True
+4,0,onto MDL,50,4,6.009678302856076e-14,False,-0.00043811785463704567,0.4652088184521418,0.3231124071257798,False,False,True
+4,0,onto grow,50,41,0.008023155217605295,False,0.0023516377291416136,0.061139832684513486,0.19443497786391611,False,False,True
+4,0,onto grow and prune,50,41,0.008023155217605295,False,0.0023516377291416136,0.061139832684513486,0.19443497786391611,False,False,True
+4,10,onto MDL,50,14,2.281258540806125e-10,False,-4.648061705179174e-05,0.5097974849608639,0.8937091813412903,False,False,True
+4,10,onto grow,50,47,0.36651375942182585,True,-0.0035794931171793066,0.08454696118090776,0.10082664734213999,False,False,True
+4,10,onto grow and prune,50,48,0.374525672276769,True,-0.0029274719133446448,0.17907537170252064,0.19710240726771086,False,False,True
+4,20,onto MDL,50,25,6.078505843571378e-11,False,-0.0028480708687979284,0.05108666110389699,0.01577335634728748,False,True,False
+4,20,onto grow,50,50,0.23337393373979182,True,0.0022991908032037258,0.10737600575610173,0.17237490700974653,False,False,True
+4,20,onto grow and prune,50,49,0.10739430306125297,True,0.002464307340831073,0.0808543667009045,0.1403651310023357,False,False,True
+4,30,onto MDL,50,49,0.9995986807582445,True,-0.0075901048152034875,0.003505585565059998,0.0023458099258936668,True,True,True
+4,30,onto grow,50,50,0.019954056762947512,False,0.007931583424974721,0.0017779629774352657,0.006656357336094241,True,True,True
+4,30,onto grow and prune,50,50,0.05476293915034592,True,0.007585019714868717,0.004912360514200742,0.012785338260257167,True,True,True
+4,40,onto MDL,50,50,0.7849115068116961,True,-0.013763186584818309,0.0010251160635235124,0.0005228166870663649,True,True,True
+4,40,onto grow,50,50,0.5806930923150632,True,0.023495313603546935,1.8673260271384606e-07,4.996333594850335e-08,True,True,True
+4,40,onto grow and prune,50,50,0.4206885534285245,True,0.023596832681638914,1.0490046520317264e-07,2.1690100774112125e-08,True,True,True
+4,50,onto MDL,50,50,0.19786263086445077,True,0.012377517572104053,0.0002857006608838475,0.000195250913248652,True,True,True
+4,50,onto grow,50,50,0.7181090795699334,True,0.04038307981851763,9.769962616701378e-14,1.4763468981946096e-15,True,True,True
+4,50,onto grow and prune,50,50,0.839838376622123,True,0.04002551265029013,4.440892098500626e-14,1.1378238340990511e-15,True,True,True
+4,60,onto MDL,50,50,0.00017007779518874529,False,0.011745808696861594,0.0006715891126169993,0.0009754014355390173,True,True,True
+4,60,onto grow,50,50,0.009506797642009982,False,0.0196379880678945,1.1326234101716182e-06,2.5985704465769914e-06,True,True,True
+4,60,onto grow and prune,50,50,0.0035086177751395696,False,0.017810165320359315,4.268415494834699e-06,7.497444267506702e-06,True,True,True
+4,70,onto MDL,50,50,0.05775352200217879,True,0.013313863339226016,0.00021178724761483636,0.0002593627266312863,True,True,True
+4,70,onto grow,50,50,0.3240611850081543,True,0.0153133189977645,0.0001778852638985029,0.00019575378140047122,True,True,True
+4,70,onto grow and prune,50,50,0.3493197556650465,True,0.015236213226084554,0.0001858580190070569,0.00020093807436410708,True,True,True
+4,80,onto MDL,50,50,0.011672184646874428,False,0.009538181984263065,1.7496796758109667e-06,2.660665925772433e-06,True,True,True
+4,80,onto grow,50,50,0.07163371445306561,True,0.006730964676699718,0.006525205201144857,0.0112936671537753,True,True,True
+4,80,onto grow and prune,50,50,0.07176017018485106,True,0.006729895475053149,0.006525205201144857,0.01130688995101116,True,True,True
+4,90,onto MDL,50,50,0.0016783079468313173,False,0.009496456742616873,1.4422642710343325e-05,5.977539287821946e-06,True,True,True
+4,90,onto grow,50,50,0.018411528792881594,False,0.010334362648039557,5.6804637029728156e-06,3.191439749443588e-06,True,True,True
+4,90,onto grow and prune,50,50,0.018411528792881594,False,0.010334362648039557,5.6804637029728156e-06,3.191439749443588e-06,True,True,True
+4,100,onto MDL,50,50,0.02831181579857028,False,0.0062296988223423215,0.0021224348550905603,0.002394956527138198,True,True,True
+4,100,onto grow,50,50,0.0017188199546184133,False,0.010478214840442153,2.4281497463363166e-05,1.7476241983591324e-05,True,True,True
+4,100,onto grow and prune,50,50,0.0017188199546184133,False,0.010478214840442153,2.4281497463363166e-05,1.7476241983591324e-05,True,True,True

results/summary_runtimeOFF.csv

@@ -0,0 +1,89 @@
+lambda,method,percentage,train_accuracy_mean,train_accuracy_std,test_accuracy_mean,test_accuracy_std,test_f1_macro_mean,test_f1_macro_std,test_f1_weighted_mean,test_f1_weighted_std,invalid_rate_mean,invalid_rate_std,abstain_rate_mean,abstain_rate_std
+3,no guidance,0,0.6388529072640422,0.03952662244205825,0.6168897730621868,0.04753046746687279,0.3111571765455101,0.05465433344289358,0.5430189181937055,0.05771583771498859,0.005341290893015017,0.011121009344117931,0.0,0.0
+3,no guidance,10,0.5841604531561706,0.03383844510621713,0.6163073975832597,0.04464975256710303,0.31027281141220725,0.0521341543454993,0.5429104924255147,0.05321854694007122,0.005682287061597389,0.011291820342475188,0.0,0.0
+3,no guidance,20,0.5332731748384854,0.023200193908408214,0.6240645446507516,0.03901842984071439,0.315081522346614,0.04618532728352603,0.5515528475238854,0.04557390003972097,0.00465900383141761,0.012650409401920098,0.0,0.0
+3,no guidance,30,0.47158560735006133,0.02436977523807611,0.605957264957265,0.03938531423036807,0.29597118876671136,0.05248446163789167,0.5309208111882454,0.04887727964228036,0.030711170055997616,0.04370551686155085,0.0,0.0
+3,no guidance,40,0.41803003349256024,0.025944630432240386,0.5666819923371648,0.04582860026326609,0.25507505979748335,0.0557520704625289,0.49481589679384347,0.053108874518593155,0.11137665782493364,0.06455473411149996,0.0,0.0
+3,no guidance,50,0.3759064953604568,0.029713989902991844,0.4619086354258768,0.16318726692303942,0.2229824707706249,0.06402768582688166,0.42420627365758845,0.13498331712658737,0.27356086059534335,0.21742546640362373,0.0,0.0
+3,no guidance,60,0.38028276505792563,0.022242075878376382,0.3414650751547303,0.14633674381398637,0.1767813410516594,0.062063213157851745,0.35114052550672226,0.14798004750806926,0.4882357795461244,0.2024836721058042,0.0,0.0
+3,no guidance,70,0.4215418657003239,0.021352578246468818,0.08263660477453576,0.09144832449467678,0.08199014507284758,0.049443060126524094,0.07683185080554944,0.1046697265017986,0.8285564397288535,0.11552749029294476,0.0,0.0
+3,no guidance,80,0.4695820933765259,0.021938471104783374,0.05633657530209249,0.043055415361064185,0.06948308397203609,0.034347376588941124,0.05098254087441277,0.05150348280154277,0.8613672266430886,0.0542540695074589,0.0,0.0
+3,no guidance,90,0.5171427185709842,0.021914627968386224,0.05041202475685229,0.01796686881907412,0.06418170103292234,0.0324220221385047,0.04364884265900254,0.023718857531861468,0.8687674624226348,0.024352459403761777,0.0,0.0
+3,no guidance,100,0.5657463716393053,0.021732471887116047,0.05099204244031825,0.01752727337364296,0.059672726902934965,0.031099184416450123,0.041165503369427286,0.02275208118277108,0.8693501326259947,0.023992776330712732,0.0,0.0
+3,onto MDL,0,0.6389042991269972,0.03956474227468718,0.6169581491305629,0.047540587913191164,0.31110340628476313,0.05462642124996452,0.5430335005106134,0.05772026101896871,0.005341290893015017,0.011121009344117931,0.0,0.0
+3,onto MDL,10,0.5844003001519061,0.03382072858744621,0.6169575596816976,0.04465673101043061,0.31015463639611923,0.05253352000181822,0.5430515520969774,0.05372477453546684,0.005750368405540803,0.01143791743446728,0.0,0.0
+3,onto MDL,20,0.5327588901700251,0.02414935173613175,0.6233135867963454,0.03975560136909951,0.31402488531367273,0.046802187637587284,0.5502939131160357,0.046784670743862354,0.0038370173887415132,0.008648498799091454,0.0,0.0
+3,onto MDL,30,0.4641207196324969,0.02476915360224016,0.6104361921603301,0.03924070592210571,0.2903063002623682,0.05513373047044626,0.5232100052169779,0.05390191093056463,0.005310934276451503,0.010337792698777614,0.0,0.0
+3,onto MDL,40,0.3831679752557696,0.028760618604354814,0.5815081049218981,0.04125165180825187,0.23901945979860936,0.05905853590224364,0.46855825116841415,0.05868506085009354,0.010140878278809289,0.015400513351290758,0.0,0.0
+3,onto MDL,50,0.34508570801076155,0.0252668557078928,0.566325081049219,0.06562659565541913,0.2433615663338964,0.04808460663442294,0.47450017734154865,0.05351363330351805,0.08095313881520776,0.11453370205396776,0.0,0.0
+3,onto MDL,60,0.3240410695656948,0.019501558708094124,0.38466902446212786,0.1517417896585143,0.1861703343958637,0.0680359851749197,0.3492853798093988,0.1623950151573808,0.39439935160624817,0.20229429427229112,0.0,0.0
+3,onto MDL,70,0.33802198063654165,0.02177711210905344,0.1476410256410256,0.087922758622338,0.0853631946205963,0.04460085073910187,0.07908177706685779,0.10203998392266218,0.6960554081933392,0.10774386141600663,0.0,0.0
+3,onto MDL,80,0.37001800911436883,0.02068294646539826,0.12230120837017386,0.04698212164752055,0.06667463393130833,0.02727671853084857,0.04877370905088743,0.04878714759234747,0.7241682876510462,0.04841717481265179,0.0,0.0
+3,onto MDL,90,0.43877642343381096,0.05552903406518964,0.0936946654877689,0.03988059181441733,0.051835287540872896,0.02196207230918846,0.03362751250567282,0.014358657360611543,0.7728343648688476,0.06923884718886578,0.0,0.0
+3,onto MDL,100,0.5774918647852266,0.014894216388702385,0.05068464485705859,0.017583320865805593,0.03489564600809644,0.020096985112630864,0.02126942248640234,0.011269870686716825,0.8692466843501325,0.02407883718657421,0.0,0.0
+3,onto grow,0,0.6381253317227621,0.04080956243613949,0.6158252284114353,0.048876464025745164,0.3126294344505097,0.05605159042911064,0.5428218977649776,0.058407485350901706,0.005683171234895359,0.011549376445475141,0.0,0.0
+3,onto grow,10,0.5859864748622778,0.03175151433998456,0.6167073386383731,0.04106350745250583,0.31081894618720757,0.04879604941448284,0.5441982818438719,0.04954570951394121,0.005547597995873841,0.01077968049977749,0.0,0.0
+3,onto grow,20,0.5323988176943208,0.022907317120981045,0.6236298261125848,0.03706714356045253,0.3168413915709931,0.04894186328595407,0.5521045255022796,0.04607958081368585,0.004144709696433818,0.008134481832043683,0.0,0.0
+3,onto grow,30,0.4667472135287981,0.02392146548762496,0.6124686118479221,0.04070342846811778,0.3061541425230521,0.05348976567403071,0.536732205900261,0.05183356047303468,0.005375478927203048,0.010239951147170015,0.0,0.0
+3,onto grow,40,0.39175488204396125,0.02932869745863945,0.5893539640436193,0.045056650671004346,0.27516837247487835,0.06495083672893301,0.5015573207934746,0.06453724457915029,0.009898320070733836,0.014264502862381177,0.0,0.0
+3,onto grow,50,0.35006021340068444,0.024397244009246127,0.5771072796934866,0.06423575349111744,0.2694087209984639,0.05674468546241094,0.4985782539534183,0.06111908849558393,0.06695579133510166,0.10434345547721596,0.0,0.0
+3,onto grow,60,0.32384434195354966,0.020354708177710786,0.3995405246094901,0.14863179009158387,0.19552970813701592,0.06940080733921404,0.3653839165810933,0.15629331193853263,0.37306424992631887,0.19901050586716207,0.0,0.0
+3,onto grow,70,0.33736322041032957,0.021573351239635347,0.15176186265841435,0.09689921031843843,0.086123153614651,0.04649224475185513,0.08309762630348504,0.10986832857963429,0.6893616268788683,0.11705166458239832,0.0,0.0
+3,onto grow,80,0.36961521989787516,0.0175732456701891,0.11842587680518711,0.03467939098263886,0.06405440472817552,0.023406683297188428,0.043444080738791295,0.026985786236127463,0.727497789566755,0.028915814754404294,0.0,0.0
+3,onto grow,90,0.40181450978238986,0.01928214680643499,0.1138296492779251,0.035232926148552376,0.05768099888717647,0.02307373711883154,0.03910824051605616,0.025946728552540856,0.7281134689065724,0.025964626443445323,0.0,0.0
+3,onto grow,100,0.4377397097311444,0.014450409394694195,0.1082552313586796,0.03444699640517807,0.05013518609951471,0.02095578386381706,0.033110210169585916,0.01436419126420106,0.7288989095195992,0.016674051989307395,0.0,0.0
+3,onto grow and prune,0,0.6381253317227621,0.04080956243613949,0.6158252284114353,0.048876464025745164,0.3126294344505097,0.05605159042911064,0.5428218977649776,0.058407485350901706,0.005683171234895359,0.011549376445475141,0.0,0.0
+3,onto grow and prune,10,0.5861231720931935,0.031741563441969774,0.6168803418803419,0.0408818040434855,0.3111570646673489,0.048770143757419844,0.5444861932018557,0.04942949858839964,0.005547597995873841,0.010861021374355826,0.0,0.0
+3,onto grow and prune,20,0.5325185673237065,0.023009209560660583,0.6237320954907163,0.03725590341835852,0.3170496968168371,0.04890198179453058,0.552365893120093,0.04619522545981799,0.00407633362805775,0.008133106229761213,0.0,0.0
+3,onto grow and prune,30,0.4664902725159684,0.024073178957570576,0.6119218980253464,0.04080913476895638,0.3054588088467814,0.0538902573865484,0.5358917881264212,0.052304159229899255,0.005238726790450911,0.010167603199646172,0.0,0.0
+3,onto grow and prune,40,0.39158357583411113,0.02913669440606561,0.5890120837017389,0.04510444546200834,0.27483086625745007,0.06438434574018938,0.5010514436802922,0.06400022174413517,0.009966696139109905,0.01431943031614745,0.0,0.0
+3,onto grow and prune,50,0.35010309485898344,0.024408465502308827,0.576525198938992,0.06327509040233555,0.2682629669304948,0.05650474919277487,0.4975898622670888,0.06044029317242812,0.06753698791629822,0.10333977533383669,0.0,0.0
+3,onto grow and prune,60,0.323647303208331,0.020169662230595662,0.4005010315355143,0.14909051931499792,0.19547459379955173,0.06971643955902587,0.3657663753293993,0.1568613373266914,0.3716917182434424,0.1999898440561303,0.0,0.0
+3,onto grow and prune,70,0.337286132615897,0.021612815542245676,0.15172767462422632,0.09679698502592989,0.08605265191555507,0.04630512210036064,0.0830329068749528,0.10966114635864001,0.6893958149130562,0.11696351291277206,0.0,0.0
+3,onto grow and prune,80,0.3695638280349201,0.017558138948158167,0.11842587680518711,0.03467939098263886,0.06405343573592745,0.023408051765538476,0.04344333536013894,0.026986326962535563,0.727497789566755,0.028915814754404294,0.0,0.0
+3,onto grow and prune,90,0.4017716832299273,0.01928280277829113,0.1138296492779251,0.035232926148552376,0.05768002989492841,0.02307485661809885,0.0391074951374038,0.025947165877646165,0.7281134689065724,0.025964626443445323,0.0,0.0
+3,onto grow and prune,100,0.4377397097311444,0.014450409394694195,0.1082552313586796,0.03444699640517807,0.05013518609951471,0.02095578386381706,0.033110210169585916,0.01436419126420106,0.7288989095195992,0.016674051989307395,0.0,0.0
+4,no guidance,0,0.6388529072640422,0.03952662244205825,0.6168897730621868,0.04753046746687279,0.3111571765455101,0.05465433344289358,0.5430189181937055,0.05771583771498859,0.005341290893015017,0.011121009344117931,0.0,0.0
+4,no guidance,10,0.5841604531561706,0.03383844510621713,0.6163073975832597,0.04464975256710303,0.31027281141220725,0.0521341543454993,0.5429104924255147,0.05321854694007122,0.005682287061597389,0.011291820342475188,0.0,0.0
+4,no guidance,20,0.5332731748384854,0.023200193908408214,0.6240645446507516,0.03901842984071439,0.315081522346614,0.04618532728352603,0.5515528475238854,0.04557390003972097,0.00465900383141761,0.012650409401920098,0.0,0.0
+4,no guidance,30,0.47158560735006133,0.02436977523807611,0.605957264957265,0.03938531423036807,0.29597118876671136,0.05248446163789167,0.5309208111882454,0.04887727964228036,0.030711170055997616,0.04370551686155085,0.0,0.0
+4,no guidance,40,0.41803003349256024,0.025944630432240386,0.5666819923371648,0.04582860026326609,0.25507505979748335,0.0557520704625289,0.49481589679384347,0.053108874518593155,0.11137665782493364,0.06455473411149996,0.0,0.0
+4,no guidance,50,0.3759064953604568,0.029713989902991844,0.4619086354258768,0.16318726692303942,0.2229824707706249,0.06402768582688166,0.42420627365758845,0.13498331712658737,0.27356086059534335,0.21742546640362373,0.0,0.0
+4,no guidance,60,0.38028276505792563,0.022242075878376382,0.3414650751547303,0.14633674381398637,0.1767813410516594,0.062063213157851745,0.35114052550672226,0.14798004750806926,0.4882357795461244,0.2024836721058042,0.0,0.0
+4,no guidance,70,0.4215418657003239,0.021352578246468818,0.08263660477453576,0.09144832449467678,0.08199014507284758,0.049443060126524094,0.07683185080554944,0.1046697265017986,0.8285564397288535,0.11552749029294476,0.0,0.0
+4,no guidance,80,0.4695820933765259,0.021938471104783374,0.05633657530209249,0.043055415361064185,0.06948308397203609,0.034347376588941124,0.05098254087441277,0.05150348280154277,0.8613672266430886,0.0542540695074589,0.0,0.0
+4,no guidance,90,0.5171427185709842,0.021914627968386224,0.05041202475685229,0.01796686881907412,0.06418170103292234,0.0324220221385047,0.04364884265900254,0.023718857531861468,0.8687674624226348,0.024352459403761777,0.0,0.0
+4,no guidance,100,0.5657463716393053,0.021732471887116047,0.05099204244031825,0.01752727337364296,0.059672726902934965,0.031099184416450123,0.041165503369427286,0.02275208118277108,0.8693501326259947,0.023992776330712732,0.0,0.0
+4,onto MDL,0,0.6390327787843847,0.03962916431645832,0.617026525198939,0.04759994049887874,0.31086586291060103,0.05426243203391457,0.5429549709632112,0.05767007097597026,0.005341290893015017,0.011121009344117931,0.0,0.0
+4,onto MDL,10,0.5844003001519061,0.03382072858744621,0.6169575596816976,0.04465673101043061,0.31015463639611923,0.05253352000181822,0.5430515520969774,0.05372477453546684,0.005750368405540803,0.01143791743446728,0.0,0.0
+4,onto MDL,20,0.5326562894635699,0.024125271396599066,0.6230730916592986,0.03975999819558803,0.31260664490205353,0.047508395931406606,0.5491447756242062,0.04732378145198322,0.0037683465959327893,0.008610592285241303,0.0,0.0
+4,onto MDL,30,0.46318737531799625,0.025081449398492647,0.609545829649278,0.039096042682294045,0.28701421285051487,0.05488274046368217,0.5201925859638176,0.05393363322649409,0.005241968759210124,0.01034426165287532,0.0,0.0
+4,onto MDL,40,0.38324544739105765,0.0282727831760543,0.582259946949602,0.04136927214909046,0.2388994756316518,0.05849067819948041,0.4686369749780094,0.05805228811276696,0.009731800766283502,0.014897949259069286,0.0,0.0
+4,onto MDL,50,0.34537749592781714,0.024492251034080523,0.5658986147951665,0.06739542578876698,0.24344399172230424,0.048598015891434046,0.47454019282231713,0.05408928336625079,0.08224580017683464,0.11766656799547058,0.0,0.0
+4,onto MDL,60,0.3240754589212833,0.019645348865576426,0.38924049513704684,0.14798775185027715,0.18843049387322489,0.06705535149838679,0.35528395020050346,0.15820146151574074,0.3893854995579133,0.1973250376248378,0.0,0.0
+4,onto MDL,70,0.33773945350390744,0.0219390774942566,0.1494868847627468,0.09049896838925954,0.0860820809957927,0.04550775237002938,0.0814012755509098,0.10508623194715755,0.6935932802829354,0.11026337198217985,0.0,0.0
+4,onto MDL,80,0.3690242683797288,0.019409246073807503,0.12267727674624222,0.04834158617049805,0.06721385248192649,0.02841301292039623,0.04924410470858079,0.04946439205487909,0.7227323902151488,0.049703081148738754,0.0,0.0
+4,onto MDL,90,0.40140374091765957,0.017025635554982783,0.11273533745947537,0.03083665959413659,0.05710082646966416,0.0210768396393462,0.03770507506900629,0.01395678230955511,0.7292416740347775,0.016536345801261676,0.0,0.0
+4,onto MDL,100,0.4625975036146342,0.05615960312316669,0.09545269672855874,0.04150746491313745,0.04694982021392166,0.02291462726277543,0.0305061636858996,0.015323428924184194,0.7608405540819334,0.06295490566079402,0.0,0.0
+4,onto grow,0,0.6386216804846355,0.04060650245059486,0.6164096669613911,0.0494181191173883,0.31274669903102703,0.05598721953629108,0.5429788780348631,0.05865102339282895,0.005751547303271427,0.01166720371006383,0.0,0.0
+4,onto grow,10,0.5857459872984498,0.03273934087936657,0.6164759799587386,0.04343144814291895,0.30939237113274465,0.050730760786082234,0.543267575941218,0.05178322981652921,0.0056828765104627,0.011188019007482569,0.0,0.0
+4,onto grow,20,0.5324330972382364,0.02273500363988187,0.6237312113174183,0.03709180930681595,0.3179087841165954,0.048964187959505426,0.5526823757780325,0.04610988884512442,0.0043162393162393,0.008279306081572603,0.0,0.0
+4,onto grow,30,0.4660966891780596,0.024272210107898904,0.6119177718832891,0.04072686268735373,0.30294936656264776,0.05367264115599525,0.5340417912770246,0.05212923451265871,0.005307102858826979,0.010362466882154626,0.0,0.0
+4,onto grow,40,0.3918738080857995,0.028734161291797098,0.5889463601532567,0.045461636497013326,0.2720815036174336,0.06450926425305109,0.4990106532588867,0.06454449471279136,0.00945151783082815,0.014650051272554858,0.0,0.0
+4,onto grow,50,0.34900680832372477,0.024951769001070104,0.5772699675803123,0.06308417294889297,0.2658422869178436,0.05505674159052636,0.4955518763995069,0.05983450707126211,0.06556793398172706,0.10295877976419242,0.0,0.0
+4,onto grow,60,0.3238615640842621,0.019884383500389,0.3945882699675803,0.15080650380176883,0.1937558651282168,0.07198743125722211,0.36063856293040997,0.16056660274574883,0.37946360153256703,0.20185988114126016,0.0,0.0
+4,onto grow,70,0.3373974084445176,0.02160764216433364,0.15169319186560562,0.09685485766102156,0.08581070171452818,0.04630021149701852,0.08292057163824566,0.10977758046151875,0.689429413498379,0.11708733119221992,0.0,0.0
+4,onto grow,80,0.36949539706070755,0.017510276702376138,0.11866637194223398,0.034721714869617594,0.06426563212964537,0.023429416967887615,0.043581649808420414,0.027010010634303885,0.727497789566755,0.028915814754404294,0.0,0.0
+4,onto grow,90,0.4016946320393857,0.019291160604968875,0.11386383731211314,0.035226096903687644,0.05788235273124956,0.02314628194281105,0.03920552629313851,0.02595600380065336,0.7281134689065724,0.025964626443445323,0.0,0.0
+4,onto grow,100,0.43753421548710625,0.014396419013920017,0.10828941939286763,0.034471108780613005,0.050317693376642963,0.02104961618314362,0.03321152834368946,0.014426827211870533,0.7288989095195992,0.016674051989307395,0.0,0.0
+4,onto grow and prune,0,0.6386216804846355,0.04060650245059486,0.6164096669613911,0.0494181191173883,0.31274669903102703,0.05598721953629108,0.5429788780348631,0.05865102339282895,0.005751547303271427,0.01166720371006383,0.0,0.0
+4,onto grow and prune,10,0.5860368601182306,0.032443870853861086,0.6171959917477159,0.04226668292344806,0.3098844416021374,0.050652922421797336,0.5438837535481813,0.05139732362829782,0.0056828765104627,0.011266412158656466,0.0,0.0
+4,onto grow and prune,20,0.5325442815571295,0.02282635779141224,0.6238334806955496,0.037288033775284064,0.31811759956009,0.04892256844366508,0.5529441641848359,0.04622456519306627,0.004247863247863232,0.008279376951686023,0.0,0.0
+4,onto grow and prune,30,0.465882446604074,0.0242204197494982,0.611405540819334,0.04084295726780461,0.30234053365828484,0.053964518196367474,0.5332877994047854,0.052512880190722165,0.00513616268788681,0.010264459882689472,0.0,0.0
+4,onto grow and prune,40,0.3916767876425264,0.028526544756113124,0.5887421161214265,0.04550705412570358,0.27188874365254934,0.0636462964939437,0.49864050462676635,0.06378134675594277,0.009485705865016184,0.014717825456109787,0.0,0.0
+4,onto grow and prune,50,0.3489640732809897,0.02492443602732733,0.5774748010610079,0.06270398703735681,0.2654643396892027,0.05574172225143165,0.4954792076795703,0.06014204181900106,0.06549955791335098,0.10179916451245527,0.0,0.0
+4,onto grow and prune,60,0.32357887223411846,0.019756997334974675,0.39493339227821983,0.1507742388714561,0.19267546795527862,0.07098952176510183,0.3601022141364179,0.16038563041855572,0.37959534335396405,0.20175336710988606,0.0,0.0
+4,onto grow and prune,70,0.3373203206500851,0.02164716618832977,0.15165900383141762,0.09675256114446727,0.08573906952447617,0.04611187434730983,0.08285498260128568,0.10957034521970342,0.689463601532567,0.11699918650787963,0.0,0.0
+4,onto grow and prune,80,0.3694440051977525,0.017494762211385916,0.11866637194223398,0.034721714869617594,0.0642646631373973,0.02343079287898333,0.04358090442976806,0.027010554687340916,0.727497789566755,0.028915814754404294,0.0,0.0
+4,onto grow and prune,90,0.4016946320393857,0.019291160604968875,0.11386383731211314,0.035226096903687644,0.05788235273124956,0.02314628194281105,0.03920552629313851,0.02595600380065336,0.7281134689065724,0.025964626443445323,0.0,0.0
+4,onto grow and prune,100,0.43753421548710625,0.014396419013920017,0.10828941939286763,0.034471108780613005,0.050317693376642963,0.02104961618314362,0.03321152834368946,0.014426827211870533,0.7288989095195992,0.016674051989307395,0.0,0.0