* permimp_frames.do
*
* Table 3 of "Which H2O frame should you use to explain your model?"
*
* Permutation importance on the training frame against the testing frame,
* with the same fitted model and the same number of scored observations on
* both sides, so that any gap reflects the frame and not the sample size.
*
*
* Reports importance as a share of the within-frame total. With reps() > 1
* the raw importance is on the loss scale, and testing loss is larger than
* training loss, so the raw columns do not have the same total and raw
* differences across frames are not comparable.

clear all
set more off
// number of simulations
local nsim 10

capture program drop makedata
program define makedata
    args n seed
    clear
    set seed `seed'
    set obs `n'
    gen double x1 = runiform(-3,3)
    gen double x2 = rnormal()
    gen double x3 = 0.9*x2 + sqrt(1-0.9^2)*rnormal()
    gen double x4 = runiform(-2,2)
    gen byte   g  = runiform()<0.5
    gen double x5 = runiform()
    gen byte   x6 = runiform()<0.5
    gen double y  = 4*sin(1.5*x1) + 2*x2 + 3*x4*(2*g-1) + rnormal(0,1.5)
end

h2o init
tempfile res
postfile pf int rep str2 frame str9 vname double imp using `res'

forvalues r = 1/`nsim' {
    makedata 4000 `=20260801 + `r''

    quietly _h2oframe put, into(all`r')
    quietly _h2oframe split all`r', into(tr`r' va`r' te`r') ///
        split(0.6 0.2 0.2) rseed(19)
    quietly _h2oframe change tr`r'

    quietly h2oml gbregress y x1 x2 x3 x4 g x5 x6, ///
        ntrees(500) maxdepth(12) minobsleaf(1) lrate(0.1) h2orseed(19)

    * samples(700) on BOTH frames, so the two are scored on equal numbers
    foreach fr in tr te {
        quietly h2omlgraph permimp, frame(`fr'`r') reps(10) samples(700) ///
            h2orseed(19) savedata(_pi, replace) nodraw
        preserve
            quietly use _pi, clear
            quietly collapse (mean) importance, by(variable)
            forvalues i = 1/`=_N' {
                post pf (`r') ("`fr'") (variable[`i']) (importance[`i'])
            }
        restore
    }
    di "rep `r' done"
}
postclose pf

use `res', clear
save pires_split, replace

di _n "===== RAW importance (loss scale), mean over `nsim' reps ====="
di    "  the column totals below are the reason we report shares"
table vname frame, statistic(mean imp) nformat(%8.3f)

di _n "===== RAW column totals per frame, mean over reps ====="
preserve
    collapse (sum) imp, by(rep frame)
    table frame, statistic(mean imp) nformat(%8.1f)
restore

bysort rep frame: egen double tot = total(imp)
gen double share = 100*imp/tot

di _n "===== TABLE 3: within-frame share (%), mean over `R' reps ====="
table vname frame, statistic(mean share) nformat(%7.2f)

di _n "===== per-rep count: training share exceeds testing share ====="
keep rep frame vname share
quietly reshape wide share, i(rep vname) j(frame) string
gen double diff = sharetr - sharete
gen byte wins = sharetr > sharete
table vname, statistic(mean sharete) statistic(mean sharetr) ///
    statistic(mean diff) statistic(sum wins) nformat(%7.2f)

di "----permimp_frames.do successful----"