External validation and transportability reporting

development <- simulate_gazepoint_governed_data(18L, 6L, 1L, seed = 2901L)
predictors <- c("tracking_ratio", "blink_rate", "gaze_dispersion")
task <- create_gazepoint_synthetic_task(development, "recording_quality", "new_participants")
manifest <- create_gazepoint_synthetic_manifest(task$outcome, predictors)
folds <- create_gazepoint_group_folds(
  development, task$outcome, predictors, manifest,
  task$generalization_target,
  "participant_id", "trial_id", "stimulus_id",
  v = 3L, repeats = 1L, seed = 2901L
)
development_evaluation <- evaluate_gazepoint_group_folds(
  folds, task, predictors, "glm", seed = 2901L,
  assess_calibration = TRUE, calibration_bootstrap = 0L
)
model <- fit_gazepoint_model(
  development, task, predictors, "glm", seed = 2901L
)

external <- simulate_gazepoint_governed_data(10L, 6L, 1L, seed = 2902L)
external$participant_id <- paste0("E", external$participant_id)
external$trial_id <- paste0("E", external$trial_id)
external$stimulus_id <- paste0("E", external$stimulus_id)
declaration <- declare_gazepoint_external_dataset(
  external,
  label = "synthetic_external_site",
  independent = TRUE,
  origin = "Independent deterministic synthetic generation"
)
report <- evaluate_gazepoint_external_transportability(
  model,
  development_data = development,
  external_data = external,
  declaration = declaration,
  development_evaluation = development_evaluation,
  bootstrap = 0L,
  seed = 2902L
)
report
#> <gp3ml_transportability_report>
#>   Status: externally_validated
#>   Reason: The explicitly independent dataset passed schema and identifier-overlap gates.
#>                       metric development_estimate external_estimate
#>                     accuracy           0.87037037         0.8833333
#>            balanced_accuracy           0.49425287         0.5000000
#>                  sensitivity           0.00000000         0.0000000
#>                  specificity           0.98850575         1.0000000
#>                    precision           0.00000000                NA
#>                       recall           0.00000000         0.0000000
#>                           f1                  NaN                NA
#>                          mcc          -0.08304548                NA
#>                      roc_auc           0.57340641         0.7277628
#>                       pr_auc           0.30964591         0.3291055
#>                        brier           0.11055175         0.0956074
#>                     log_loss           0.39533485         0.3260348
#>                          ece           0.11646202                NA
#>    calibration_intercept_abs           3.23095109                NA
#>  calibration_slope_abs_error           1.47811645                NA
#>    difference
#>   0.012962963
#>   0.005747126
#>   0.000000000
#>   0.011494253
#>            NA
#>   0.000000000
#>            NA
#>            NA
#>   0.154356393
#>   0.019459639
#>  -0.014944353
#>  -0.069300020
#>            NA
#>            NA
#>            NA
report$performance_comparison
#>                         metric development_estimate external_estimate
#> 1                     accuracy           0.87037037         0.8833333
#> 2            balanced_accuracy           0.49425287         0.5000000
#> 3                  sensitivity           0.00000000         0.0000000
#> 4                  specificity           0.98850575         1.0000000
#> 5                    precision           0.00000000                NA
#> 6                       recall           0.00000000         0.0000000
#> 7                           f1                  NaN                NA
#> 8                          mcc          -0.08304548                NA
#> 9                      roc_auc           0.57340641         0.7277628
#> 10                      pr_auc           0.30964591         0.3291055
#> 11                       brier           0.11055175         0.0956074
#> 12                    log_loss           0.39533485         0.3260348
#> 13                         ece           0.11646202                NA
#> 14   calibration_intercept_abs           3.23095109                NA
#> 15 calibration_slope_abs_error           1.47811645                NA
#>      difference
#> 1   0.012962963
#> 2   0.005747126
#> 3   0.000000000
#> 4   0.011494253
#> 5            NA
#> 6   0.000000000
#> 7            NA
#> 8            NA
#> 9   0.154356393
#> 10  0.019459639
#> 11 -0.014944353
#> 12 -0.069300020
#> 13           NA
#> 14           NA
#> 15           NA
report$group_coverage
#>          unit     identifier development_groups external_groups
#> 1 participant participant_id                 18              10
#> 2    stimulus    stimulus_id                  6               6
#>   overlapping_groups external_novel_groups external_coverage_prop status
#> 1                  0                    10                      1   pass
#> 2                  0                     6                      1   pass

not_validated <- evaluate_gazepoint_external_transportability(
  model,
  development_data = development,
  external_data = NULL
)
not_validated
#> <gp3ml_transportability_report>
#>   Status: not_externally_validated
#>   Reason: No independent external dataset was supplied.