Rolling origin evaluation on validation set (time series)
eval_ts(
y,
x = NULL,
fit_func = crossvalidation::fit_lm,
predict_func = crossvalidation::predict_lm,
fcast_func = NULL,
fit_params = NULL,
q = 0.2,
initial_window = 5,
horizon = 3,
fixed_window = TRUE,
level = c(80, 95),
seed = 123,
eval_metric = NULL,
cl = NULL,
errorhandling = c("stop", "remove", "pass"),
packages = c("stats", "Rcpp"),
verbose = FALSE,
show_progress = TRUE,
...
)response time series; a vector or a matrix
input covariates' matrix (optional) for ML models
a function for fitting the model (if validation of ML model)
a function for predicting values from the model (if validation of ML model)
time series forecasting function (e.g forecast::thetaf)
a list; additional (model-specific) parameters to be passed
to fit_func
a float; percentage of original data in the validation test.
an integer; the initial number of consecutive values in each training set sample
an integer; the number of consecutive values in test set sample
a boolean; if FALSE, all training samples start at 1
a numeric vector; confidence levels for prediction intervals.
random seed for reproducibility of results
a function measuring the test errors; if not provided: RMSE for regression and accuracy for classification
an integer; the number of clusters for parallel execution
specifies how a task evalution error should be handled. If value is "stop", then execution will be stopped if an error occurs. If value is "remove", the result for that task will not be returned. If value is "pass", then the error object generated by task evaluation will be included with the rest of the results. The default value is "stop".
character vector of packages that the tasks depend on
logical flag enabling verbose messages. This can be very useful for troubleshooting.
show evolution of the algorithm
additional parameters
require(forecast)
data("AirPassengers")
# Example 1 -----
res <- eval_ts(y=AirPassengers, initial_window = 10,
horizon = 3, fcast_func = forecast::thetaf)
#>
|
| | 0%
|
|== | 3%
|
|===== | 7%
|
|======= | 10%
|
|========== | 14%
|
|============ | 17%
|
|============== | 21%
|
|================= | 24%
|
|=================== | 28%
|
|====================== | 31%
|
|======================== | 34%
|
|=========================== | 38%
|
|============================= | 41%
|
|=============================== | 45%
|
|================================== | 48%
|
|==================================== | 52%
|
|======================================= | 55%
|
|========================================= | 59%
|
|=========================================== | 62%
|
|============================================== | 66%
|
|================================================ | 69%
|
|=================================================== | 72%
|
|===================================================== | 76%
|
|======================================================== | 79%
|
|========================================================== | 83%
|
|============================================================ | 86%
|
|=============================================================== | 90%
|
|================================================================= | 93%
|
|==================================================================== | 97%
|
|======================================================================| 100%
print(colMeans(res))
#> ME RMSE MAE MPE MAPE
#> -1.57869221 90.05413641 82.58512560 0.02507898 0.18879144
# Example 2 -----
if (FALSE) {
fcast_func <- function (y, h, ...)
{
forecast::forecast(forecast::auto.arima(y, ...),
h=h, ...)
}
res <- eval_ts(y=AirPassengers, initial_window = 10, horizon = 3,
fcast_func = fcast_func)
print(colMeans(res))
}
# Example 3 -----
fcast_func <- function (y, h, ...)
{
forecast::forecast(forecast::ets(y, ...),
h=h, ...)
}
res <- eval_ts(y=AirPassengers,
initial_window = 10, horizon = 3, fcast_func = fcast_func)
#>
|
| | 0%
|
|== | 3%
|
|===== | 7%
|
|======= | 10%
|
|========== | 14%
|
|============ | 17%
|
|============== | 21%
|
|================= | 24%
|
|=================== | 28%
|
|====================== | 31%
|
|======================== | 34%
|
|=========================== | 38%
|
|============================= | 41%
|
|=============================== | 45%
|
|================================== | 48%
|
|==================================== | 52%
|
|======================================= | 55%
|
|========================================= | 59%
|
|=========================================== | 62%
|
|============================================== | 66%
|
|================================================ | 69%
|
|=================================================== | 72%
|
|===================================================== | 76%
|
|======================================================== | 79%
|
|========================================================== | 83%
|
|============================================================ | 86%
|
|=============================================================== | 90%
|
|================================================================= | 93%
|
|==================================================================== | 97%
|
|======================================================================| 100%
print(colMeans(res))
#> ME RMSE MAE MPE MAPE
#> 2.28306322 77.73500334 70.67642841 0.01311649 0.16116931
# Example 4 -----
xreg <- cbind(1, 1:length(AirPassengers))
res <- eval_ts(y=AirPassengers, x=xreg,
fit_func = crossvalidation::fit_lm,
predict_func = crossvalidation::predict_lm,
initial_window = 10,
horizon = 3,
fixed_window = TRUE)
#>
|
| | 0%
|
|== | 3%
|
|===== | 7%
|
|======= | 10%
|
|========== | 14%
|
|============ | 17%
|
|============== | 21%
|
|================= | 24%
|
|=================== | 28%
|
|====================== | 31%
|
|======================== | 34%
|
|=========================== | 38%
|
|============================= | 41%
|
|=============================== | 45%
|
|================================== | 48%
|
|==================================== | 52%
|
|======================================= | 55%
|
|========================================= | 59%
|
|=========================================== | 62%
|
|============================================== | 66%
|
|================================================ | 69%
|
|=================================================== | 72%
|
|===================================================== | 76%
|
|======================================================== | 79%
|
|========================================================== | 83%
|
|============================================================ | 86%
|
|=============================================================== | 90%
|
|================================================================= | 93%
|
|==================================================================== | 97%
|
|======================================================================| 100%
print(colMeans(res))
#> ME RMSE MAE MPE MAPE
#> -3.43845350 125.30041052 118.60947405 0.04494995 0.27387743
# Example 5 -----
res <- eval_ts(y=AirPassengers, fcast_func = forecast::thetaf,
initial_window = 10,
horizon = 3,
fixed_window = TRUE)
#>
|
| | 0%
|
|== | 3%
|
|===== | 7%
|
|======= | 10%
|
|========== | 14%
|
|============ | 17%
|
|============== | 21%
|
|================= | 24%
|
|=================== | 28%
|
|====================== | 31%
|
|======================== | 34%
|
|=========================== | 38%
|
|============================= | 41%
|
|=============================== | 45%
|
|================================== | 48%
|
|==================================== | 52%
|
|======================================= | 55%
|
|========================================= | 59%
|
|=========================================== | 62%
|
|============================================== | 66%
|
|================================================ | 69%
|
|=================================================== | 72%
|
|===================================================== | 76%
|
|======================================================== | 79%
|
|========================================================== | 83%
|
|============================================================ | 86%
|
|=============================================================== | 90%
|
|================================================================= | 93%
|
|==================================================================== | 97%
|
|======================================================================| 100%
print(colMeans(res))
#> ME RMSE MAE MPE MAPE
#> -1.57869221 90.05413641 82.58512560 0.02507898 0.18879144
#' # Example 6 -----
xreg <- cbind(1, 1:length(AirPassengers))
res <- eval_ts(y=AirPassengers, x=xreg,
fit_func = crossvalidation::fit_lm,
predict_func = crossvalidation::predict_lm,
initial_window = 10,
horizon = 3,
fixed_window = TRUE)
#>
|
| | 0%
|
|== | 3%
|
|===== | 7%
|
|======= | 10%
|
|========== | 14%
|
|============ | 17%
|
|============== | 21%
|
|================= | 24%
|
|=================== | 28%
|
|====================== | 31%
|
|======================== | 34%
|
|=========================== | 38%
|
|============================= | 41%
|
|=============================== | 45%
|
|================================== | 48%
|
|==================================== | 52%
|
|======================================= | 55%
|
|========================================= | 59%
|
|=========================================== | 62%
|
|============================================== | 66%
|
|================================================ | 69%
|
|=================================================== | 72%
|
|===================================================== | 76%
|
|======================================================== | 79%
|
|========================================================== | 83%
|
|============================================================ | 86%
|
|=============================================================== | 90%
|
|================================================================= | 93%
|
|==================================================================== | 97%
|
|======================================================================| 100%
print(colMeans(res))
#> ME RMSE MAE MPE MAPE
#> -3.43845350 125.30041052 118.60947405 0.04494995 0.27387743
# Example 7 -----
x <- ts(matrix(rnorm(50), nrow = 25))
fcast_func <- function(y, h = 5, type_forecast=c("mean", "median"))
{
type_forecast <- match.arg(type_forecast)
if (type_forecast == "mean")
{
means <- colMeans(y)
return(list(mean = t(replicate(n = h, expr = means))))
} else {
medians <- apply(y, 2, median)
return(list(mean = t(replicate(n = h, expr = medians))))
}
}
print(fcast_func(x))
#> $mean
#> Series 1 Series 2
#> [1,] -0.1348717 -0.05683619
#> [2,] -0.1348717 -0.05683619
#> [3,] -0.1348717 -0.05683619
#> [4,] -0.1348717 -0.05683619
#> [5,] -0.1348717 -0.05683619
#>
res <- crossvalidation::eval_ts(y = x, fcast_func = fcast_func,
fit_params = list(type_forecast = "median"))
#>
|
| | 0%
|
|============== | 20%
|
|============================ | 40%
|
|========================================== | 60%
|
|======================================================== | 80%
|
|======================================================================| 100%
colMeans(res)
#> ME RMSE MAE MPE MAPE
#> 0.1330267 0.9143332 0.6842368 9.0203227 15.3930320
res <- crossvalidation::eval_ts(y = x, fcast_func = fcast_func,
fit_params = list(type_forecast = "mean"))
#>
|
| | 0%
|
|============== | 20%
|
|============================ | 40%
|
|========================================== | 60%
|
|======================================================== | 80%
|
|======================================================================| 100%
colMeans(res)
#> ME RMSE MAE MPE MAPE
#> 0.1859247 0.9017510 0.6813851 8.0465056 12.7188156
# Example 8 -----
eval_metric <- function(predicted, observed)
{
error <- observed - predicted
res <- apply(error, 2, function(x) sqrt(mean(x ^ 2, na.rm = FALSE)))
return(res)
}
res <- crossvalidation::eval_ts(y = x, fcast_func = fcast_func,
fit_params = list(type_forecast = "mean"), eval_metric = eval_metric)
#>
|
| | 0%
|
|============== | 20%
|
|============================ | 40%
|
|========================================== | 60%
|
|======================================================== | 80%
|
|======================================================================| 100%
colMeans(res)
#> Series 1 Series 2
#> 1.096814 0.518267