generate_data_for_scenarios.Rmd

---
title: "Untitled"
output: html_document
date: "2023-11-09"
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
# Set the working directory
project_dir = "E:/13 Gapfill_scenario/"
source_dir = "E:/source/"; setwd(source_dir); source("calculate_metrics.R")
setwd(project_dir)
library(data.table)
library(tidyverse)
library(foreach)
library(doParallel)
```
# read in data from 2010 to 2022
```{r}
# use ANN data
{
  site_ref <- data.table::fread(file.path(project_dir, "output.csv"))
  site_ref <- subset(site_ref, Year > 2009)
  site_data <- data.table::fread(file.path(project_dir, "ANN_site_data.csv"))
  names(site_data)
  site_data <- subset(site_data, Year > 2009)
  site_data$DOY_sin <- sin((site_data$DoY-1)*(2*pi/12))
  site_data$DOY_cos <- cos((site_data$DoY-1)*(2*pi/12))
  nrow(site_data)
  site_data$INDEX = 1:nrow(site_data)
  tail(site_data)
  site_data$NEE_U50_orig = site_data$modelled_NEE
  sum(is.na(site_data$NEE_U50_orig)) # should be 0
  unique(site_data$Year)
  
  site_ref_subset = site_ref[, c("X","TIMESTAMP_END", "Year", "Hour", "Date", "Min", "Month", "DoY", "Time")]
  tmp = left_join(site_ref_subset, site_data)
  site_data = tmp
}

# use BART data
{
  site_data <- data.table::fread(file.path(project_dir, "output.csv"))
  names(site_data)
  site_data <- subset(site_data, Year > 2009)
  site_data$DOY_sin <- sin((site_data$DoY-1)*(2*pi/12))
  site_data$DOY_cos <- cos((site_data$DoY-1)*(2*pi/12))
  nrow(site_data)
  site_data$INDEX = 1:nrow(site_data)
  tail(site_data)
}
```
# experimental scenario 1: gap lengths
```{r}
site_data_dir =  project_dir

# Specify the number of cores to use
num_cores <- detectCores()
# Register parallel backend
cl <- makeCluster(num_cores)
registerDoParallel(cl)

foreach(scenario_num = 1:13) %dopar% {
  library(data.table)
  scenario <- paste0("s", scenario_num)
  site_data_dir <- paste0(site_data_dir, scenario)
  if (!dir.exists(data_dir)) {
    dir.create(data_dir, recursive = TRUE)
  }
  # data_dir <- paste0(site_data_dir, "/ANN_data_train_test") # change wd here
  data_dir <- paste0(site_data_dir, "/data_train_test") # change wd here
  
  if (!dir.exists(data_dir)) {
    dir.create(data_dir, recursive = TRUE)
  }
  
  setwd(data_dir)
  fold_factor <- c(1/5, 1, 3, 6, 8, 10, 12,14,16, 18, 24, 36, 48)
  fold_number <- rep(rep(1:10, each = fold_factor[scenario_num] * 5 * 48), length.out = nrow(site_data))
  site_data$fold_number <- fold_number
  # plot(site_data$fold_number)
  
  for (j in 1:10) { # 10 fold CV
    train <- site_data  # Create a deep copy of site_data for train
    # plot(as.Date(site_data$Date), site_data$NEE_U50_orig)
    train$NEE_U50_orig <- ifelse(train$fold_number != j, train$NEE_U50_orig, NA)
    fwrite(train, file = paste0("train", j, ".csv"), row.names = FALSE)

    test <- site_data  # Create a deep copy of site_data for test
    test$NEE_U50_orig <- ifelse(test$fold_number == j, test$NEE_U50_orig, NA)
    fwrite(test, file = paste0("test", j, ".csv"), row.names = FALSE)
 
  }
}
stopCluster(cl)
```

# experimental scenario 2: gap locations
```{r}
scenario <- paste0("gap_location")
site_data_dir <- paste0(project_dir, scenario)
if (!dir.exists(site_data_dir)) {
  dir.create(site_data_dir, recursive = TRUE)
}

data_dir <- paste0(site_data_dir, "/ANN_data_train_test") # change wd here
if (!dir.exists(data_dir)) {
  dir.create(data_dir, recursive = TRUE)
}

setwd(data_dir)

for (month in 1:12) {
  train <- site_data  # Create a deep copy of site_data for train
  train$NEE_U50_orig <- ifelse(train$Month != month, train$NEE_U50_orig, NA)
  fwrite(train, file = paste0("train", month, ".csv"), row.names = FALSE)
  
  test <- site_data  # Create a deep copy of site_data for test
  test$NEE_U50_orig <- ifelse(test$Month == month, test$NEE_U50_orig, NA)
  fwrite(test, file = paste0("test", month, ".csv"), row.names = FALSE)
}
```

# experimental scenario 3: subset of year
```{r}
start.year = c(2011:2022)
end.year = 2022
set.seed(1000)
setwd(data_dir)
data_subset_year = site_data[site_data$Year %in% c(start.year, end.year), ]

# shuffle data and add fold number
shuffle_data <- data_subset_year[sample(nrow(data_subset_year )), ]

# Calculate the number of rows for each fold
n_rows <- nrow(shuffle_data)
fold_size <- ceiling(n_rows / 10)

# Initialize the fold_number column
shuffle_data$fold_number <- NA

# Label the data with fold_number from 1 to 10 based on percentiles
for (i in 1:10) {
  start_index <- (i - 1) * fold_size + 1
  end_index <- min(i * fold_size, n_rows)
  shuffle_data$fold_number[start_index:end_index] <- i
}

# save train and test data
output_dir = paste0("E:/13 Gapfill_scenario/subset_years_", start.year, "-", end.year)
if (!file.exists(output_dir)) {
  dir.create(output_dir, recursive = TRUE)
}
setwd(output_dir)

# Loop for fold number 1 to 10
for (fold_number in 1:10) {
  train = shuffle_data
  train$NEE_U50_orig <- ifelse(train$fold_number != fold_number, train$NEE_U50_orig, NA)
  
  test = shuffle_data
  test$NEE_U50_orig = ifelse(test$fold_number == fold_number, test$NEE_U50_orig, NA)
  # save data
  fwrite(train, file = paste0("train", fold_number, ".csv"), row.names = FALSE)
  fwrite(test, file = paste0("test", fold_number, ".csv"), row.names = FALSE)
}
```