Kibi

R/differens_between_outputs.R

@@ -0,0 +1,124 @@
+#' Title
+#'
+#' @param variables
+#'
+#' @return
+#' @export
+#'
+#' @examples
+differens_between_outputs <- function(path = NULL,
+                                      type = "control",
+                                      output_version_old = NULL,
+                                      output_version_new = NULL,
+                                      rfmo = c("CCAMLR",
+                                               "CECAF",
+                                               "GFCM",
+                                               "IATTC",
+                                               "ICCAT",
+                                               "ICES",
+                                               "IOTC",
+                                               "NAFO",
+                                               "SEAFO",
+                                               "SPRFMO",
+                                               "WCPFC",
+                                               "WECAFC")) {
+  path <-
+    "Q:/mynd/kibi/projects_wks_wgs_rcgs/ISSG_RWP/2024/table_2_1_development/personal/output/"
+
+  output_version_new <- "20240517_094805"
+  output_version_old <- "20240515_132424"
+  rfmos <- c("ICES")
+  type <- "control"
+
+  # compare control output
+
+  if (type == "control") {
+    old <-
+      read.csv(
+        paste0(
+          path,
+          output_version_old,
+          "_table_2_1_template_control_27_eu_countries.csv"
+        ),
+        sep = ";"
+      )
+    new <-
+      read.csv(
+        paste0(
+          path,
+          output_version_new,
+          "_table_2_1_template_control_27_eu_countries.csv"
+        ),
+        sep = ";"
+      )
+
+    old <-
+      dplyr::rename(old,
+                    "tons_country_old" = "tons_country",
+                    "tons_eu_old" = "tons_eu")
+    new <-
+      dplyr::rename(new,
+                    "tons_country_new" = "tons_country",
+                    "tons_eu_new" = "tons_eu")
+
+    ## Check for duplicated records
+
+    old_distinct <- dplyr::distinct(old, region, rfmo, spp, geo, area, year, data_source)
+    old_distinct$id <- row.names(old_distinct)
+    new_distinct <- dplyr::distinct(new, region, rfmo, spp, geo, area, year, data_source)
+    new_distinct$id <- row.names(new_distinct)
+
+    new <- dplyr::left_join(new, new_distinct)
+
+    new_dup <- dplyr::summarise(dplyr::group_by(new, region, rfmo, spp, geo, area, year, data_source), no = length(id))
+    old_dup <- dplyr::summarise(dplyr::group_by(old_distinct, region, rfmo, spp, geo, area, year, data_source), no = length(id))
+
+
+
+    comb <- dplyr::full_join(old, new)
+
+    comb_diff <-
+      subset(comb,
+             tons_country_old != tons_country_new )#| tons_eu_old != tons_eu_new)
+
+    comb_diff <-
+      subset(comb_diff,
+             rfmo %in% rfmos)
+
+    write.table(
+      comb_diff,
+      paste0(
+        path,
+        format(x = Sys.time(),
+               format = "%Y%m%d_%H%M%S_"),
+        "difference_between_control_files.csv"
+      ),
+      row.names = F,
+      sep = ";"
+    )
+
+
+
+  }
+
+
+  # compare table 2.1 input
+
+  old <- read.csv(paste0(path, output_version_old, "_table_2_1_template_27_eu_countries.csv"), sep = ";")
+  new <- read.csv(paste0(path, output_version_new, "_table_2_1_template_27_eu_countries.csv"), sep = ";")
+
+  old <- dplyr::select(dplyr::rename(old, "landings_old" = "landings", "tac_old" = "tac", "share_landing_old" = "share_landing"),
+                -source_national, -share_landing_old, -source_eu, -thresh, -reg_coord, -covered_length, -selected_bio, -comments)
+  new <- dplyr::select(dplyr::rename(new, "landings_new" = "landings", "tac_new" = "tac", "share_landing_old" = "share_landing"),
+                       -source_national, -share_landing_old, -source_eu, -thresh, -reg_coord, -covered_length, -selected_bio, -comments)
+
+  comb <- dplyr::full_join(old, new)
+
+  comb_diff <- subset(comb, landings_old != landings_new)
+
+  return(comb_diff)
+
+
+
+
+}

R/global_load_cl_landing_rdb_data.R

@@ -92,7 +92,7 @@ global_load_cl_landing_rdb_data <- function(input_path_file_rcg_stats) {
   # Include code for EU28 + warning if UK countries are missing
   cl_landing_rdb_data_2 <- rbind(cl_landing_rdb_data_ctry, cl_landing_rdb_data_eu27)
 
-  cl_landing_rdb_data_t <- tidyr::spread(cl_landing_rdb_data_2, key = Year, value = TLW)
+  cl_landing_rdb_data_t <- tidyr::spread(cl_landing_rdb_data_2, key = Year, value = TLW, fill = 0) # To make sure the mean get right!
   cl_landing_rdb_data_t <- dplyr::rename(cl_landing_rdb_data_t, Scientific_Name  = Species)
   return(cl_landing_rdb_data_t)
 }

R/global_load_cl_landing_rdbes_data.R

@@ -1,59 +1,109 @@
 #' @name global_load_cl_landing_rdbes_data
-#' @title Load CL landing RDBES data
-#' @description Process for for load the data from CL landing RDBES data file.
-#' @param input_path_cl_landing_rdbes_data {\link[base]{character}} expected. Input CL landing RDBES data file path (.csv format expected).
+#' @title Load CL landing RDB data
+#' @description Process for for load the data from CL landing RDB data file.
+#' @param input_path_file_rcg_stats {\link[base]{character}} expected. Input rcg stats data file path (.csv format expected).
 #' @return A tibble.
 #' @importFrom readr read_csv
+#' @importFrom dplyr group_by summarise rename
+#' @importFrom tidyr spread
 #' @export
-global_load_cl_landing_rdbes_data <- function(input_path_cl_landing_rdbes_data) {
+global_load_cl_landing_rdbes_data <- function(input_path_file_rdbes_stats) {
   cat(format(x = Sys.time(),
              format = "%Y-%m-%d %H:%M:%S"),
-      " - Start process for load data from CL landing RDBES data.\n",
+      " - Start process for load data from CL landings RDB data.\n",
       sep = "")
   # global arguments verifications ----
-  # to do in the future
+  Year <- NULL
+  Species <- NULL
+  Area <- NULL
+  geo <- NULL
+  OfficialLandingCatchWeight <- NULL
+  FlagCountry <- NULL
+  TLW <- NULL
   # process ----
-  cl_landing_rdbes_data <- withCallingHandlers(expr = readr::read_csv(file = input_path_cl_landing_rdbes_data,
-                                                                      col_names = c("cl_commercial_landing_id",
-                                                                                    "landing_country",
-                                                                                    "flag_country",
-                                                                                    "year",
-                                                                                    "quarter",
-                                                                                    "month",
-                                                                                    "area",
-                                                                                    "fishing_ground",
-                                                                                    "region",
-                                                                                    "statistical_rectangle",
-                                                                                    "subpolygon",
-                                                                                    "species_aphia_id",
-                                                                                    "species",
-                                                                                    "species_desc",
-                                                                                    "stock",
-                                                                                    "landing_category",
-                                                                                    "size_category_scale",
-                                                                                    "size_category",
-                                                                                    "fishing_activity_category_national",
-                                                                                    "fishing_activity_category_european_level5",
-                                                                                    "fishing_activity_category_european_level6",
-                                                                                    "harbour",
-                                                                                    "harbour_desc",
-                                                                                    "vessel_length_category",
-                                                                                    "unallocated_catch_weight",
-                                                                                    "area_misreported_catch_weight",
-                                                                                    "official_landing_catch_weight",
-                                                                                    "landing_multiplier",
-                                                                                    "official_landing_value"),
-                                                                      col_types = "dccddccccccdccccccccccccnnnnn",
-                                                                      skip = 1,
-                                                                      na = c("",
-                                                                             "NA",
-                                                                             "NULL")),
-                                               warning = function(input_path_cl_landing_rdbes_data) {
-                                                 stop("Troubles with the input file, check the format associated.\n")
-                                               })
+  # Define EU countries
+  EU27_2020 <- c("AUT",
+                 "BEL",
+                 "BGR",
+                 "HRV",
+                 "CYP",
+                 "CZE",
+                 "DNK",
+                 "EST",
+                 "FIN",
+                 "FRA",
+                 "DEU",
+                 "GRC",
+                 "HUN",
+                 "IRL",
+                 "ITA",
+                 "LVA",
+                 "LTU",
+                 "LUX",
+                 "MLT",
+                 "NLD",
+                 "POL",
+                 "PRT",
+                 "ROU",
+                 "SVK",
+                 "SVN",
+                 "ESP",
+                 "SWE")
+
+
+  cl_landing_rdbes_data <-
+    data.table::fread(paste0(input_path_file_rdbes_stats, "CommercialLanding.csv"),
+          sep = ",",
+          quote = "") # If read.csv, then set quote="" otherwise it skips a lot of data without warning
+
   cat(format(x = Sys.time(),
              format = "%Y-%m-%d %H:%M:%S"),
-      " - Successful process for load data from CL landing RDBES data.\n",
+      " - Successful process for load data from CL landings RDBES data.\n",
       sep = "")
-  return(cl_landing_rdbes_data)
+
+  # Only include BMS and LAN
+
+  unique(cl_landing_rdbes_data$CLcatchCategory)
+
+  cl_landing_rdbes_data <- subset(cl_landing_rdbes_data, CLcatchCategory %in% c("Lan", "BMS"))
+
+  # transform RDBES data into the same format as EUROSTAT
+  #3 Add country codes from geo data file
+  geo_data <- utils::read.table(file = system.file("geo.def",
+                                                   package = "rwptool"),
+                                header = TRUE,
+                                sep = ";")
+
+  cl_landing_rdbes_data_1 <- dplyr::left_join(cl_landing_rdbes_data, geo_data,
+                                            by = c("CLvesselFlagCountry" = "geo"),
+                                            keep = T)
+
+  # Add scientific name
+  rdbes_spp <- icesVocab::getCodeList("SpecWoRMS")
+  cl_landing_rdbes_data_2 <- left_join(cl_landing_rdbes_data_1, rdbes_spp, by = c("CLspeciesCode" = "Key"))
+  cl_landing_rdbes_data_2$CLspeciesName <- cl_landing_rdbes_data_2$Description
+
+  # Summarise and create a EU27_2020 line
+  cl_landing_rdbes_data_ctry <-
+    dplyr::summarise(
+      dplyr::group_by(cl_landing_rdbes_data_2, CLyear, CLspeciesName, CLarea, geo),
+      TLW = sum(CLofficialWeight / 1000, na.rm = T),
+      .groups = "drop"
+    )
+  cl_landing_rdbes_data_eu27 <-
+    dplyr::summarise(
+      dplyr::group_by(
+        dplyr::filter(cl_landing_rdbes_data_2, level_description %in% !!EU27_2020) ,
+        CLyear, CLspeciesName, CLarea
+      ),
+      geo = "EU27_2020",
+      TLW = sum(CLofficialWeight / 1000, na.rm = T),
+      .groups = "drop"
+    )
+  # Include code for EU28 + warning if UK countries are missing
+  cl_landing_rdbes_data_3 <- rbind(cl_landing_rdbes_data_ctry, cl_landing_rdbes_data_eu27)
+
+  cl_landing_rdbes_data_t <- tidyr::spread(cl_landing_rdbes_data_3, key = CLyear, value = TLW, fill = 0) # To make sure the mean get right!
+  cl_landing_rdbes_data_t <- dplyr::rename(cl_landing_rdbes_data_t, Scientific_Name  = CLspeciesName)
+  return(cl_landing_rdbes_data_t)
 }

R/global_load_eurostat_data.R

@@ -24,7 +24,7 @@ global_load_eurostat_data <- function(path) {
   # process ----
   eurostat_files <- list.files(path = path)
   eurostat_files <- eurostat_files[stringr::str_detect(string = eurostat_files,
-                                                       pattern = "^fish_ca.*.tsv$")]
+                                                       pattern = "fish_ca.*.tsv$")]
   if (length(x = eurostat_files) != 0) {
     eurostat_files_final <- lapply(X = seq_len(length.out = length(x = eurostat_files)),
                                    FUN = function(eurostat_file_id) {