Created lsip_lad data and fetch_lsip() function

NAMESPACE

@@ -8,6 +8,7 @@ export(create_project)
 export(fetch_countries)
 export(fetch_lads)
 export(fetch_las)
+export(fetch_lsip)
 export(fetch_mayoral)
 export(fetch_pcons)
 export(fetch_regions)

R/datasets_documentation.R

@@ -198,3 +198,29 @@
 #' https://get-information-schools.service.gov.uk/Guidance/LaNameCodes and
 #' https://tinyurl.com/EESScreenerLAs
 "wd_pcon_lad_la_rgn_ctry"
+
+#' Local Skills Improvement Plan (LSIP) areas to
+#' Local Authority District (LAD) Lookup
+#'
+#' A lookup table mapping Local Skills Improvement Plan (LSIP)
+#' areas to Local Authority Districts (LADs) in England. This dataset provides
+#' a mapping between LSIP areas and LADs as provided by
+#' the ONS Geography Portal.
+#'
+#' @details
+#' - Each LAD is assigned to a single LSIP area.
+#' - Mappings may change over time and can be tracked using the
+#'   `most_recent_year_included` and `first_available_year_included`
+#'    columns.
+#' @format ## `lsip_lad`
+#' A data frame with one row per LAD in England and the following columns:
+#' \describe{
+#'   \item{lad_code}{9-character code for the Local Authority District}
+#'   \item{lad_name}{Name of the Local Authority District}
+#'   \item{lsip_code}{9-character code for the LSIP area}
+#'   \item{lsip_name}{Name of the Local Skills Improvement Plan (LSIP) area}
+#'   \item{most_recent_year_included}{The most recent year in which this location appears in the lookup}
+#'   \item{first_available_year_included}{The first year in which this location appears in the lookup}
+#' }
+#' @source https://geoportal.statistics.gov.uk/search?q=lad%20lsip
+"lsip_lad"

R/datasets_utils.R

@@ -586,3 +586,81 @@ get_cauth_lad <- function(year) {
   # Tidy up the output file (defined earlier in this script)
   tidy_raw_lookup(output)
 }
+
+#' Fetch and combine LSIP-LAD lookup data for multiple years
+#'
+#' Downloads, binds, and tidies LSIP-LAD lookup data from ONS Geography portal
+#' API for multiple years.
+#' The function constructs the correct URLs for each year, fetches the data,
+#'  adds a year column, and combines all years into a single data frame.
+#' It then collapses the time series to add `first_available_year_included`
+#' and `most_recent_year_included` columns, and removes duplicates.
+#' Currently supports data for the years 2023 and 2025.
+#' To add support for additional years, update the `yr_specific_url` list
+#' with the appropriate year and URL segment.
+#'
+#' @return A data frame containing the combined LSIP-LAD lookup for all years,
+#' with columns for codes, names, year, and operational period.
+#' @keywords internal
+#' @noRd
+get_lsip_lad <- function() {
+  # Base URL components
+  url_prefix_1 <- "https://services1.arcgis.com/"
+  url_prefix_2 <- "ESMARspQHYMw9BZ9/arcgis/rest/services/"
+  url_suffix <- "/FeatureServer/0/query?outFields=*&where=1%3D1&f=json"
+
+  # Year-specific URL segments
+  yr_specific_url <- list(
+    "2023" = "LAD23_LSIP23_EN_LU",
+    "2025" = "LAD25_LSIP25_EN_LU"
+  )
+  #Create an empty list to store data frames
+  data_frames <- list()
+  #Loop through each year and fetch data
+  for (year in names(yr_specific_url)) {
+    #Construct the full URL
+    full_url <- paste0(
+      url_prefix_1,
+      url_prefix_2,
+      yr_specific_url[[year]],
+      url_suffix
+    )
+
+    #Make the GET request and parse the JSON response
+    response <- httr::GET(full_url)
+    # get the content and convert from json
+    data <- jsonlite::fromJSON(httr::content(response, "text"))
+
+    #Extract the attributes and convert to data frame
+    df <- as.data.frame(data$features$attributes) |>
+      #create a year column
+      dplyr::mutate(year = as.integer(year)) |>
+      #rename columns based on position so binding works
+      dplyr::select(
+        year,
+        lad_code = 1,
+        lad_name = 2,
+        lsip_code = 3,
+        lsip_name = 4
+      )
+
+    #put the data frame into the list
+    data_frames[[year]] <- df
+  }
+  #Combine all data frames into one
+  combined_df <- do.call(rbind, data_frames)
+  #get first_available and most_recent year columns
+  combined_df <- combined_df |>
+    collapse_timeseries() |>
+    # strip extra whitespace from all columns
+    dplyr::mutate(
+      dplyr::across(
+        dplyr::everything(),
+        ~ trimws(.x)
+      )
+    ) |>
+    #make sure we remove duplicates
+    dplyr::distinct()
+
+  combined_df
+}

R/fetch.R

@@ -28,6 +28,10 @@
 #'
 #' head(fetch_mayoral())
 #'
+#' head(fetch_lsip())
+#'
+#' head(fetch_lsip(2024))
+#'
 #' fetch_lads(2024, "Wales")
 #'
 #' fetch_las(2022, "Northern Ireland")
@@ -192,3 +196,43 @@ fetch_regions <- function() {
 fetch_countries <- function() {
   dfeR::countries
 }
+
+#' Fetch Local Skills Improvement Plan (LSIP) areas lookup
+#'
+#' Fetch a data frame of Local Skills Improvement Plan (LSIP) areas
+#' for a given year based on `dfeR::lsip_lad`.
+#'
+#' @param year Year to filter the lookup to, default is "All".
+#' @family fetch_locations
+#' @return data frame of LSIP for a given year.
+#' @export
+#' @inherit fetch examples
+fetch_lsip <- function(year = "All") {
+  #convert year input to numeric if possible
+  if (is.character(year) && year != "All") {
+    year_num <- suppressWarnings(as.numeric(year))
+    if (!is.na(year_num)) {
+      year <- year_num
+    }
+  }
+
+  #add a check for year input to see if it's in range
+  min_year <- min(dfeR::lsip_lad$first_available_year_included)
+  max_year <- max(dfeR::lsip_lad$most_recent_year_included)
+  if (
+    !(year == "All" || (year %% 1 == 0 && year >= min_year && year <= max_year))
+  ) {
+    stop(
+      paste0(
+        "year must either be 'All' or a valid year between ",
+        min_year,
+        " and ",
+        max_year
+      ),
+      call. = FALSE
+    )
+  }
+  lookup_data <- dfeR::lsip_lad
+  cols <- c("lsip_code", "lsip_name")
+  summarise_locations_by_year(lookup_data, cols, year)
+}

R/fetch_utils.R

@@ -27,35 +27,26 @@ check_fetch_location_inputs <- function(year_input, country_input) {
   }
 }
 
-#' Fetch locations for a given lookup
+#' Summarise and filter locations by operational years
 #'
-#' Helper function for the fetch_xxx() functions to save repeating code
+#' Shared logic for summarising location lookups and filtering
+#' by operational years.
+#' Used by fetch_locations and fetch_lsip.
 #'
-#' @param lookup_data lookup data to use to extract locations from
-#' @param cols columns to extract from the main lookup table
-#' @param year year of locations to extract, "All" will skip any filtering and
-#' return all possible locations
-#' @param countries countries for locations to be take from, "All" will skip
-#' any filtering and return all
-#'
-#' @return a data frame of location names and codes
+#' @param lookup_data The lookup data frame.
+#' @param cols Character vector of columns to keep and group by.
+#' @param year Year to filter to, or "All" for no filtering.
+#' @return A data frame summarised and filtered by operational years.
 #' @keywords internal
 #' @noRd
-fetch_locations <- function(lookup_data, cols, year, countries) {
-  # Return only the cols we specified
-  # We know their position from the dplyr selection of the lookup
-  # This is used wherever this function returns an output
+summarise_locations_by_year <- function(lookup_data, cols, year = "All") {
   cols_to_return <- seq_along(cols)
-
-  # Pull in main lookup data
   lookup <- dplyr::select(
     lookup_data,
     dplyr::all_of(
       c(cols, "first_available_year_included", "most_recent_year_included")
     )
   )
-
-  # Resummarise the years to each unique location
   resummarised_lookup <- lookup |>
     dplyr::summarise(
       "first_available_year_included" = min(
@@ -64,34 +55,49 @@ fetch_locations <- function(lookup_data, cols, year, countries) {
       "most_recent_year_included" = max(.data$most_recent_year_included),
       .by = dplyr::all_of(cols)
     )
-
-  # Return early without filtering if defaults are used
-  if (all(year == "All", countries == "All")) {
+  if (year == "All") {
     return(dplyr::distinct(resummarised_lookup[, cols_to_return]))
   }
-
-  # Filter based on year selection if specified
-  if (year != "All") {
-    # Flag the rows that are in the year asked for
-    resummarised_lookup <- resummarised_lookup |>
-      dplyr::mutate(
-        "in_specified_year" = ifelse(
-          as.numeric(.data$most_recent_year_included) >= year &
-            as.numeric(.data$first_available_year_included) <= year,
-          TRUE,
-          FALSE
-        )
+  resummarised_lookup <- resummarised_lookup |>
+    dplyr::mutate(
+      "in_specified_year" = ifelse(
+        as.numeric(.data$most_recent_year_included) >= year &
+          as.numeric(.data$first_available_year_included) <= year,
+        TRUE,
+        FALSE
       )
+    )
+  resummarised_lookup <- with(
+    resummarised_lookup,
+    subset(resummarised_lookup, in_specified_year == TRUE)
+  ) |>
+    dplyr::select(-c("in_specified_year"))
+  dplyr::distinct(resummarised_lookup[, cols_to_return])
+}
+
+#' Fetch locations for a given lookup
+#'
+#' Helper function for the fetch_xxx() functions to save repeating code
+#'
+#' @param lookup_data lookup data to use to extract locations from
+#' @param cols columns to extract from the main lookup table
+#' @param year year of locations to extract, "All" will skip any filtering and
+#' return all possible locations
+#' @param countries countries for locations to be take from, "All" will skip
+#' any filtering and return all
+#'
+#' @return a data frame of location names and codes
+#' @keywords internal
+#' @noRd
+fetch_locations <- function(lookup_data, cols, year, countries) {
+  resummarised_lookup <- summarise_locations_by_year(lookup_data, cols, year)
 
-    # Filter to only those locations
-    resummarised_lookup <- with(
-      resummarised_lookup,
-      subset(resummarised_lookup, in_specified_year == TRUE)
-    ) |>
-      dplyr::select(-c("in_specified_year")) # remove temp column
+  # Return early without filtering if defaults are used
+  if (all(year == "All", countries == "All")) {
+    return(resummarised_lookup)
   }
 
-  # Filter based on country selcetion if specified
+  # Filter based on country selection if specified
   if (paste0(countries, collapse = "") != "All") {
     # Get the code column
     # Take new_la_code if present (as sometimes there may also be old_la code)
@@ -119,5 +125,5 @@ fetch_locations <- function(lookup_data, cols, year, countries) {
       )
   }
 
-  dplyr::distinct(resummarised_lookup[, cols_to_return])
+  resummarised_lookup
 }

_pkgdown.yml

@@ -23,6 +23,7 @@ reference:
   - regions
   - geog_time_identifiers
   - wd_pcon_lad_la_rgn_ctry
+  - lsip_lad
 
 - title: Fetch geography lists
   desc: Pull geography lookups from the ONS Geography Portal

data-raw/lsip_lad.R

@@ -0,0 +1,38 @@
+# ------------------------------------------------------------------------------
+# Script to create the LSIP-LAD lookup dataset for the dfeR package
+#
+# This script fetches, processes, and saves the Local Authority District (LAD)
+# to Local Skills Improvement Plan (LSIP) lookup for multiple years.
+#
+# Data Source:
+#   - ONS Open Geography Portal API (https://geoportal.statistics.gov.uk/)
+#   - Each year's data is accessed via a unique ArcGIS REST API endpoint from
+#     the ONS Geography portal x, constructed from a common prefix, a
+#     year-specific path, and a query suffix. See the get_lsip_lad() function
+#     in R/datasets_utils.R for details on URL construction and year coverage.
+# What this script does:
+#   1. Calls get_lsip_lad() to fetch and combine LSIP-LAD data
+#      for all available years.
+#   2. The resulting data frame includes columns for LAD and LSIP codes
+#      and names, the year, and operational period columns.
+#   3. Saves the processed lookup as an internal package dataset.
+#
+# Usage:
+#   - Run this code when new LSIP-LAD data is available or to refresh the lookup
+#   - Ensure get_lsip_lad() is up to date with the correct endpoints for all
+#     years required.
+# How to update the data:
+#   1. Check the ONS Open Geography Portal for new or updated LSIP-LAD datasets
+#      and note the new URLs.
+#   2. Update the `yr_specific_url` list in get_lsip_lad() (R/datasets_utils.R)
+#      to include the section of the URL that corresponds to that year.
+#   3. Run this script to fetch, process, and save the latest data.
+#   4. Re-document and test the package as needed.
+#
+# ------------------------------------------------------------------------------
+
+# use get_lsip_lad to get the data from ONS
+lsip_lad <- get_lsip_lad()
+
+# Save the data to the package's data directory
+usethis::use_data(lsip_lad, overwrite = TRUE)

inst/WORDLIST

@@ -6,6 +6,8 @@ DfE
 EESScreenerLAs
 GOR
 JBLOGGS
+LADs
+LSIP
 LUP
 LaNameCodes
 Lifecycle
@@ -41,6 +43,7 @@ gov
 las
 lauraselby
 lockfile
+lsip
 lup
 num
 odbc