Added FbRefScraper class to simplify scraping.

fbref_scraper.py

@@ -0,0 +1,269 @@
+"""
+Script by: https://github.com/matze298
+Based on code from: https://github.com/parth1902/Scrape-FBref-data
+"""
+
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import numpy as np
+import re
+import sys, getopt
+import csv
+import copy
+
+## Links to competitions
+league_links = {
+  "Bundesliga": ["https://fbref.com/en/comps/20/", "/Bundesliga-Stats"],
+  "Premier League": ["https://fbref.com/en/comps/9/", "/Premier-League-Stats"],
+  "Ligue 1": ["https://fbref.com/en/comps/13/", "/Ligue-1-Stats"],
+  "Serie A": ["https://fbref.com/en/comps/11/", "/Serie-A-Stats"],
+  "La Liga": ["https://fbref.com/en/comps/12/", "/La-Liga-Stats"]
+}
+
+year_links = {
+  "Bundesliga": {2018: 1634, 2019: 2109, 2020: 3248},
+  "Premier League": {2018: 1631, 2019: 1889, 2020: 3232},
+  "Ligue 1": {2018: 1632, 2019: 2109, 2020: 3243},
+  "Serie A": {2018: 1640, 2019: 1896, 2020: 3260},
+  "La Liga": {2018: 1652, 2019: 1886, 2020: 3239}
+}
+
+## Table settings
+#standard(stats)
+stats = ["player","nationality","position","squad","age","birth_year","games","games_starts","minutes","goals","assists","pens_made","pens_att","cards_yellow","cards_red","goals_per90","assists_per90","goals_assists_per90","goals_pens_per90","goals_assists_pens_per90","xg","npxg","xa","xg_per90","xa_per90","xg_xa_per90","npxg_per90","npxg_xa_per90"]
+stats3 = ["players_used","possession","games","games_starts","minutes","goals","assists","pens_made","pens_att","cards_yellow","cards_red","goals_per90","assists_per90","goals_assists_per90","goals_pens_per90","goals_assists_pens_per90","xg","npxg","xa","xg_per90","xa_per90","xg_xa_per90","npxg_per90","npxg_xa_per90"] 
+#goalkeeping(keepers)
+keepers = ["player","nationality","position","squad","age","birth_year","games_gk","games_starts_gk","minutes_gk","goals_against_gk","goals_against_per90_gk","shots_on_target_against","saves","save_pct","wins_gk","draws_gk","losses_gk","clean_sheets","clean_sheets_pct","pens_att_gk","pens_allowed","pens_saved","pens_missed_gk"]
+keepers3 = ["players_used","games_gk","games_starts_gk","minutes_gk","goals_against_gk","goals_against_per90_gk","shots_on_target_against","saves","save_pct","wins_gk","draws_gk","losses_gk","clean_sheets","clean_sheets_pct","pens_att_gk","pens_allowed","pens_saved","pens_missed_gk"]
+#advance goalkeeping(keepersadv)
+keepersadv = ["player","nationality","position","squad","age","birth_year","minutes_90s","goals_against_gk","pens_allowed","free_kick_goals_against_gk","corner_kick_goals_against_gk","own_goals_against_gk","psxg_gk","psnpxg_per_shot_on_target_against","psxg_net_gk","psxg_net_per90_gk","passes_completed_launched_gk","passes_launched_gk","passes_pct_launched_gk","passes_gk","passes_throws_gk","pct_passes_launched_gk","passes_length_avg_gk","goal_kicks","pct_goal_kicks_launched","goal_kick_length_avg","crosses_gk","crosses_stopped_gk","crosses_stopped_pct_gk","def_actions_outside_pen_area_gk","def_actions_outside_pen_area_per90_gk","avg_distance_def_actions_gk"]
+keepersadv2 = ["minutes_90s","goals_against_gk","pens_allowed","free_kick_goals_against_gk","corner_kick_goals_against_gk","own_goals_against_gk","psxg_gk","psnpxg_per_shot_on_target_against","psxg_net_gk","psxg_net_per90_gk","passes_completed_launched_gk","passes_launched_gk","passes_pct_launched_gk","passes_gk","passes_throws_gk","pct_passes_launched_gk","passes_length_avg_gk","goal_kicks","pct_goal_kicks_launched","goal_kick_length_avg","crosses_gk","crosses_stopped_gk","crosses_stopped_pct_gk","def_actions_outside_pen_area_gk","def_actions_outside_pen_area_per90_gk","avg_distance_def_actions_gk"]
+#shooting(shooting)
+shooting = ["player","nationality","position","squad","age","birth_year","minutes_90s","goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"]
+shooting2 = ["minutes_90s","goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"]
+shooting3 = ["goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"]
+#passing(passing)
+passing = ["player","nationality","position","squad","age","birth_year","minutes_90s","passes_completed","passes","passes_pct","passes_total_distance","passes_progressive_distance","passes_completed_short","passes_short","passes_pct_short","passes_completed_medium","passes_medium","passes_pct_medium","passes_completed_long","passes_long","passes_pct_long","assists","xa","xa_net","assisted_shots","passes_into_final_third","passes_into_penalty_area","crosses_into_penalty_area","progressive_passes"]
+passing2 = ["passes_completed","passes","passes_pct","passes_total_distance","passes_progressive_distance","passes_completed_short","passes_short","passes_pct_short","passes_completed_medium","passes_medium","passes_pct_medium","passes_completed_long","passes_long","passes_pct_long","assists","xa","xa_net","assisted_shots","passes_into_final_third","passes_into_penalty_area","crosses_into_penalty_area","progressive_passes"]
+#passtypes(passing_types)
+passing_types = ["player","nationality","position","squad","age","birth_year","minutes_90s","passes","passes_live","passes_dead","passes_free_kicks","through_balls","passes_pressure","passes_switches","crosses","corner_kicks","corner_kicks_in","corner_kicks_out","corner_kicks_straight","passes_ground","passes_low","passes_high","passes_left_foot","passes_right_foot","passes_head","throw_ins","passes_other_body","passes_completed","passes_offsides","passes_oob","passes_intercepted","passes_blocked"]
+passing_types2 = ["passes","passes_live","passes_dead","passes_free_kicks","through_balls","passes_pressure","passes_switches","crosses","corner_kicks","corner_kicks_in","corner_kicks_out","corner_kicks_straight","passes_ground","passes_low","passes_high","passes_left_foot","passes_right_foot","passes_head","throw_ins","passes_other_body","passes_completed","passes_offsides","passes_oob","passes_intercepted","passes_blocked"]
+#goal and shot creation(gca)
+gca = ["player","nationality","position","squad","age","birth_year","minutes_90s","sca","sca_per90","sca_passes_live","sca_passes_dead","sca_dribbles","sca_shots","sca_fouled","gca","gca_per90","gca_passes_live","gca_passes_dead","gca_dribbles","gca_shots","gca_fouled","gca_og_for"]
+gca2 = ["sca","sca_per90","sca_passes_live","sca_passes_dead","sca_dribbles","sca_shots","sca_fouled","gca","gca_per90","gca_passes_live","gca_passes_dead","gca_dribbles","gca_shots","gca_fouled","gca_og_for"]
+#defensive actions(defense)
+defense = ["player","nationality","position","squad","age","birth_year","minutes_90s","tackles","tackles_won","tackles_def_3rd","tackles_mid_3rd","tackles_att_3rd","dribble_tackles","dribbles_vs","dribble_tackles_pct","dribbled_past","pressures","pressure_regains","pressure_regain_pct","pressures_def_3rd","pressures_mid_3rd","pressures_att_3rd","blocks","blocked_shots","blocked_shots_saves","blocked_passes","interceptions","clearances","errors"]
+defense2 = ["tackles","tackles_won","tackles_def_3rd","tackles_mid_3rd","tackles_att_3rd","dribble_tackles","dribbles_vs","dribble_tackles_pct","dribbled_past","pressures","pressure_regains","pressure_regain_pct","pressures_def_3rd","pressures_mid_3rd","pressures_att_3rd","blocks","blocked_shots","blocked_shots_saves","blocked_passes","interceptions","clearances","errors"]
+#possession(possession)
+possession = ["player","nationality","position","squad","age","birth_year","minutes_90s","touches","touches_def_pen_area","touches_def_3rd","touches_mid_3rd","touches_att_3rd","touches_att_pen_area","touches_live_ball","dribbles_completed","dribbles","dribbles_completed_pct","players_dribbled_past","nutmegs","carries","carry_distance","carry_progressive_distance","progressive_carries","carries_into_final_third","carries_into_penalty_area","pass_targets","passes_received","passes_received_pct","miscontrols","dispossessed"]
+possession2 = ["touches","touches_def_pen_area","touches_def_3rd","touches_mid_3rd","touches_att_3rd","touches_att_pen_area","touches_live_ball","dribbles_completed","dribbles","dribbles_completed_pct","players_dribbled_past","nutmegs","carries","carry_distance","carry_progressive_distance","progressive_carries","carries_into_final_third","carries_into_penalty_area","pass_targets","passes_received","passes_received_pct","miscontrols","dispossessed"]
+#playingtime(playingtime)
+playingtime = ["player","nationality","position","squad","age","birth_year","minutes_90s","games","minutes","minutes_per_game","minutes_pct","games_starts","minutes_per_start","games_subs","minutes_per_sub","unused_subs","points_per_match","on_goals_for","on_goals_against","plus_minus","plus_minus_per90","plus_minus_wowy","on_xg_for","on_xg_against","xg_plus_minus","xg_plus_minus_per90","xg_plus_minus_wowy"]
+playingtime2 = ["games","minutes","minutes_per_game","minutes_pct","games_starts","minutes_per_start","games_subs","minutes_per_sub","unused_subs","points_per_match","on_goals_for","on_goals_against","plus_minus","plus_minus_per90","plus_minus_wowy","on_xg_for","on_xg_against","xg_plus_minus","xg_plus_minus_per90","xg_plus_minus_wowy"]
+#miscallaneous(misc)
+misc = ["player","nationality","position","squad","age","birth_year","minutes_90s","cards_yellow","cards_red","cards_yellow_red","fouls","fouled","offsides","crosses","interceptions","tackles_won","pens_won","pens_conceded","own_goals","ball_recoveries","aerials_won","aerials_lost","aerials_won_pct"]
+misc2 = ["cards_yellow","cards_red","cards_yellow_red","fouls","fouled","offsides","crosses","interceptions","tackles_won","pens_won","pens_conceded","own_goals","ball_recoveries","aerials_won","aerials_lost","aerials_won_pct"]
+
+
+## Scrape using Beautiful Soup & convert to pandas
+def get_tables(url):
+    res = requests.get(url)
+    ## The next two lines get around the issue with comments breaking the parsing.
+    comm = re.compile("<!--|-->")
+    soup = BeautifulSoup(comm.sub("",res.text),'lxml')
+    all_tables = soup.findAll("tbody")
+    team_table = all_tables[0]
+    player_table = all_tables[1]
+    return player_table, team_table
+
+def get_frame(features, player_table):
+    pre_df_player = dict()
+    features_wanted_player = features
+    rows_player = player_table.find_all('tr')
+    for row in rows_player:
+        if(row.find('th',{"scope":"row"}) != None):
+
+            for f in features_wanted_player:
+                cell = row.find("td",{"data-stat": f})
+                a = cell.text.strip().encode()
+                text=a.decode("utf-8")
+                if(text == ''):
+                    text = '0'
+                if((f!='player')&(f!='nationality')&(f!='position')&(f!='squad')&(f!='age')&(f!='birth_year')):
+                    text = float(text.replace(',',''))
+                if f in pre_df_player:
+                    pre_df_player[f].append(text)
+                else:
+                    pre_df_player[f] = [text]
+    df_player = pd.DataFrame.from_dict(pre_df_player)
+    return df_player
+
+def get_frame_team(features, team_table):
+    pre_df_squad = dict()
+    #Note: features does not contain squad name, it requires special treatment
+    features_wanted_squad = features
+    rows_squad = team_table.find_all('tr')
+    for row in rows_squad:
+        if(row.find('th',{"scope":"row"}) != None):
+            name = row.find('th',{"data-stat":"squad"}).text.strip().encode().decode("utf-8")
+            if 'squad' in pre_df_squad:
+                pre_df_squad['squad'].append(name)
+            else:
+                pre_df_squad['squad'] = [name]
+            for f in features_wanted_squad:
+                cell = row.find("td",{"data-stat": f})
+                a = cell.text.strip().encode()
+                text=a.decode("utf-8")
+                if(text == ''):
+                    text = '0'
+                if((f!='player')&(f!='nationality')&(f!='position')&(f!='squad')&(f!='age')&(f!='birth_year')):
+                    text = float(text.replace(',',''))
+                if f in pre_df_squad:
+                    pre_df_squad[f].append(text)
+                else:
+                    pre_df_squad[f] = [text]
+    df_squad = pd.DataFrame.from_dict(pre_df_squad)
+    return df_squad
+
+def frame_for_category(category,top,end,features):
+    url = (top + category + end)
+    player_table, team_table = get_tables(url)
+    df_player = get_frame(features, player_table)
+    return df_player
+
+def frame_for_category_team(category,top,end,features):
+    url = (top + category + end)
+    player_table, team_table = get_tables(url)
+    df_team = get_frame_team(features, team_table)
+    return df_team
+
+
+#Function to get the player data for outfield player, includes all categories - standard stats, shooting
+#passing, passing types, goal and shot creation, defensive actions, possession, and miscallaneous
+def get_outfield_data(top, end):
+    df1 = frame_for_category('stats',top,end,stats)
+    df2 = frame_for_category('shooting',top,end,shooting2)
+    df3 = frame_for_category('passing',top,end,passing2)
+    df4 = frame_for_category('passing_types',top,end,passing_types2)
+    df5 = frame_for_category('gca',top,end,gca2)
+    df6 = frame_for_category('defense',top,end,defense2)
+    df7 = frame_for_category('possession',top,end,possession2)
+    df8 = frame_for_category('misc',top,end,misc2)
+    df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=1)
+    df = df.loc[:,~df.columns.duplicated()]
+    return df
+
+
+#Function to get keeping and advance goalkeeping data
+def get_keeper_data(top,end):
+    df1 = frame_for_category('keepers',top,end,keepers)
+    df2 = frame_for_category('keepersadv',top,end,keepersadv2)
+    df = pd.concat([df1, df2], axis=1)
+    df = df.loc[:,~df.columns.duplicated()]
+    return df
+
+
+#Function to get team-wise data accross all categories as mentioned above
+def get_team_data(top,end):
+    df1 = frame_for_category_team('stats',top,end,stats3)
+    df2 = frame_for_category_team('keepers',top,end,keepers3)
+    df3 = frame_for_category_team('keepersadv',top,end,keepersadv2)
+    df4 = frame_for_category_team('shooting',top,end,shooting3)
+    df5 = frame_for_category_team('passing',top,end,passing2)
+    df6 = frame_for_category_team('passing_types',top,end,passing_types2)
+    df7 = frame_for_category_team('gca',top,end,gca2)
+    df8 = frame_for_category_team('defense',top,end,defense2)
+    df9 = frame_for_category_team('possession',top,end,possession2)
+    df10 = frame_for_category_team('misc',top,end,misc2)
+    df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8, df9, df10], axis=1)
+    df = df.loc[:,~df.columns.duplicated()]
+    return df
+
+
+class FbRefScraper:
+  def __init__(self, top5=False, leagues=["Bundesliga"], seasons=[2021]):
+    if top5:
+      self.leagues = ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]
+    else:
+      self.leagues = leagues    
+    self.seasons = seasons
+    return
+
+  def scrape_players(self, csv_path=None):
+    df_outfield = pd.DataFrame()
+    for league in self.leagues:
+      if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]:
+        print("League must be one of ",  ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"])
+        continue
+      for season in self.seasons:
+        print(f"Loading player data for {league} season {season-1}/{season}")
+        link = copy.deepcopy(league_links[league])
+        if season != 2021:
+          link[0] = link[0] + f"{year_links[league][season]}/"
+          link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "")
+        df_outfield_league = get_outfield_data(link[0], link[1])
+        df_outfield_league["season"] = season
+        df_outfield = df_outfield.append(df_outfield_league, ignore_index=True)
+    if csv_path is not None:
+      df_outfield.to_csv(csv_path, index=False)
+    return df_outfield
+
+  def scrape_teams(self, csv_path=None, include_vs=False):
+    df_teams = pd.DataFrame()
+    for league in self.leagues:
+      if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]:
+        print("League must be one of ",  ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"])
+        continue
+      for season in self.seasons:
+        print(f"Loading team data for {league} season {season-1}/{season}")
+        link = copy.deepcopy(league_links[league])
+        if season != 2021:
+          link[0] = link[0] + f"{year_links[league][season]}/"
+          link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "")
+        df_teams_league = get_team_data(link[0], link[1])
+        df_teams_league["season"] = season
+        if not include_vs:
+          df_teams_league = df_teams_league[["vs" not in t for t in df_teams_league["squad"]]]
+        df_teams = df_teams.append(df_teams_league, ignore_index=True)
+    if csv_path is not None:
+      df_teams.to_csv(csv_path, index=False)
+    return df_teams
+
+  def scrape_goalkeepers(self, csv_path=None):
+    df_gks = pd.DataFrame()
+    for league in self.leagues:
+      if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]:
+        print("League must be one of ",  ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"])
+        continue
+      for season in self.seasons:
+        print(f"Loading goalkeeper data for {league} season {season-1}/{season}")
+        link = copy.deepcopy(league_links[league])
+        if season != 2021:
+          link[0] = link[0] + f"{year_links[league][season]}/"
+          link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "")
+        df_gks_league = get_keeper_data(link[0], link[1])
+        df_gks_league["season"] = season
+        df_gks = df_gks.append(df_gks_league, ignore_index=True)
+    if csv_path is not None:
+      df_gks.to_csv(csv_path, index=False)
+    return df_gks
+
+if __name__ == "__main__":
+  # Example Usage
+  # Leagues must be in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]
+  # Season is end-year of the season (20/21 --> 2021)
+  scraper = FbRefScraper(leagues=["Premier League"], seasons=[2019])
+  # Outfield players
+  # Add csv_path to store df to csv
+  df_outfield = scraper.scrape_players(csv_path="outfield.csv")
+  print(df_outfield.head())
+
+  # Team data
+  df_teams = scraper.scrape_teams(include_vs=False)
+  print(df_teams.head())
+
+  # Goalkeepers
+  df_gks = scraper.scrape_goalkeepers()
+  print(df_gks.head())
+