diff --git a/fbref_scraper.py b/fbref_scraper.py new file mode 100644 index 0000000..9624c37 --- /dev/null +++ b/fbref_scraper.py @@ -0,0 +1,269 @@ +""" +Script by: https://github.com/matze298 +Based on code from: https://github.com/parth1902/Scrape-FBref-data +""" + +import requests +from bs4 import BeautifulSoup +import pandas as pd +import numpy as np +import re +import sys, getopt +import csv +import copy + +## Links to competitions +league_links = { + "Bundesliga": ["https://fbref.com/en/comps/20/", "/Bundesliga-Stats"], + "Premier League": ["https://fbref.com/en/comps/9/", "/Premier-League-Stats"], + "Ligue 1": ["https://fbref.com/en/comps/13/", "/Ligue-1-Stats"], + "Serie A": ["https://fbref.com/en/comps/11/", "/Serie-A-Stats"], + "La Liga": ["https://fbref.com/en/comps/12/", "/La-Liga-Stats"] +} + +year_links = { + "Bundesliga": {2018: 1634, 2019: 2109, 2020: 3248}, + "Premier League": {2018: 1631, 2019: 1889, 2020: 3232}, + "Ligue 1": {2018: 1632, 2019: 2109, 2020: 3243}, + "Serie A": {2018: 1640, 2019: 1896, 2020: 3260}, + "La Liga": {2018: 1652, 2019: 1886, 2020: 3239} +} + +## Table settings +#standard(stats) +stats = ["player","nationality","position","squad","age","birth_year","games","games_starts","minutes","goals","assists","pens_made","pens_att","cards_yellow","cards_red","goals_per90","assists_per90","goals_assists_per90","goals_pens_per90","goals_assists_pens_per90","xg","npxg","xa","xg_per90","xa_per90","xg_xa_per90","npxg_per90","npxg_xa_per90"] +stats3 = ["players_used","possession","games","games_starts","minutes","goals","assists","pens_made","pens_att","cards_yellow","cards_red","goals_per90","assists_per90","goals_assists_per90","goals_pens_per90","goals_assists_pens_per90","xg","npxg","xa","xg_per90","xa_per90","xg_xa_per90","npxg_per90","npxg_xa_per90"] +#goalkeeping(keepers) +keepers = ["player","nationality","position","squad","age","birth_year","games_gk","games_starts_gk","minutes_gk","goals_against_gk","goals_against_per90_gk","shots_on_target_against","saves","save_pct","wins_gk","draws_gk","losses_gk","clean_sheets","clean_sheets_pct","pens_att_gk","pens_allowed","pens_saved","pens_missed_gk"] +keepers3 = ["players_used","games_gk","games_starts_gk","minutes_gk","goals_against_gk","goals_against_per90_gk","shots_on_target_against","saves","save_pct","wins_gk","draws_gk","losses_gk","clean_sheets","clean_sheets_pct","pens_att_gk","pens_allowed","pens_saved","pens_missed_gk"] +#advance goalkeeping(keepersadv) +keepersadv = ["player","nationality","position","squad","age","birth_year","minutes_90s","goals_against_gk","pens_allowed","free_kick_goals_against_gk","corner_kick_goals_against_gk","own_goals_against_gk","psxg_gk","psnpxg_per_shot_on_target_against","psxg_net_gk","psxg_net_per90_gk","passes_completed_launched_gk","passes_launched_gk","passes_pct_launched_gk","passes_gk","passes_throws_gk","pct_passes_launched_gk","passes_length_avg_gk","goal_kicks","pct_goal_kicks_launched","goal_kick_length_avg","crosses_gk","crosses_stopped_gk","crosses_stopped_pct_gk","def_actions_outside_pen_area_gk","def_actions_outside_pen_area_per90_gk","avg_distance_def_actions_gk"] +keepersadv2 = ["minutes_90s","goals_against_gk","pens_allowed","free_kick_goals_against_gk","corner_kick_goals_against_gk","own_goals_against_gk","psxg_gk","psnpxg_per_shot_on_target_against","psxg_net_gk","psxg_net_per90_gk","passes_completed_launched_gk","passes_launched_gk","passes_pct_launched_gk","passes_gk","passes_throws_gk","pct_passes_launched_gk","passes_length_avg_gk","goal_kicks","pct_goal_kicks_launched","goal_kick_length_avg","crosses_gk","crosses_stopped_gk","crosses_stopped_pct_gk","def_actions_outside_pen_area_gk","def_actions_outside_pen_area_per90_gk","avg_distance_def_actions_gk"] +#shooting(shooting) +shooting = ["player","nationality","position","squad","age","birth_year","minutes_90s","goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"] +shooting2 = ["minutes_90s","goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"] +shooting3 = ["goals","pens_made","pens_att","shots_total","shots_on_target","shots_free_kicks","shots_on_target_pct","shots_total_per90","shots_on_target_per90","goals_per_shot","goals_per_shot_on_target","xg","npxg","npxg_per_shot","xg_net","npxg_net"] +#passing(passing) +passing = ["player","nationality","position","squad","age","birth_year","minutes_90s","passes_completed","passes","passes_pct","passes_total_distance","passes_progressive_distance","passes_completed_short","passes_short","passes_pct_short","passes_completed_medium","passes_medium","passes_pct_medium","passes_completed_long","passes_long","passes_pct_long","assists","xa","xa_net","assisted_shots","passes_into_final_third","passes_into_penalty_area","crosses_into_penalty_area","progressive_passes"] +passing2 = ["passes_completed","passes","passes_pct","passes_total_distance","passes_progressive_distance","passes_completed_short","passes_short","passes_pct_short","passes_completed_medium","passes_medium","passes_pct_medium","passes_completed_long","passes_long","passes_pct_long","assists","xa","xa_net","assisted_shots","passes_into_final_third","passes_into_penalty_area","crosses_into_penalty_area","progressive_passes"] +#passtypes(passing_types) +passing_types = ["player","nationality","position","squad","age","birth_year","minutes_90s","passes","passes_live","passes_dead","passes_free_kicks","through_balls","passes_pressure","passes_switches","crosses","corner_kicks","corner_kicks_in","corner_kicks_out","corner_kicks_straight","passes_ground","passes_low","passes_high","passes_left_foot","passes_right_foot","passes_head","throw_ins","passes_other_body","passes_completed","passes_offsides","passes_oob","passes_intercepted","passes_blocked"] +passing_types2 = ["passes","passes_live","passes_dead","passes_free_kicks","through_balls","passes_pressure","passes_switches","crosses","corner_kicks","corner_kicks_in","corner_kicks_out","corner_kicks_straight","passes_ground","passes_low","passes_high","passes_left_foot","passes_right_foot","passes_head","throw_ins","passes_other_body","passes_completed","passes_offsides","passes_oob","passes_intercepted","passes_blocked"] +#goal and shot creation(gca) +gca = ["player","nationality","position","squad","age","birth_year","minutes_90s","sca","sca_per90","sca_passes_live","sca_passes_dead","sca_dribbles","sca_shots","sca_fouled","gca","gca_per90","gca_passes_live","gca_passes_dead","gca_dribbles","gca_shots","gca_fouled","gca_og_for"] +gca2 = ["sca","sca_per90","sca_passes_live","sca_passes_dead","sca_dribbles","sca_shots","sca_fouled","gca","gca_per90","gca_passes_live","gca_passes_dead","gca_dribbles","gca_shots","gca_fouled","gca_og_for"] +#defensive actions(defense) +defense = ["player","nationality","position","squad","age","birth_year","minutes_90s","tackles","tackles_won","tackles_def_3rd","tackles_mid_3rd","tackles_att_3rd","dribble_tackles","dribbles_vs","dribble_tackles_pct","dribbled_past","pressures","pressure_regains","pressure_regain_pct","pressures_def_3rd","pressures_mid_3rd","pressures_att_3rd","blocks","blocked_shots","blocked_shots_saves","blocked_passes","interceptions","clearances","errors"] +defense2 = ["tackles","tackles_won","tackles_def_3rd","tackles_mid_3rd","tackles_att_3rd","dribble_tackles","dribbles_vs","dribble_tackles_pct","dribbled_past","pressures","pressure_regains","pressure_regain_pct","pressures_def_3rd","pressures_mid_3rd","pressures_att_3rd","blocks","blocked_shots","blocked_shots_saves","blocked_passes","interceptions","clearances","errors"] +#possession(possession) +possession = ["player","nationality","position","squad","age","birth_year","minutes_90s","touches","touches_def_pen_area","touches_def_3rd","touches_mid_3rd","touches_att_3rd","touches_att_pen_area","touches_live_ball","dribbles_completed","dribbles","dribbles_completed_pct","players_dribbled_past","nutmegs","carries","carry_distance","carry_progressive_distance","progressive_carries","carries_into_final_third","carries_into_penalty_area","pass_targets","passes_received","passes_received_pct","miscontrols","dispossessed"] +possession2 = ["touches","touches_def_pen_area","touches_def_3rd","touches_mid_3rd","touches_att_3rd","touches_att_pen_area","touches_live_ball","dribbles_completed","dribbles","dribbles_completed_pct","players_dribbled_past","nutmegs","carries","carry_distance","carry_progressive_distance","progressive_carries","carries_into_final_third","carries_into_penalty_area","pass_targets","passes_received","passes_received_pct","miscontrols","dispossessed"] +#playingtime(playingtime) +playingtime = ["player","nationality","position","squad","age","birth_year","minutes_90s","games","minutes","minutes_per_game","minutes_pct","games_starts","minutes_per_start","games_subs","minutes_per_sub","unused_subs","points_per_match","on_goals_for","on_goals_against","plus_minus","plus_minus_per90","plus_minus_wowy","on_xg_for","on_xg_against","xg_plus_minus","xg_plus_minus_per90","xg_plus_minus_wowy"] +playingtime2 = ["games","minutes","minutes_per_game","minutes_pct","games_starts","minutes_per_start","games_subs","minutes_per_sub","unused_subs","points_per_match","on_goals_for","on_goals_against","plus_minus","plus_minus_per90","plus_minus_wowy","on_xg_for","on_xg_against","xg_plus_minus","xg_plus_minus_per90","xg_plus_minus_wowy"] +#miscallaneous(misc) +misc = ["player","nationality","position","squad","age","birth_year","minutes_90s","cards_yellow","cards_red","cards_yellow_red","fouls","fouled","offsides","crosses","interceptions","tackles_won","pens_won","pens_conceded","own_goals","ball_recoveries","aerials_won","aerials_lost","aerials_won_pct"] +misc2 = ["cards_yellow","cards_red","cards_yellow_red","fouls","fouled","offsides","crosses","interceptions","tackles_won","pens_won","pens_conceded","own_goals","ball_recoveries","aerials_won","aerials_lost","aerials_won_pct"] + + +## Scrape using Beautiful Soup & convert to pandas +def get_tables(url): + res = requests.get(url) + ## The next two lines get around the issue with comments breaking the parsing. + comm = re.compile("") + soup = BeautifulSoup(comm.sub("",res.text),'lxml') + all_tables = soup.findAll("tbody") + team_table = all_tables[0] + player_table = all_tables[1] + return player_table, team_table + +def get_frame(features, player_table): + pre_df_player = dict() + features_wanted_player = features + rows_player = player_table.find_all('tr') + for row in rows_player: + if(row.find('th',{"scope":"row"}) != None): + + for f in features_wanted_player: + cell = row.find("td",{"data-stat": f}) + a = cell.text.strip().encode() + text=a.decode("utf-8") + if(text == ''): + text = '0' + if((f!='player')&(f!='nationality')&(f!='position')&(f!='squad')&(f!='age')&(f!='birth_year')): + text = float(text.replace(',','')) + if f in pre_df_player: + pre_df_player[f].append(text) + else: + pre_df_player[f] = [text] + df_player = pd.DataFrame.from_dict(pre_df_player) + return df_player + +def get_frame_team(features, team_table): + pre_df_squad = dict() + #Note: features does not contain squad name, it requires special treatment + features_wanted_squad = features + rows_squad = team_table.find_all('tr') + for row in rows_squad: + if(row.find('th',{"scope":"row"}) != None): + name = row.find('th',{"data-stat":"squad"}).text.strip().encode().decode("utf-8") + if 'squad' in pre_df_squad: + pre_df_squad['squad'].append(name) + else: + pre_df_squad['squad'] = [name] + for f in features_wanted_squad: + cell = row.find("td",{"data-stat": f}) + a = cell.text.strip().encode() + text=a.decode("utf-8") + if(text == ''): + text = '0' + if((f!='player')&(f!='nationality')&(f!='position')&(f!='squad')&(f!='age')&(f!='birth_year')): + text = float(text.replace(',','')) + if f in pre_df_squad: + pre_df_squad[f].append(text) + else: + pre_df_squad[f] = [text] + df_squad = pd.DataFrame.from_dict(pre_df_squad) + return df_squad + +def frame_for_category(category,top,end,features): + url = (top + category + end) + player_table, team_table = get_tables(url) + df_player = get_frame(features, player_table) + return df_player + +def frame_for_category_team(category,top,end,features): + url = (top + category + end) + player_table, team_table = get_tables(url) + df_team = get_frame_team(features, team_table) + return df_team + + +#Function to get the player data for outfield player, includes all categories - standard stats, shooting +#passing, passing types, goal and shot creation, defensive actions, possession, and miscallaneous +def get_outfield_data(top, end): + df1 = frame_for_category('stats',top,end,stats) + df2 = frame_for_category('shooting',top,end,shooting2) + df3 = frame_for_category('passing',top,end,passing2) + df4 = frame_for_category('passing_types',top,end,passing_types2) + df5 = frame_for_category('gca',top,end,gca2) + df6 = frame_for_category('defense',top,end,defense2) + df7 = frame_for_category('possession',top,end,possession2) + df8 = frame_for_category('misc',top,end,misc2) + df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8], axis=1) + df = df.loc[:,~df.columns.duplicated()] + return df + + +#Function to get keeping and advance goalkeeping data +def get_keeper_data(top,end): + df1 = frame_for_category('keepers',top,end,keepers) + df2 = frame_for_category('keepersadv',top,end,keepersadv2) + df = pd.concat([df1, df2], axis=1) + df = df.loc[:,~df.columns.duplicated()] + return df + + +#Function to get team-wise data accross all categories as mentioned above +def get_team_data(top,end): + df1 = frame_for_category_team('stats',top,end,stats3) + df2 = frame_for_category_team('keepers',top,end,keepers3) + df3 = frame_for_category_team('keepersadv',top,end,keepersadv2) + df4 = frame_for_category_team('shooting',top,end,shooting3) + df5 = frame_for_category_team('passing',top,end,passing2) + df6 = frame_for_category_team('passing_types',top,end,passing_types2) + df7 = frame_for_category_team('gca',top,end,gca2) + df8 = frame_for_category_team('defense',top,end,defense2) + df9 = frame_for_category_team('possession',top,end,possession2) + df10 = frame_for_category_team('misc',top,end,misc2) + df = pd.concat([df1, df2, df3, df4, df5, df6, df7, df8, df9, df10], axis=1) + df = df.loc[:,~df.columns.duplicated()] + return df + + +class FbRefScraper: + def __init__(self, top5=False, leagues=["Bundesliga"], seasons=[2021]): + if top5: + self.leagues = ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"] + else: + self.leagues = leagues + self.seasons = seasons + return + + def scrape_players(self, csv_path=None): + df_outfield = pd.DataFrame() + for league in self.leagues: + if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]: + print("League must be one of ", ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]) + continue + for season in self.seasons: + print(f"Loading player data for {league} season {season-1}/{season}") + link = copy.deepcopy(league_links[league]) + if season != 2021: + link[0] = link[0] + f"{year_links[league][season]}/" + link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "") + df_outfield_league = get_outfield_data(link[0], link[1]) + df_outfield_league["season"] = season + df_outfield = df_outfield.append(df_outfield_league, ignore_index=True) + if csv_path is not None: + df_outfield.to_csv(csv_path, index=False) + return df_outfield + + def scrape_teams(self, csv_path=None, include_vs=False): + df_teams = pd.DataFrame() + for league in self.leagues: + if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]: + print("League must be one of ", ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]) + continue + for season in self.seasons: + print(f"Loading team data for {league} season {season-1}/{season}") + link = copy.deepcopy(league_links[league]) + if season != 2021: + link[0] = link[0] + f"{year_links[league][season]}/" + link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "") + df_teams_league = get_team_data(link[0], link[1]) + df_teams_league["season"] = season + if not include_vs: + df_teams_league = df_teams_league[["vs" not in t for t in df_teams_league["squad"]]] + df_teams = df_teams.append(df_teams_league, ignore_index=True) + if csv_path is not None: + df_teams.to_csv(csv_path, index=False) + return df_teams + + def scrape_goalkeepers(self, csv_path=None): + df_gks = pd.DataFrame() + for league in self.leagues: + if league not in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]: + print("League must be one of ", ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"]) + continue + for season in self.seasons: + print(f"Loading goalkeeper data for {league} season {season-1}/{season}") + link = copy.deepcopy(league_links[league]) + if season != 2021: + link[0] = link[0] + f"{year_links[league][season]}/" + link[1] = f"/{season-1}-{season}-" + link[1].replace("/", "") + df_gks_league = get_keeper_data(link[0], link[1]) + df_gks_league["season"] = season + df_gks = df_gks.append(df_gks_league, ignore_index=True) + if csv_path is not None: + df_gks.to_csv(csv_path, index=False) + return df_gks + +if __name__ == "__main__": + # Example Usage + # Leagues must be in ["Bundesliga", "Premier League", "Ligue 1", "Serie A", "La Liga"] + # Season is end-year of the season (20/21 --> 2021) + scraper = FbRefScraper(leagues=["Premier League"], seasons=[2019]) + # Outfield players + # Add csv_path to store df to csv + df_outfield = scraper.scrape_players(csv_path="outfield.csv") + print(df_outfield.head()) + + # Team data + df_teams = scraper.scrape_teams(include_vs=False) + print(df_teams.head()) + + # Goalkeepers + df_gks = scraper.scrape_goalkeepers() + print(df_gks.head()) +