Improve banstick's proxy ip collection

ansible/files/paper-config/plugins/BanStick/config.yml

@@ -99,7 +99,19 @@ proxy:
     ban:
       length: 163800000
       message: 'VPS / VPN Proxy use is generally prohibited. Issue a ticket at support in the CivMC Discord to discuss: https://discord.gg/nDnsU6vJqg'
-    url: https://raw.githubusercontent.com/client9/ipcat/master/datacenters.csv
+    url: https://raw.githubusercontent.com/growlfm/ipcat/refs/heads/main/datacenters.csv
+    period: 576000
+    delay: 4200
+  proxylist:
+    enable: true
+    defaultScore: 3.0
+    ban:
+      length: 86400000
+      message: VPS / VPN Proxy use is generally prohibited. Modmail to www.reddit.com/r/CivMC to discuss
+    urls:
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt
     period: 576000
     delay: 4200
 # Configure the tor check source; set autoban to true to not only track tor usage but also pre-set bans for tor users

ansible/files/pvp-config/plugins/BanStick/config.yml

@@ -99,7 +99,19 @@ proxy:
     ban:
       length: 163800000
       message: 'VPS / VPN Proxy use is generally prohibited. Issue a ticket at support in the CivMC Discord to discuss: https://discord.gg/nDnsU6vJqg'
-    url: https://raw.githubusercontent.com/client9/ipcat/master/datacenters.csv
+    url: https://raw.githubusercontent.com/growlfm/ipcat/refs/heads/main/datacenters.csv
+    period: 576000
+    delay: 4200
+  proxylist:
+    enable: true
+    defaultScore: 3.0
+    ban:
+      length: 86400000
+      message: VPS / VPN Proxy use is generally prohibited. Modmail to www.reddit.com/r/CivMC to discuss
+    urls:
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt
     period: 576000
     delay: 4200
 # Configure the tor check source; set autoban to true to not only track tor usage but also pre-set bans for tor users

plugins/banstick-paper/src/main/java/com/programmerdan/minecraft/banstick/data/BSIPData.java

@@ -302,7 +302,8 @@ public static BSIPData byExactIP(BSIP ip) {
                     allIPDataID.put(data.idid, data);
                     return data;
                 } else {
-                    BanStick.getPlugin().warning("Failed to retrieve IP Data by exact IP: {0} - not found", ip);
+                    // No reason to log here, just spams console, especially on import
+                    // BanStick.getPlugin().warning("Failed to retrieve IP Data by exact IP: {0} - not found", ip);
                 }
             }
         } catch (SQLException se) {

plugins/banstick-paper/src/main/java/com/programmerdan/minecraft/banstick/proxy/PROXYListLoader.java

@@ -0,0 +1,139 @@
+package com.programmerdan.minecraft.banstick.proxy;
+
+import com.programmerdan.minecraft.banstick.BanStick;
+import com.programmerdan.minecraft.banstick.data.BSBan;
+import com.programmerdan.minecraft.banstick.data.BSIP;
+import com.programmerdan.minecraft.banstick.data.BSIPData;
+import com.programmerdan.minecraft.banstick.handler.ProxyLoader;
+import inet.ipaddr.IPAddress;
+import inet.ipaddr.IPAddressString;
+import org.bukkit.configuration.ConfigurationSection;
+import org.jetbrains.annotations.NotNull;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.net.URL;
+import java.util.*;
+
+public final class PROXYListLoader extends ProxyLoader {
+
+    private @NotNull List<String> urls;
+    private float proxyScore;
+    private boolean autoBan;
+    private long banLength;
+    private String banMessage;
+
+    public PROXYListLoader(ConfigurationSection config) {
+        super(config);
+    }
+
+    @Override
+    public void run() {
+        for (String url : this.urls) {
+            try {
+                BanStick.getPlugin().debug("Scraping {0}", url);
+
+                URL connection = new URI(url).toURL();
+                InputStream readIn = connection.openStream();
+                BufferedReader in = new BufferedReader(new InputStreamReader(readIn));
+                String line = in.readLine();
+                int errors = 0;
+                long lines = 0;
+                while (line != null) {
+                    lines++;
+                    if (lines % 50 == 0) {
+                        BanStick.getPlugin().info("Checked {0} entries from proxylist so far", lines);
+                    }
+                    if (errors > 10) {
+                        break;
+                    }
+
+                    String[] parts = line.split(":");
+                    if (parts.length != 2) {
+                        errors++;
+                        continue;
+                    }
+                    String ip = parts[0];
+                    String port = parts[1];
+                    IPAddressString ipAddressString = new IPAddressString(ip);
+                    ipAddressString.validate();
+                    IPAddress ipAddress = ipAddressString.toAddress();
+
+                    BSIP found = BSIP.byIPAddress(ipAddress);
+                    if (found == null) {
+                        found = BSIP.create(ipAddress);
+                    }
+
+                    BSIPData data = BSIPData.byExactIP(found);
+                    if (data == null) {
+                        data = BSIPData.create(found, null, null, null, null, null, null, null,
+                            null, null, null, null, proxyScore, "proxylist Proxy Loader", "Anon Proxy on Port: " + port);
+                    }
+
+                    if (autoBan) {
+                        boolean wasmatch = false;
+
+                        List<BSBan> ban = BSBan.byProxy(data, true);
+                        if (!(ban == null || ban.isEmpty())) {
+                            // look for match; if unexpired, extend.
+                            for (int i = ban.size() - 1; i >= 0; i--) {
+                                BSBan pickOne = ban.get(i);
+                                if (pickOne.isAdminBan()) {
+                                    continue; // skip admin entered bans.
+                                }
+                                if (pickOne.getBanEndTime() != null && pickOne.getBanEndTime().after(new Date())) {
+                                    if (this.banLength < 0) { // endless
+                                        pickOne.clearBanEndTime();
+                                    } else {
+                                        pickOne.setBanEndTime(new Date(System.currentTimeMillis()
+                                            + this.banLength));
+                                    }
+                                    wasmatch = true;
+                                    break;
+                                }
+                            }
+                        }
+                        if (!wasmatch) {
+                            BSBan.create(data, this.banMessage,
+                                this.banLength < 0 ? null : new Date(System.currentTimeMillis()
+                                    + this.banLength), false);
+                        }
+                    }
+
+                    line = in.readLine();
+                }
+                if (errors > 10) {
+                    BanStick.getPlugin().warning("Cancelled proxylist load due to too many errors.");
+                }
+                BanStick.getPlugin().info("Finished loading {0} entries from proxylist: {1}", lines, url);
+            } catch (Exception e) {
+                BanStick.getPlugin().warning("Failed reading from proxylist: " + url);
+                BanStick.getPlugin().warning("  Failure message: ", e);
+            }
+        }
+    }
+
+    @Override
+    public void setup(ConfigurationSection config) {
+        this.proxyScore = (float) config.getDouble("defaultScore", 3.0d);
+        this.autoBan = config.isConfigurationSection("ban");
+        if (this.autoBan) {
+            this.banLength = config.getLong("ban.length", -1L);
+            this.banMessage = config.getString("ban.message", null);
+        }
+
+        if (config.isList("urls")) {
+            this.urls = config.getStringList("urls");
+            BanStick.getPlugin().info("Loaded {0} proxylist URLs", this.urls.size());
+        } else {
+            this.urls = new ArrayList<>();
+            BanStick.getPlugin().warning("No proxylist URLs configured, loader will not run.");
+        }
+    }
+
+    @Override
+    public String name() {
+        return "proxylist";
+    }
+}

plugins/banstick-paper/src/main/java/com/programmerdan/minecraft/banstick/scraper/ScrapeFreeProxyList.java

@@ -29,12 +29,13 @@ public class ScrapeFreeProxyList extends ScraperWorker {
     private long banLength;
 
     private String[] urls = new String[]{
-        "https://free-proxy-list.net",
-        "http://www.sslproxies.org/",
-        "http://www.us-proxy.org/",
-        "http://free-proxy-list.net/uk-proxy.html",
-        "http://www.socks-proxy.net/",
-        "http://free-proxy-list.net/anonymous-proxy.html"
+        "https://free-proxy-list.net/en/",
+        "https://free-proxy-list.net/en/socks-proxy.html",
+        "https://free-proxy-list.net/en/us-proxy.html",
+        "https://free-proxy-list.net/en/uk-proxy.html",
+        "https://free-proxy-list.net/en/ssl-proxy.html",
+        "https://free-proxy-list.net/en/anonymous-proxy.html",
+        "https://free-proxy-list.net/en/google-proxy.html",
     };
 
     public ScrapeFreeProxyList(ConfigurationSection config) {
@@ -66,31 +67,36 @@ public void scrape() {
      */
     public void scrapeOne(String url) {
         try {
+            int count = 0;
             BanStick.getPlugin().debug("Scraping {0}", url);
             Document doc = Jsoup.connect(url).userAgent(
                 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0").get();
-            Elements iptable = doc.select("table#proxylisttable");
 
-            Elements body = iptable.select("tbody");
-            Elements trs = body.select("tr");
-            BanStick.getPlugin().debug("Found {0} proxy IPs to scrape", trs.size());
-            for (Element tr : trs) {
-                Elements tds = tr.select("td");
+            Element table = doc.selectXpath("//*[@id=\"list\"]/div/div[2]/div/table").first();
+            if (table == null) {
+                BanStick.getPlugin().warning("No table found at " + url);
+                return;
+            }
+
+            Elements rows = table.select("tbody > tr");
+            BanStick.getPlugin().debug("Found {0} proxy IPs to scrape", rows.size());
+            for (Element row : rows) {
+                Elements cell = row.select("td");
                 String ip = null;
                 try {
-                    ip = tds.first().text();
+                    ip = cell.first().text();
                 } catch (Exception e) {
                     continue;
                 }
                 String country = null;
                 try {
-                    country = tds.get(3).text();
+                    country = cell.get(3).text();
                 } catch (Exception e) {
                     // intentionally ignore errors.
                 }
                 String port = null;
                 try {
-                    port = tds.get(1).text();
+                    port = cell.get(1).text();
                 } catch (Exception e) {
                     // intentionally ignore errors.
                 }
@@ -136,10 +142,14 @@ public void scrapeOne(String url) {
                                 : new Date(System.currentTimeMillis() + this.banLength), false);
                         }
                     }
+
+                    count++;
                 } catch (IPAddressStringException iase) {
                     continue;
                 }
             }
+
+            BanStick.getPlugin().info("Scraped {0} proxy IPs from {1}", count, url);
         } catch (IOException ioe) {
             BanStick.getPlugin().warning("Failure during scrape of " + url, ioe);
             this.registerError();

plugins/banstick-paper/src/main/resources/config.yml

@@ -97,9 +97,22 @@ proxy:
     ban:
       length: 163800000
       message: VPS / VPN Proxy use is generally prohibited. Modmail to www.reddit.com/r/Devoted to discuss
-    url: https://raw.githubusercontent.com/client9/ipcat/master/datacenters.csv
+    url: https://raw.githubusercontent.com/growlfm/ipcat/refs/heads/main/datacenters.csv
     period: 576000
     delay: 4200
+  proxylist:
+    enable: true
+    defaultScore: 3.0
+    ban:
+      length: 86400000
+      message: VPS / VPN Proxy use is generally prohibited. Modmail to www.reddit.com/r/CivMC to discuss
+    urls:
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt
+      - https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt
+    period: 576000
+    delay: 4200
+
 # Configure the tor check source; set autoban to true to not only track tor usage but also pre-set bans for tor users
 tor:
   check: true