ResearchGate Scraper.user.js

// ==UserScript==
// @name        ResearchGate Scraper
// @namespace   Violentmonkey Scripts
// @match       https://www.researchgate.net/institution/University-of-Barcelona/members/*
// @grant       none
// @version     1.0
// @author      Your Name
// @description Script to scrape professor data from ResearchGate
// ==/UserScript==

function scrapePage() {
  const members = document.querySelectorAll('.nova-legacy-v-person-list-item__body');
  let data = JSON.parse(localStorage.getItem('scrapedData')) || [];

  for (const member of members) {
    const name = member.querySelector('h5').textContent;
    const link = "https://www.researchgate.net" + member.querySelector('a').getAttribute('href');
    const departmentInfo = member.querySelector('.nova-legacy-v-person-list-item__info-section li');
    const department = departmentInfo ? departmentInfo.textContent : '';

    const fieldsOfInterest = ["soc", "law", "eco", "human", "polit"];
    if (fieldsOfInterest.some(field => department.toLowerCase().includes(field))) {
      data.push({
        University: 'UB',
        Departament: department,
        Position: '',
        Treatment: '',
        Name: name,
        Email: '',
        Professor_Link: link
      });
    }
  }

  localStorage.setItem('scrapedData', JSON.stringify(data));

  // Set a random delay of 5 to 10 seconds before navigating to the next page
  const randomDelay = Math.random() * 5000 + 5000;
  setTimeout(navigateToNextPage, randomDelay);
}

function navigateToNextPage() {
  const currentPageNumber = parseInt(window.location.pathname.split('/').pop(), 10);
  if (!isNaN(currentPageNumber) && currentPageNumber < 16420) {
    const nextPageNumber = currentPageNumber + 1;
    const nextPageUrl = `https://www.researchgate.net/institution/University-of-Barcelona/members/${nextPageNumber}`;
    window.location.href = nextPageUrl;
  }
}

// Wait for the page to load, then run the scrapePage function
window.addEventListener('load', scrapePage);