From 112d6d521efab8bfd61983ded952f3c5fd5f7c6a Mon Sep 17 00:00:00 2001 From: diamondpete <87245367+diamondpete@users.noreply.github.com> Date: Mon, 11 May 2026 21:53:35 -0400 Subject: [PATCH 1/4] feat: New Architecture for Title Case Parser --- Contents/Code/PAactors.py | 2 +- Contents/Code/PAgenres.py | 2 +- Contents/Code/PAparseTitle.py | 428 ++++++++++++++++++++++++++++++++++ Contents/Code/PAutils.py | 177 +------------- 4 files changed, 434 insertions(+), 175 deletions(-) create mode 100644 Contents/Code/PAparseTitle.py diff --git a/Contents/Code/PAactors.py b/Contents/Code/PAactors.py index a11f87c1a..a72d90d4d 100644 --- a/Contents/Code/PAactors.py +++ b/Contents/Code/PAactors.py @@ -53,7 +53,7 @@ def processActors(self, metadata, siteNum): for actorLink in self.actorsTable: skip = False # Save the potential new Actor or Actress to a new variable, replace   with a true space, and strip off any surrounding whitespace - actorName = PAutils.parseTitle(actorLink['name'].replace('\xc2\xa0', ' ').replace(',', '').strip().lower(), 0) + actorName = PAutils.parseTitle(actorLink['name'].replace('\xc2\xa0', ' ').replace(',', '').strip(), 0, title_type='name') actorPhoto = actorLink['photo'].strip() actorName = ' '.join(actorName.split()) diff --git a/Contents/Code/PAgenres.py b/Contents/Code/PAgenres.py index 0cda3183d..7244655d9 100644 --- a/Contents/Code/PAgenres.py +++ b/Contents/Code/PAgenres.py @@ -40,7 +40,7 @@ def processGenres(self, metadata, siteNum): break if not found: - genreName = PAutils.parseTitle(genreName, siteNum) + genreName = PAutils.parseTitle(genreName, siteNum, title_type='genre') if not found and not skip: if len(genreName) > 25: diff --git a/Contents/Code/PAparseTitle.py b/Contents/Code/PAparseTitle.py new file mode 100644 index 000000000..25bafcf47 --- /dev/null +++ b/Contents/Code/PAparseTitle.py @@ -0,0 +1,428 @@ +import PAsearchSites + + +class Token(object): + def __init__(self, text, kind): + self.text = text + self.kind = kind + self.normalized = None + + +class TitleCaseEngine(object): + + # ----------------------------- + # CONSTANTS + # ----------------------------- + + WORD_RE = re.compile(r'\w+', re.UNICODE) + SPACE_RE = re.compile(r'\s+', re.UNICODE) + + LOWER_EXCEPTIONS = set([ + 'a', 'y', 'n', 'an', 'of', 'the', 'and', 'for', 'to', 'onto', + 'but', 'or', 'nor', 'at', 'with', 'vs', 'com', 'co', 'org', + ]) + + UPPER_EXCEPTIONS = set([ + 'bbc', 'xxx', 'bbw', 'bf', 'bff', 'bts', 'pov', 'dp', 'gf', 'bj', 'wtf', 'cfnm', 'bwc', 'fm', 'tv', + 'hd', 'milf', 'gilf', 'dilf', 'dtf', 'zz', 'xxxl', 'usa', 'nsa', 'hr', 'ii', 'iii', 'iv', 'bbq', + 'avn', 'xtc', 'atv', 'joi', 'rpg', 'wunf', 'uk', 'asap', 'sss', 'nf', 'pawg', 'ama', + ]) + + ACRONYMS = set([ + 'ai', 'vr', 'hd', 'uhd', 'sd', 'hdr', '4k', '3d', '2d', + ]) + + SIZE_CODES = set([ + 'xs', 's', 'm', 'l', 'xl', 'xxl', 'xxxl', 'xxxxl' + ]) + + NAME_EXCEPTIONS = set([ + 'ai' + ]) + + NAME_EXCEPTION_SITES = set([ + '912', '13671' '1411', '1600', '1685' + ]) + + CONTRACTION_EXCEPTIONS = set([ + 're', 't', 's', 'd', 'll', 've', 'm', 'am', 'ed' + ]) + + SYMBOLS = set(['-', '/', '.', '+', '\'']) + + SYMBOL_RULES = { + '-': {'join': True, 'treat_as': 'compound'}, + '/': {'join': True, 'treat_as': 'compound'}, + '.': {'join': True, 'treat_as': 'initials_or_acronym'}, + '+': {'join': True, 'treat_as': 'compound'}, + '\'': {'join': True, 'treat_as': 'contraction'}, + } + + MANUAL_CORRECTIONS = { + 'im': 'I\'m', 'theyll': 'They\'ll', 'cant': 'Can\'t', 'ive': 'I\'ve', 'shes': 'She\'s', 'theyre': 'They\'re', 'tshirt': 'T-Shirt', 'dont': 'Don\'t', + 'wasnt': 'Wasn\'t', 'youre': 'You\'re', 'ill': 'I\'ll', 'whats': 'What\'s', 'didnt': 'Didn\'t', 'isnt': 'Isn\'t', 'senor': 'Señor', 'senorita': 'Señorita', + 'thats': 'That\'s', 'gstring': 'G-String', 'milfs': 'MILFs', 'oreilly': 'O\'Reilly', 'bangbros': 'BangBros', 'bday': 'B-Day', 'dms': 'DMs', 'bffs': 'BFFs', + 'ohmy': 'OhMy', 'wont': 'Won\'t', 'whos': 'Who\'s', 'shouldnt': 'Shouldn\'t', 'lasirena': 'LaSirena', 'espanol': 'español', 'jmac': 'J-Mac', 'youd': 'You\'d', + 'redwolf': 'RedWolf', 'mccray': 'McCray', 'mccullough': 'McCullough', 'mccall': 'McCall', 'mccarthy': 'McCarthy', + } + + POST_PROCESS_RULES = [ + # Normalize curly quotes + (r'“', '"', 0), + (r'”', '"', 0), + (r'’', '\'', 0), + # Rotate trailing ", the/a/an" to front + (r'^(.*?)(,\s*(the|a|an))$', lambda m: m.group(3).capitalize() + ' ' + m.group(1), re.IGNORECASE), + # Add missing space after ! : ? (but not before domains) + (r'(?<=[!:\?])(?=\w)(?!(co\b|net\b|com\b|org\b|porn\b|E\d|xxx\b))', ' ', re.IGNORECASE), + # Add missing space after period when followed by a letter (not domains) + (r'(?<=\.)(?=[A-Za-z])(?!co\b|net\b|com\b|org\b|porn\b|E\d|xxx\b)', ' ', re.IGNORECASE), + # Remove stray period at end of title (but not ".." etc.) + (r'(? 0 and i + 1 < length and s[i - 1].isalnum() and s[i + 1].isalnum(): + prev = tokens[-1] + prev.text = prev.text + '\'' + i += 1 + continue + + # Standalone apostrophe + tok = Token('\'', 'symbol') + tokens.append(tok) + self.trace('TOKEN(symbol):', tok.text) + i += 1 + continue + + # EVERYTHING ELSE = punctuation + tok = Token(ch, 'punct') + tokens.append(tok) + self.trace('TOKEN(punct):', tok.text) + i += 1 + + return tokens + + # ----------------------------- + # MISSING BUILT-INS + # ----------------------------- + + def next(self, iterable, default=None): + for item in iterable: + return item + return default + + def any(self, iterable): + for item in iterable: + if item: + return True + return False + + # ----------------------------- + # WORD RULE ENGINE + # ----------------------------- + + def apply_word_rules(self, tokens, siteNum): + site_name = PAsearchSites.getSearchSiteName(siteNum) + clean_site = self.nonword_re.sub('', site_name.replace(' ', '')).lower() + + for idx, token in enumerate(tokens): + if token.kind != 'word': + continue + token.normalized = self.normalize_word(token.text, idx, tokens, site_name, siteNum, clean_site) + + self.capitalize_first_word(tokens) + return tokens + + def is_acronym_or_size(self, clean_lower, clean_word, siteNum): + if clean_lower in self.NAME_EXCEPTIONS and siteNum in self.NAME_EXCEPTION_SITES: + return False, None + + if clean_lower in self.LOWER_EXCEPTIONS: + return False, None + + if self.title_type == 'name': + return False, None + + if clean_lower in self.SIZE_CODES: + return True, clean_word.upper() + + if clean_lower in self.ACRONYMS: + return True, clean_word.upper() + + if 2 <= len(clean_word) <= 4 and clean_word.upper() == clean_word: + return True, clean_word.upper() + + return False, None + + def normalize_word(self, word, idx, tokens, site_name, siteNum, clean_site): + + clean_word = self.nonword_re.sub('', word) + clean_lower = clean_word.lower() + + self.trace('FSM(word):', word) + + # Site name match + if clean_site and clean_lower == clean_site: + self.trace(' -> RULE: site match') + return self.manual_word_fix(site_name) + + # Symbol-containing word + symbol = self.next((s for s in self.SYMBOLS if s in word), None) + if symbol: + self.trace(' -> RULE: symbol word') + return self.manual_word_fix(self.handle_symbol_word(word, site_name, siteNum)) + + # Contraction handling + if '\'' in word: + base, _, suffix = word.partition('\'') + clean_suffix = self.nonword_re.sub('', suffix).lower() + if clean_suffix in self.CONTRACTION_EXCEPTIONS: + self.trace(' -> RULE: contraction') + base_norm = self.normalize_word(base, idx, tokens, site_name, siteNum, clean_site) + return base_norm + '\'' + clean_suffix + + # Acronym / size code + is_special, special_val = self.is_acronym_or_size(clean_lower, clean_word, siteNum) + if is_special: + self.trace(' -> RULE: acronym/size') + return self.manual_word_fix(special_val) + + # Explicit upper exceptions + if clean_lower in self.UPPER_EXCEPTIONS: + self.trace(' -> RULE: upper exception') + return self.manual_word_fix(word.upper()) + + # Force upper if all caps and not a lower exception + if clean_word and clean_word.isupper() and clean_lower not in self.LOWER_EXCEPTIONS: + self.trace(' -> RULE: force upper') + return self.manual_word_fix(word.upper()) + + # Explicit lower exceptions + if clean_lower in self.LOWER_EXCEPTIONS: + self.trace(' -> RULE: lower exception') + return self.manual_word_fix(word.lower()) + + # Brand/stylized casing: mixed case, leave as-is (then manual fix) + if self.any(c.islower() for c in list(word)) and self.any(c.isupper() for c in list(word)): + self.trace(' -> RULE: mixed-case brand') + return self.manual_word_fix(word) + + # Default: capitalize + self.trace(' -> RULE: default capitalize') + return self.manual_word_fix(word.capitalize()) + + # ----------------------------- + # SYMBOL HANDLER + # ----------------------------- + + def handle_symbol_word(self, word, site_name, siteNum): + self.trace('SYMBOL-HANDLER:', word) + + symbol = None + for s in self.SYMBOLS: + if s in word: + symbol = s + break + + if not symbol: + self.trace(' -> No symbol found') + return word + + rule = self.SYMBOL_RULES.get(symbol, {'join': True, 'treat_as': 'compound'}) + self.trace(' -> Symbol:', symbol, 'Rule:', rule) + + parts = word.split(symbol) + sep = symbol if rule['join'] else '' + + result_parts = [] + + for idx, part in enumerate(parts): + clean = self.nonword_re.sub('', part) + lower_clean = clean.lower() + + self.trace(' PART:', part) + + if not part: + result_parts.append(part) + continue + + if rule['treat_as'] == 'contraction' and lower_clean in self.CONTRACTION_EXCEPTIONS: + self.trace(' -> contraction part') + norm = part.lower() + + elif rule['treat_as'] == 'initials_or_acronym' and len(clean) == 1: + self.trace(' -> initial') + norm = clean.upper() + + else: + self.trace(' -> apply normalize_word') + norm = self.normalize_word(part, idx, None, site_name, siteNum, None) + + norm = self.manual_word_fix(norm) + result_parts.append(norm) + + final = sep.join(result_parts) + self.trace('SYMBOL-HANDLER RESULT:', final) + return final + + # ----------------------------- + # FIRST WORD CAPITALIZATION + # ----------------------------- + + def capitalize_first_word(self, tokens): + for token in tokens: + if token.kind == 'word': + text = token.normalized or token.text + clean = self.nonword_re.sub('', text) + if not clean: + return + if len(clean) > 1: + token.normalized = text[0].upper() + text[1:] + else: + token.normalized = text.upper() + return + + # ----------------------------- + # MANUAL WORD FIX + # ----------------------------- + + def manual_word_fix(self, word): + cached = self.manual_fix_cache.get(word) + if cached is not None: + return cached + + clean = self.nonword_re.sub('', word).lower() + + if clean in self.MANUAL_CORRECTIONS: + corrected = self.MANUAL_CORRECTIONS[clean] + self.trace('MANUAL-FIX:', word, '->', corrected) + fixed = re.sub(re.escape(clean), corrected, word, flags=re.IGNORECASE) + self.manual_fix_cache[word] = fixed + return fixed + + self.manual_fix_cache[word] = word + return word + + # ----------------------------- + # RECONSTRUCT + # ----------------------------- + + def reconstruct(self, tokens): + parts = [] + for t in tokens: + parts.append(t.normalized if t.normalized is not None else t.text) + return ''.join(parts) + + # ----------------------------- + # POST-PROCESS + # ----------------------------- + + def post_process(self, output): + self.trace('POST-PROCESS START:', output) + + for pattern, repl, flags in self.POST_PROCESS_RULES: + new_output = re.sub(pattern, repl, output, flags=flags) + if new_output != output: + self.trace('RULE HIT:', pattern, '->', new_output) + output = new_output + + self.trace('POST-PROCESS END:', output) + return output diff --git a/Contents/Code/PAutils.py b/Contents/Code/PAutils.py index dadf8325c..07b15b293 100644 --- a/Contents/Code/PAutils.py +++ b/Contents/Code/PAutils.py @@ -12,6 +12,7 @@ from HTMLParser import HTMLParser import PAsearchSites +from PAparseTitle import TitleCaseEngine UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' @@ -291,71 +292,9 @@ def saveRequest(url, req): return True -def parseTitle(s, siteNum): - s = re.sub(r'w\/(?!\s)', 'w/ ', s, flags=re.IGNORECASE) - s = re.sub(r'\,(?![\s|\d])', ', ', s) - s = s.replace('_', ' ') - s = s.replace('’', '\'') - s = s.replace('´', '\'') - s = preParseTitle(s) - word_list = re.split(' ', s) - - if not word_list: - return s - - pattern = re.compile(r'\W') - final = [] - - for idx, word in enumerate(word_list): - parsed = parseWord(word, siteNum) - - # Capitalize first word only - if idx == 0: - cleanWord = re.sub(pattern, '', parsed) - if cleanWord: - parsed = parsed[0].upper() + parsed[1:] if len(cleanWord) > 1 else parsed.upper() - - final.append(parsed) - - output = ' '.join(final) - output = postParseTitle(output) - - return output - - -def parseWord(word, siteNum): - lower_exceptions = ['a', 'y', 'n', 'an', 'of', 'the', 'and', 'for', 'to', 'onto', 'but', 'or', 'nor', 'at', 'with', 'vs', 'in', 'on', 'com', 'co', 'org'] - upper_exceptions = ( - 'bbc', 'xxx', 'bbw', 'bf', 'bff', 'bts', 'pov', 'dp', 'gf', 'bj', 'wtf', 'cfnm', 'bwc', 'fm', 'tv', 'ai', - 'hd', 'milf', 'gilf', 'dilf', 'dtf', 'zz', 'xxxl', 'usa', 'nsa', 'hr', 'ii', 'iii', 'iv', 'bbq', 'avn', 'xtc', 'atv', - 'joi', 'rpg', 'wunf', 'uk', 'asap', 'sss', 'nf', 'pawg', 'ama', 'xs', 'xl', 'xxl', 'xxxl' - ) - symbols_map = {'-': '-', '/': '/', '.': r'\.', '+': r'\+', '\'': r'\''} - - pattern = re.compile(r'\W') - cleanWord = re.sub(pattern, '', word) - cleanWordLower = cleanWord.lower() - cleanSiteName = re.sub(pattern, '', PAsearchSites.getSearchSiteName(siteNum).replace(' ', '')) - - if cleanSiteName.lower() == cleanWordLower: - word = PAsearchSites.getSearchSiteName(siteNum) - elif any(symbol in word for symbol in symbols_map): - for symbol, escaped_symbol in symbols_map.items(): - if symbol in word: - word = parseTitleSymbol(word, siteNum, escaped_symbol) - break - elif cleanWordLower in upper_exceptions: - word = word.upper() - elif cleanWord.isupper() and cleanWordLower not in lower_exceptions: - word = word.upper() - elif not (cleanWord.islower() or cleanWord.isupper() or cleanWordLower in lower_exceptions): - pass - else: - word = word.lower() if cleanWordLower in lower_exceptions else word.capitalize() - - word = manualWordFix(word) - - return word +def parseTitle(s, siteNum, title_type='title'): + engine = TitleCaseEngine(title_type, debug=Prefs['debug_enable']) + return engine.parse_title(s, siteNum) def any(s): @@ -365,114 +304,6 @@ def any(s): return False -def parseTitleSymbol(word, siteNum, symbol): - lower_exceptions = set(['vs']) - contraction_exceptions = set(['re', 't', 's', 'd', 'll', 've', 'm', 'am', 'ed']) - symbols = set(['-', '/', r'\.', r'\+']) - nonword_re = re.compile(r'\W') - - parts = re.split(symbol, word) - # Normalize first part - first = parseWord(parts[0], siteNum) - if first not in lower_exceptions: - if first and re.search(r'^\W', first): - # Preserve leading punctuation then uppercase the next char(s) - if len(first) >= 2: - first = first[:2].upper() + first[2:] - else: - first = first.upper() - elif len(first) > 1: - first = first[0].upper() + first[1:] - else: - first = first.upper() - - sep = symbol.replace('\\', '') - result = first + sep - - # Process remaining parts - total = len(parts) - for idx, part in enumerate(parts[1:], 1): - clean = nonword_re.sub('', part) - if symbol in symbols: - if idx == 1 and not first: - result += part.capitalize() - elif len(part) > 1: - result += parseWord(part, siteNum) - else: - result += part.capitalize() - elif clean.lower() in contraction_exceptions: - result += part.lower() - else: - result += parseWord(part, siteNum) - - if idx != total - 1: - result += sep - - return result - - -def postParseTitle(output): - replace = [(r'“', '\"'), (r'”', '\"'), (r'’', '\''), (r'W/', 'w/'), (r'A\.\sJ\.', 'A.J.'), (r'T\.\sJ\.', 'T.J.'), (r'(? Date: Mon, 11 May 2026 22:00:45 -0400 Subject: [PATCH 2/4] feat(reptyle): Remove genresDB and Add Manual Fallback for D18 --- Contents/Code/networkReptyle.py | 64 +++++++++------------------------ 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/Contents/Code/networkReptyle.py b/Contents/Code/networkReptyle.py index 2e1419ca9..efee0cd1b 100644 --- a/Contents/Code/networkReptyle.py +++ b/Contents/Code/networkReptyle.py @@ -207,8 +207,6 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, if (len(actors) > 1) and subSite != 'Mylfed': genres.append('Threesome') - genres.extend(PAutils.getDictValuesFromKey(genresDB, subSite)) - for genreLink in genres: genreName = genreLink @@ -217,59 +215,20 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, # Posters art.append(detailsPageElements['img']) if Prefs['data18_enable']: - providers = ['TeamSkeet', 'MYLF', 'Family Strokes', 'Pervz', 'FreeUse', 'Swappz', subSite] - data18Url = getSceneURLFromData18(metadata.title, providers, date_object if sceneDate else None) - if data18Url: - getData18Images(data18Url, art, metadata) + scene_id = PAutils.getDictKeyFromValues(data18ManualMappings, detailsPageElements['id']) + if scene_id: + data18Url = ('https://www.data18.com/scenes/%s' % scene_id[0]) else: - getArtwork(metadata, art) + providers = ['TeamSkeet', 'MYLF', 'Family Strokes', 'Pervz', 'FreeUse', 'Swappz', subSite] + data18Url = getSceneURLFromData18(metadata.title, providers, date_object if sceneDate else None) + + getData18Images(data18Url, art, metadata) if data18Url else getArtwork(metadata, art) else: getArtwork(metadata, art) return metadata -genresDB = { - 'Anal Mom': ['Anal', 'MILF'], - 'BFFs': ['Teen', 'Group Sex'], - 'Black Valley Girls': ['Teen', 'Ebony'], - 'DadCrush': ['Step Dad', 'Step Daughter'], - 'DaughterSwap': ['Step Dad', 'Step Daughter'], - 'Dyked': ['Hardcore', 'Teen', 'Lesbian'], - 'Exxxtra Small': ['Teen', 'Small Tits'], - 'Family Strokes': ['Taboo Family'], - 'Foster Tapes': ['Taboo Sex'], - 'Freeuse Fantasy': ['Freeuse'], - 'Full Of JOI': ['JOI'], - 'Ginger Patch': ['Redhead'], - 'Innocent High': ['School Girl'], - 'Little Asians': ['Asian', 'Teen'], - 'Lone MILF': ['Solo'], - 'Milf Body': ['Gym', 'Fitness'], - 'Milfty': ['Cheating'], - 'MomDrips': ['Creampie'], - 'MylfBlows': ['Blowjob'], - 'MylfBoss': ['Office', 'Boss'], - 'MylfDom': ['BDSM'], - 'Mylfed': ['Lesbian', 'Girl on Girl', 'GG'], - 'Not My Grandpa': ['Older/Younger'], - 'Oye Loca': ['Latina'], - 'PervMom': ['Step Mom'], - 'POV Life': ['POV'], - 'Shoplyfter': ['Strip'], - 'ShoplyfterMylf': ['Strip', 'MILF'], - 'Sis Loves Me': ['Step Sister'], - 'Teen Curves': ['Big Ass'], - 'Teen Pies': ['Teen', 'Creampie'], - 'TeenJoi': ['Teen', 'JOI'], - 'Teens Do Porn': ['Teen'], - 'Teens Love Black Cocks': ['Teens', 'BBC'], - 'Teeny Black': ['Teen', 'Ebony'], - 'Thickumz': ['Thick'], - 'Titty Attack': ['Big Tits'], -} - - familystrokesDB = { 'Ask Your Mother', 'Black Step Dad', 'Dad Crush', 'Family Strokes', 'Family Strokes Features', 'Foster Tapes', 'Not My Grandpa', 'Perv Mom', 'Perv Nana', 'Sis Loves Me', 'Tiny Sis', @@ -314,3 +273,12 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, 'Teens Love Black Cocks', 'Teens Love Money', 'Teeny Black', 'The Loft', 'The Real Workout', 'Thickumz', 'This Girl Sucks', 'Titty Attack', 'Tomboyz', } + + +data18ManualMappings = { + 169646: ['thats-better-than-stealing-it'], + 1313219: ['delicious-firsts'], + 1349311: ['thanksgiving-the-hijab-way'], + 1341218: ['the-vamp-next-door'], + 1341212: ['home-for-the-holidays'], +} From c3df17b33a8816f5ce5adf536c655120e4ed21d3 Mon Sep 17 00:00:00 2001 From: diamondpete <87245367+diamondpete@users.noreply.github.com> Date: Mon, 11 May 2026 22:03:42 -0400 Subject: [PATCH 3/4] fix: Nubiles Scraper --- Contents/Code/PAcaptchaHelper.py | 125 +++++++++++++++++++++++++++++++ Contents/Code/PAutils.py | 6 ++ Contents/Code/networkNubiles.py | 23 +++++- 3 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 Contents/Code/PAcaptchaHelper.py diff --git a/Contents/Code/PAcaptchaHelper.py b/Contents/Code/PAcaptchaHelper.py new file mode 100644 index 000000000..fc62414da --- /dev/null +++ b/Contents/Code/PAcaptchaHelper.py @@ -0,0 +1,125 @@ +import hashlib + + +class CaptchaHelper(object): + + def __init__(self, debug=False): + self.debug = debug + self.USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' + + @staticmethod + def solve_pow(challenge, difficulty): + """ + Find a nonce such that SHA-256(challenge + ':' + nonce) + has `difficulty` leading zero bits. + """ + target = 1 << (256 - difficulty) + nonce = 0 + + while True: + s = "%s:%d" % (challenge, nonce) + h = hashlib.sha256(s.encode('utf-8')).digest() + + # Convert digest to integer (big‑endian) + h_int = int(h.encode('hex'), 16) + + if h_int < target: + return nonce + + nonce += 1 + + def get_verified_cookies(self, base_url): + """ + Solve the homegrown PoW captcha and return a dict of verified session cookies. + + Returns None on failure. Returns an empty dict if the site doesn't actually + present the captcha (already accessible). + """ + base = base_url.rstrip('/') + gallery_url = base + '/video/gallery' + + session = requests.Session() + session.headers.update({ + 'User-Agent': self.USER_AGENT, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept-Encoding': 'gzip, deflate, br', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', + }) + + try: + r = session.get(gallery_url, timeout=15) + except requests.RequestException as e: + Log.Warn("get_verified_cookies: GET %s failed: %s" % (gallery_url, e)) + return None + + if r.status_code == 429: + Log.Warn("get_verified_cookies: rate-limited on %s" % gallery_url) + return None + + m = re.search(r"var\s+turnstileConfig\s*=\s*(\{.*?\});", r.text) + if not m: + return dict(session.cookies) + + try: + config = json.loads(m.group(1)) + nonce = self.solve_pow(config['challenge'], config['difficulty']) + except Exception as e: + Log.Warn("get_verified_cookies: PoW solve failed for %s: %s" % (base, e)) + return None + + verify_url = base + '/turnstile/verify' + payload = { + 'nonce': str(nonce), + 'timestamp': config['timestamp'], + 'difficulty': config['difficulty'], + 'environmentChecks': { + 'screenWidth': 1920, + 'screenHeight': 1080, + 'hasCanvas': True, + 'hasWebGL': True, + 'colorDepth': 24, + 'timezoneOffset': 300, + 'languages': 'en-US,en', + 'platform': 'Win32', + 'cookieEnabled': True, + }, + 'returnTo': config['returnTo'], + } + + try: + vr = session.post( + verify_url, + data=json.dumps(payload), + headers={ + 'Content-Type': 'application/json', + 'Referer': gallery_url, + 'Origin': base, + }, + timeout=15, + ) + except requests.RequestException as e: + Log.Warn("get_verified_cookies: POST %s failed: %s" % (verify_url, e)) + return None + + if vr.status_code != 200: + Log.Warn("get_verified_cookies: verify POST returned %d for %s" % (vr.status_code, base)) + return None + + try: + result = vr.json() + except ValueError: + Log.Warn("get_verified_cookies: non-JSON verify response for %s" % base) + return None + + if not result.get('success'): + Log.Warn("get_verified_cookies: verify failed for %s: %s" % (base, result)) + return None + + if self.debug: + Log.Debug("get_verified_cookies: solved PoW for %s (nonce=%d)" % (base, nonce)) + return dict(session.cookies) diff --git a/Contents/Code/PAutils.py b/Contents/Code/PAutils.py index 07b15b293..4d664404b 100644 --- a/Contents/Code/PAutils.py +++ b/Contents/Code/PAutils.py @@ -13,6 +13,7 @@ import PAsearchSites from PAparseTitle import TitleCaseEngine +from PAcaptchaHelper import CaptchaHelper UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' @@ -297,6 +298,11 @@ def parseTitle(s, siteNum, title_type='title'): return engine.parse_title(s, siteNum) +def nCookies(siteNum): + helper = CaptchaHelper(debug=Prefs['debug_enable']) + return helper.get_verified_cookies(PAsearchSites.getSearchBaseURL(siteNum)) + + def any(s): for v in s: if v: diff --git a/Contents/Code/networkNubiles.py b/Contents/Code/networkNubiles.py index f6f6b1a6b..e1ddb738e 100644 --- a/Contents/Code/networkNubiles.py +++ b/Contents/Code/networkNubiles.py @@ -4,9 +4,10 @@ def search(results, lang, siteNum, searchData): cookies = {'18-plus-modal': 'hidden'} - headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'} + cookies.update(PAutils.nCookies(siteNum)) + if searchData.date: - url = PAsearchSites.getSearchSearchURL(siteNum) + 'date/' + searchData.date + '/' + searchData.date + url = '%sdate/%s/%s' % (PAsearchSites.getSearchSearchURL(siteNum), searchData.date, searchData.date) req = PAutils.HTTPRequest(url, cookies=cookies, headers=headers) searchResults = HTML.ElementFromString(req.text) for searchResult in searchResults.xpath('//div[contains(@class, "content-grid-item")]'): @@ -43,9 +44,9 @@ def search(results, lang, siteNum, searchData): def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, art): cookies = {'18-plus-modal': 'hidden'} - headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8'} + cookies.update(PAutils.nCookies(siteNum)) metadata_id = str(metadata.id).split('|') - sceneURL = PAsearchSites.getSearchBaseURL(siteNum) + '/video/watch/' + metadata_id[0] + sceneURL = '%s/video/watch/%s' % (PAsearchSites.getSearchBaseURL(siteNum), metadata_id[0]) req = PAutils.HTTPRequest(sceneURL, cookies=cookies, headers=headers) detailsPageElements = HTML.ElementFromString(req.text) @@ -210,3 +211,17 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, art.extend(postersClean) return metadata + + +headers = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + 'Sec-Ch-Ua': '"Microsoft Edge";v="107", "Chromium";v="107", "Not=A?Brand";v="24"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Upgrade-Insecure-Requests': '1', + } From 04689d8e0919bab7caa183dc65cb4cf4a3d77652 Mon Sep 17 00:00:00 2001 From: diamondpete <87245367+diamondpete@users.noreply.github.com> Date: Mon, 11 May 2026 22:15:01 -0400 Subject: [PATCH 4/4] refactor: optimizations --- Contents/Code/PAutils.py | 104 +++++++++++++++++++++++--------- Contents/Code/networkNubiles.py | 2 +- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/Contents/Code/PAutils.py b/Contents/Code/PAutils.py index 4d664404b..5d9dd3904 100644 --- a/Contents/Code/PAutils.py +++ b/Contents/Code/PAutils.py @@ -211,7 +211,6 @@ def HTTPRequest(url, method='GET', **kwargs): def getFromSearchEngine(searchText, site='', **kwargs): - stop = kwargs.pop('stop', 10) lang = kwargs.pop('lang', 'en') if isinstance(site, int): @@ -260,32 +259,39 @@ def Decode(text): def getClearURL(url): - newURL = url - if url.startswith('http'): - url = urlparse.urlparse(url) - path = url.path + if not url.startswith('http'): + return url - while '//' in path: - path = path.replace('//', '/') + parsed = urlparse.urlparse(url) + path = parsed.path.replace('//', '/') - newURL = '%s://%s%s' % (url.scheme, url.netloc, path) - if url.query: - newURL += '?%s' % url.query + # Rebuild URL + new_url = '%s://%s%s' % (parsed.scheme, parsed.netloc, path) - return newURL + if parsed.query: + new_url += '?' + parsed.query + + return new_url def saveRequest(url, req): + # Build dated debug directory debug_dir = os.path.join('debug_data', datetime.now().strftime('%d-%m-%Y')) - debug_dir = os.path.realpath(debug_dir) + if not os.path.exists(debug_dir): - os.makedirs(debug_dir) + try: + os.makedirs(debug_dir) + except OSError: + pass + + # Build raw HTTP dump + raw_http = '\r\n'.join(['< Target URL: "%s"' % url, '', dump.dump_all(req).decode('UTF-8', 'replace')]) - raw_http = '< Target URL: "%s"\r\n\r\n' % url - raw_http += dump.dump_all(req).decode('UTF-8', errors='replace') + # Create gzip file + file_name = uuid.uuid4().hex + '.gz' + file_path = os.path.join(debug_dir, file_name) - file_name = '%s.gz' % uuid.uuid4().hex - with gzip.open(os.path.join(debug_dir, file_name), 'wb') as f: + with gzip.open(file_path, 'wb') as f: f.write(raw_http.encode('UTF-8')) Log('GZip request saved as "%s"' % file_name) @@ -389,33 +395,70 @@ def getSearchTitleStrip(title): def getDictValuesFromKey(dictDB, identifier): + ident = str(identifier).lower() + for key, values in dictDB.items(): - keys = list(key) if type(key) == tuple else [key] - for key in keys: - if str(key).lower() == str(identifier).lower(): + key_items = key if isinstance(key, tuple) else (key,) + + for item in key_items: + if str(item).lower() == ident: return values return [] def getDictKeyFromValues(dictDB, identifier): - keys = [] + ident = str(identifier).lower() + result = [] + for key, values in dictDB.items(): - for item in values: - if str(item).lower() == str(identifier).lower(): - keys.append(key) - break + lower_values = [str(v).lower() for v in values] - return keys + if ident in lower_values: + result.append(key) + + return result class MLStripper(HTMLParser): def __init__(self): - self.reset() + HTMLParser.__init__(self) self.text = StringIO() + self.skip_flag = False # skip script/style content + + def handle_starttag(self, tag, attrs): + if tag in ('script', 'style'): + self.skip_flag = True - def handle_data(self, d): - self.text.write(d) + def handle_endtag(self, tag): + if tag in ('script', 'style'): + self.skip_flag = False + + def handle_data(self, data): + if not self.skip_flag: + self.text.write(data) + + def handle_entityref(self, name): + try: + from htmlentitydefs import name2codepoint + except ImportError: + from html.entities import name2codepoint + cp = name2codepoint.get(name) + if cp: + self.text.write(unichr(cp)) + + def handle_charref(self, name): + try: + if name.startswith('x'): + cp = int(name[1:], 16) + else: + cp = int(name) + self.text.write(unichr(cp)) + except: + pass + + def get_data(self): + return self.text.getvalue() def get_data(self): return self.text.getvalue() @@ -429,9 +472,10 @@ def strip_tags(html): def functionTimer(fun, msg, *args): start_time = time.time() - fun(*args) + output = fun(*args) end_time = time.time() Log('%s: %s' % (msg, str(timedelta(seconds=(end_time - start_time))))) + return output def rreplace(s, r, n, o): diff --git a/Contents/Code/networkNubiles.py b/Contents/Code/networkNubiles.py index e1ddb738e..eed4bb400 100644 --- a/Contents/Code/networkNubiles.py +++ b/Contents/Code/networkNubiles.py @@ -224,4 +224,4 @@ def update(metadata, lang, siteNum, movieGenres, movieActors, movieCollections, 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1', - } +}