diff --git a/.github/ISSUE_TEMPLATE/add-resource.yml b/.github/ISSUE_TEMPLATE/add-resource.yml index 4213d27..69ec3ec 100644 --- a/.github/ISSUE_TEMPLATE/add-resource.yml +++ b/.github/ISSUE_TEMPLATE/add-resource.yml @@ -25,6 +25,14 @@ body: placeholder: https://github.com/Carvera-Community/Carvera_Controller validations: required: true + - type: input + id: source + attributes: + label: Source code + description: Optional. If the link above is a website or web app, link its source repository here. + placeholder: https://github.com/owner/project + validations: + required: false - type: dropdown id: category attributes: diff --git a/.github/ISSUE_TEMPLATE/update-entry.yml b/.github/ISSUE_TEMPLATE/update-entry.yml index 8f44098..de86d01 100644 --- a/.github/ISSUE_TEMPLATE/update-entry.yml +++ b/.github/ISSUE_TEMPLATE/update-entry.yml @@ -33,6 +33,13 @@ body: description: Leave empty to keep the current link. validations: required: false + - type: input + id: new_source + attributes: + label: New source code link + description: Optional. Adds or replaces the entry's source-code link (for web apps and sites). Type "remove" to drop it. + validations: + required: false - type: input id: new_name attributes: diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index badc18b..5e9f2e5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -6,7 +6,7 @@ This repository is a curated "awesome list" about Makera desktop CNC machines (C Check every added or changed entry and flag: -- **Format:** each entry must be exactly `- [Name](https://link) - Description.` with a plain ` - ` separator (not an en or em dash). +- **Format:** each entry must be exactly `- [Name](https://link) - Description.` with a plain ` - ` separator (not an en or em dash). Entries whose main link is a website or web app may end with an optional source link: `- [Name](https://app) - Description. ([Source code](https://repo))`. - **Description style:** one sentence, starts with a capital letter, ends with a period, under 160 characters. It must not start with "A", "An" or the entry's own name, and must not use marketing superlatives ("best", "ultimate", "powerful"). - **Order:** entries are alphabetical (case-insensitive) within each `##` section. - **Relevance:** the resource must be specific to Makera machines: software, controllers, firmware, CAM workflows and post-processors, upgrades, printable mods, tooling, guides, videos or communities. Flag generic CNC content with no Makera angle (it belongs in Awesome CNC), general 3D printing, reseller and off-topic links. diff --git a/.github/scripts/awesome_bot/ai.py b/.github/scripts/awesome_bot/ai.py index a138a10..e190f8a 100644 --- a/.github/scripts/awesome_bot/ai.py +++ b/.github/scripts/awesome_bot/ai.py @@ -37,6 +37,8 @@ marketplace listing, affiliate link, link shortener, scraped copy, or SEO/content-farm article; - it is not a duplicate of an existing entry (compare against the existing names provided); - it fits the chosen category (if not, still approve but put the better category in "category"). +When a separate source-code link is given, the main link may be the project's website or web app; +that is expected. Verdict "needs_review" when it is plausible but you are unsure: niche, little information available, commercial with unclear value, possible self-promotion of a small or brand-new project, or the page could not be read. Verdict "reject" for off-topic content, spam, low-effort pages, or anything harmful. @@ -61,6 +63,7 @@ Name: {name} Link: {url} +Source code: {source} Category chosen by submitter: {category} Pricing: {pricing} Submitter's description: {description} @@ -120,6 +123,7 @@ Section: {section} Name: {name} Link: {url} +Source code: {source} Description: {description} @@ -132,6 +136,7 @@ New section: {new_section} New name: {new_name} New link: {new_url} +New source code link: {new_source} New description: {new_description} Reason given: {reason} @@ -251,6 +256,7 @@ def assess(cfg, submission, categories, existing_names, link, pr_description="") categories="\n".join(f"- {c}" for c in categories), name=_untrusted(submission.name, 120), url=_untrusted(submission.url, 500), + source=_untrusted(submission.source, 500) or "(none)", category=category or "(none / not sure)", pricing=_untrusted(submission.pricing, 60) or "unknown", description=_untrusted(submission.description, 600) or "(none)", @@ -279,11 +285,13 @@ def page(link): user = CHANGE_USER_PROMPT.format( categories="\n".join(f"- {c}" for c in categories), section=old.section, name=_untrusted(old.name, 120), url=_untrusted(old.url, 500), + source=_untrusted(old.source, 500) or "(none)", description=_untrusted(old.description, 300), old_page=page(old_link), action="remove the entry" if new is None else "change the entry", new_section=new.section if new else "-", new_name=_untrusted(new.name, 120) if new else "-", new_url=_untrusted(new.url, 500) if new else "-", + new_source=(_untrusted(new.source, 500) or "(none)") if new else "-", new_description=_untrusted(request.new_description, 600) or "(unchanged)", reason=_untrusted(request.reason, 1200) or "(none given)", new_page=page(new_link), diff --git a/.github/scripts/awesome_bot/commands.py b/.github/scripts/awesome_bot/commands.py index 56748de..6a2d1be 100644 --- a/.github/scripts/awesome_bot/commands.py +++ b/.github/scripts/awesome_bot/commands.py @@ -132,18 +132,37 @@ def process_issue(cfg, gh, number, force=False): problems.append(f"Already listed under **{duplicate.section}** as [{duplicate.name}]({duplicate.url}).") if not section and sub.category != forms.NOT_SURE: problems.append(f"Unknown **Category** \"{ai.clean_note(sub.category, 80)}\". Pick one from the dropdown.") - - link = None + source = sub.source.strip().strip("<>") + if not source and (found := readme.find_repo_link(f"{sub.description}\n{sub.why}", exclude=url)): + source = found + notes.append(f"Using the repository mentioned in the submission as the source-code link: {found}") + if source and is_valid_url(url) and normalize_url(source) == normalize_url(url): + source = "" # the main link already is the repository + if source: + if not is_valid_url(source): + problems.append("**Source code** must be a full URL starting with `https://`.") + elif (other := doc.find_url(source)) and not duplicate: + problems.append(f"That source code is already listed under **{other.section}** as [{other.name}]({other.url}).") + sub.source = source + + link = source_link = None if not problems: - link = check_url(url) - if link.ok is False: - (notes if force else problems).append(f"The link looks broken ({link.error}).") - elif link.ok is None: - notes.append(f"Couldn't verify the link automatically ({link.error}), so a maintainer will check it.") + for target in filter(None, (url, source)): + result = check_url(target) + what = "link" if target == url else "source-code link" + if result.ok is False: + (notes if force else problems).append(f"The {what} looks broken ({result.error}).") + elif result.ok is None: + notes.append(f"Couldn't verify the {what} automatically ({result.error}), so a maintainer will check it.") + if target == url: + link = result + else: + source_link = result log.section("Checks", [ ("Duplicate", f"yes: {duplicate.name} in {duplicate.section}" if duplicate else "none"), ("Category", section or f"{sub.category} (Copilot will suggest one)"), *log.link_rows("Link", link, url), + *(log.link_rows("Source code", source_link, source) if source else []), ("Problems", problems), ]) @@ -167,16 +186,17 @@ def process_issue(cfg, gh, number, force=False): if not problems: description = (assessment and assessment.description) or readme.normalize_description( sub.description, cfg.max_description_length) - entry_problems = readme.check_entry(name, url, description, cfg.max_description_length) + entry_problems = readme.check_entry(name, url, description, cfg.max_description_length, source) problems += [p.message for p in entry_problems if p.level == "error"] notes += [p.message for p in entry_problems if p.level == "warning"] - proposal = readme.Entry(name, url, description, -1, section) + proposal = readme.Entry(name, url, description, -1, section, source) if problems: status = "needs_changes" elif force: status = "approved" elif (assessment.verdict == "approve" and not assessment.error and link and link.ok + and (source_link is None or source_link.ok) and assessment.confidence >= cfg.auto_approve_min_confidence): status = "ai_approved" elif assessment.verdict == "reject": @@ -361,13 +381,28 @@ def process_update(cfg, gh, issue, labels, force, notes): problems.append(f"That link is already on the list as [{other.name}]({other.url}).") description = (readme.normalize_description(request.new_description, cfg.max_description_length) if request.new_description.strip() else old.description) - new = readme.Entry(readme.sanitize_name(request.new_name) or old.name, url, description, -1, section or old.section) + source = old.source + if new_source := request.new_source.strip().strip("<>"): + if new_source.casefold() in ("remove", "none", "-"): + source = "" + elif not is_valid_url(new_source): + problems.append("**New source code link** must be a full URL starting with `https://`, or \"remove\".") + elif (other := doc.find_url(new_source)) and other.line != old.line: + problems.append(f"That source code is already on the list as [{other.name}]({other.url}).") + else: + source = new_source + new = readme.Entry(readme.sanitize_name(request.new_name) or old.name, url, description, -1, + section or old.section, source) if new.render() == old.render() and new.section == old.section: problems.append("Nothing would change. Fill in at least one of the **New** fields.") old_link = new_link = None if old and not problems: old_link = check_url(old.url) + if new and new.source and new.source != old.source: + source_check = check_url(new.source) + if source_check.ok is False: + (notes if force else problems).append(f"The new source-code link looks broken ({source_check.error}).") if new and new.url != old.url: new_link = check_url(new.url) if new_link.ok is False: @@ -395,7 +430,7 @@ def process_update(cfg, gh, issue, labels, force, notes): if new and assessment.category and assessment.category != new.section: notes.append(f"Automated review suggests **{assessment.category}** for this entry.") if new: - entry_problems = readme.check_entry(new.name, new.url, new.description, cfg.max_description_length) + entry_problems = readme.check_entry(new.name, new.url, new.description, cfg.max_description_length, new.source) problems += [p.message for p in entry_problems if p.level == "error"] notes += [p.message for p in entry_problems if p.level == "warning"] @@ -555,7 +590,7 @@ def review_pr(cfg, gh, number): link = check_url(entry.url) section = head_doc.section(entry.section) sub = forms.Submission( - name=entry.name, url=entry.url, category=entry.section, description=entry.description, + name=entry.name, url=entry.url, source=entry.source, category=entry.section, description=entry.description, affiliation="unknown", ) existing = [e.name for e in section.entries if e is not entry] diff --git a/.github/scripts/awesome_bot/forms.py b/.github/scripts/awesome_bot/forms.py index 1df880f..3b7c651 100644 --- a/.github/scripts/awesome_bot/forms.py +++ b/.github/scripts/awesome_bot/forms.py @@ -7,6 +7,7 @@ FIELD_LABELS = { "name": "Name", "url": "Link", + "source": "Source code", "category": "Category", "description": "Description", "pricing": "Pricing", @@ -19,6 +20,7 @@ "entry": "Entry", "action": "What should happen?", "new_url": "New link", + "new_source": "New source code link", "new_name": "New name", "new_description": "New description", "new_section": "New section", @@ -35,6 +37,7 @@ class Submission: name: str = "" url: str = "" + source: str = "" category: str = "" description: str = "" pricing: str = "" @@ -63,6 +66,7 @@ class UpdateRequest: entry: str = "" action: str = "" new_url: str = "" + new_source: str = "" new_name: str = "" new_description: str = "" new_section: str = "" diff --git a/.github/scripts/awesome_bot/readme.py b/.github/scripts/awesome_bot/readme.py index 472dc75..9553e6b 100644 --- a/.github/scripts/awesome_bot/readme.py +++ b/.github/scripts/awesome_bot/readme.py @@ -14,6 +14,10 @@ ENTRY_RE = re.compile( r"^- \[(?P[^\[\]]+)\]\((?P(?:[^()\s]|\([^()\s]*\))+)\) - (?P\S.*)$" ) +# Optional second link for entries whose main link is a website or web app: "... ([Source code](url))". +SOURCE_RE = re.compile(r"^(?P.*\S)\s+\(\[Source code\]\((?P[^()\s]+)\)\)$") +REPO_LINK_RE = re.compile( + r"https?://(?:www\.)?(?:github\.com|gitlab\.com|codeberg\.org|bitbucket\.org|git\.sr\.ht)/[^\s)>\]]+", re.I) H2_RE = re.compile(r"^## (?P.+?)\s*$") # Mirrors awesome-lint's ToC rule: these sections never appear in the table of contents. TOC_EXCLUDED = {"Contents", "Contributing", "Footnotes", "Related Lists"} @@ -28,9 +32,14 @@ class Entry: description: str line: int # 0-based index into Readme.lines section: str + source: str = "" # optional source-code link def render(self): - return format_entry(self.name, self.url, self.description) + return format_entry(self.name, self.url, self.description, self.source) + + @property + def urls(self): + return [self.url, self.source] if self.source else [self.url] @dataclass @@ -74,7 +83,10 @@ def _parse(self): section.subheadings.append(i) elif line.startswith("- ") or line.startswith("* "): if m := ENTRY_RE.match(line): - section.entries.append(Entry(m["name"], m["url"], m["desc"], i, title)) + desc, source = m["desc"], "" + if s := SOURCE_RE.match(desc): + desc, source = s["desc"], s["source"] + section.entries.append(Entry(m["name"], m["url"], desc, i, title, source)) else: section.bad_lines.append((i, line)) sections.append(section) @@ -98,12 +110,23 @@ def entries(self): return [e for s in self.categories for e in s.entries] def find_url(self, url): + """The entry using this link, as its main link or its source-code link.""" key = normalize_url(url) - return next((e for e in self.entries() if normalize_url(e.url) == key), None) + return next((e for e in self.entries() if key in map(normalize_url, e.urls)), None) + + +def format_entry(name, url, description, source=""): + line = f"- [{name}]({url}) - {description}" + return f"{line} ([Source code]({source}))" if source else line -def format_entry(name, url, description): - return f"- [{name}]({url}) - {description}" +def find_repo_link(text, exclude=""): + """First source-repository URL mentioned in free text (for submitters who paste it in the description).""" + for match in REPO_LINK_RE.finditer(text or ""): + url = match.group(0).rstrip(".,;:!?'\"") + if not exclude or normalize_url(url) != normalize_url(exclude): + return url + return "" def sort_key(name): @@ -125,6 +148,8 @@ def sanitize_name(name): def normalize_description(text, max_length): """Best-effort cleanup of free-form text into awesome-list description style.""" text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text or "") # markdown links -> text + text = re.sub(r"\(?\s*https?://\S+?\s*\)?(?=\s|$)", " ", text) # bare URLs belong in the link fields + text = re.sub(r"\s+([.,;:!?])", r"\1", text) text = re.sub(r"[`*_<>|\[\]\\]", "", text) text = re.sub(r"\s+", " ", text).strip().lstrip("-–— ").strip() text = re.sub(r"^(an?|the)\s+(?=\w)", "", text, flags=re.I) @@ -142,15 +167,18 @@ def normalize_description(text, max_length): return text -def check_entry(name, url, description, max_length): +def check_entry(name, url, description, max_length, source=""): """Problems with a single entry, independent of its position in the README.""" problems = [] if not name.strip(): problems.append(Problem("Entry name is empty.")) - if not is_valid_url(url): - problems.append(Problem(f"Link `{url}` is not an absolute http(s) URL.")) - elif url.startswith("http://"): - problems.append(Problem(f"Prefer an `https://` link for `{url}` if the site supports it.", level="warning")) + for link in filter(None, (url, source)): + if not is_valid_url(link): + problems.append(Problem(f"Link `{link}` is not an absolute http(s) URL.")) + elif link.startswith("http://"): + problems.append(Problem(f"Prefer an `https://` link for `{link}` if the site supports it.", level="warning")) + if source and normalize_url(source) == normalize_url(url): + problems.append(Problem(f"The source-code link of `{name}` is the same as its main link; leave it out.")) if not description: problems.append(Problem(f"`{name}` needs a description.")) return problems @@ -209,14 +237,14 @@ def validate(text, max_length, meta_sections): problems.append(Problem(f"Entries must look like {ENTRY_FORMAT}", line + 1)) names, previous = set(), None for entry in section.entries: - for p in check_entry(entry.name, entry.url, entry.description, max_length): + for p in check_entry(entry.name, entry.url, entry.description, max_length, entry.source): p.line = entry.line + 1 problems.append(p) - key = normalize_url(entry.url) - if key in seen: - problems.append(Problem(f"Duplicate link: `{entry.name}` points to the same page as `{seen[key]}`.", entry.line + 1)) - else: - seen[key] = entry.name + for key in {normalize_url(u) for u in entry.urls}: + if key in seen: + problems.append(Problem(f"Duplicate link: `{entry.name}` points to the same page as `{seen[key]}`.", entry.line + 1)) + else: + seen[key] = entry.name if entry.name.casefold() in names: problems.append(Problem(f"`{entry.name}` appears twice in `{section.title}`.", entry.line + 1)) names.add(entry.name.casefold()) @@ -243,13 +271,13 @@ def new_errors(before_text, after_text, max_length, meta_sections): return fresh -def insert_entry(text, section_title, name, url, description, meta_sections): +def insert_entry(text, section_title, name, url, description, meta_sections, source=""): readme = Readme(text, meta_sections) section = readme.section(section_title) if section is None or section.title in readme.meta: raise ValueError(f"Unknown section: {section_title}") lines = list(readme.lines) - new_line = format_entry(name, url, description) + new_line = format_entry(name, url, description, source) if section.entries: key = sort_key(name) after = next((e for e in section.entries if sort_key(e.name) > key), None) @@ -299,7 +327,7 @@ def find_entries(readme, query): query = m["url"] if is_valid_url(query): key = normalize_url(query) - return [e for e in readme.entries() if normalize_url(e.url) == key] + return [e for e in readme.entries() if key in map(normalize_url, e.urls)] name = sanitize_name(query).casefold() return [e for e in readme.entries() if e.name.casefold() == name] if name else [] @@ -335,5 +363,5 @@ def apply_entry_diff(text, removed, added, meta_sections): text = "\n".join(lines) for entry in added: if not Readme(text, meta_sections).find_url(entry.url): - text = insert_entry(text, entry.section, entry.name, entry.url, entry.description, meta_sections) + text = insert_entry(text, entry.section, entry.name, entry.url, entry.description, meta_sections, entry.source) return text, missing diff --git a/.github/scripts/tests/test_awesome_bot.py b/.github/scripts/tests/test_awesome_bot.py index 2c952e2..d3ac4b9 100644 --- a/.github/scripts/tests/test_awesome_bot.py +++ b/.github/scripts/tests/test_awesome_bot.py @@ -122,6 +122,23 @@ def test_apply_entry_diff_update_remove_and_idempotence(self): # removing an entry that's already gone is reported self.assertEqual(readme.apply_entry_diff(text, [alpha], [], META)[1], [alpha]) + def test_source_code_links(self): + line = "- [Web](https://web.example.com) - Browser app. ([Source code](https://github.com/o/web))" + text = SAMPLE.replace("- [Gamma](https://gamma.example.com/) - Third tool.", + "- [Gamma](https://gamma.example.com/) - Third tool.\n" + line) + doc = readme.Readme(text, META) + web = doc.section("Software").entries[-1] + self.assertEqual((web.description, web.source, web.render()), ("Browser app.", "https://github.com/o/web", line)) + self.assertIs(doc.find_url("https://github.com/O/web/"), web) + self.assertEqual([p.message for p in readme.validate(text, 160, META) if p.level == "error"], []) + dup = text.replace("https://github.com/o/web", "https://alpha.example.com") + self.assertTrue(any("Duplicate link" in p.message for p in readme.validate(dup, 160, META))) + self.assertTrue(readme.check_entry("Web", "https://w.example.com", "Browser.", 160, "https://w.example.com/")) + moved, _ = readme.apply_entry_diff(text, [web], [readme.Entry("Web", web.url, "Browser app.", -1, "Related Lists", web.source)], META) + self.assertIn(line, moved.split("## Related Lists")[1]) + self.assertEqual(readme.find_repo_link("See https://github.com/o/r. Thanks", exclude="https://x.example.com"), "https://github.com/o/r") + self.assertEqual(readme.normalize_description("tool for cnc https://github.com/o/r", 160), "Tool for cnc.") + def test_normalize_description(self): self.assertEqual(readme.normalize_description("a [cool](http://x) `tool` for cnc ", 160), "Cool tool for cnc.") long = readme.normalize_description("Word " * 60, 50) @@ -297,6 +314,14 @@ def test_approved_submission_opens_pr(self): self.assertIn(cfg.label("ai_approved"), gh.labels_added) self.assertIn("Opened #99", gh.comments[-1]) + def test_repository_in_description_becomes_source_link(self): + body = self.body(url="https://beta.example.com").replace( + "Web-based interface.", "Web app for CNC. Code: https://github.com/o/beta") + gh = self.make(body) + cfg = self.run_issue(gh) + self.assertIn("- [Beta](https://beta.example.com) - Second tool. ([Source code](https://github.com/o/beta))", + gh.files[cfg.bot_branch_prefix + "7"]) + def test_duplicate_is_flagged_without_ai(self): gh = self.make(self.body(url="https://www.alpha.example.com/")) cfg = self.run_issue(gh) @@ -348,8 +373,9 @@ def test_unknown_category(self): def update_body(entry="https://alpha.example.com", action="Change it", new_url="", new_name="", - new_description="", new_section="", reason="Moved."): - fields = [("Entry", entry), ("What should happen?", action), ("New link", new_url), ("New name", new_name), + new_description="", new_section="", reason="Moved.", new_source=""): + fields = [("Entry", entry), ("What should happen?", action), ("New link", new_url), + ("New source code link", new_source), ("New name", new_name), ("New description", new_description), ("New section", new_section), ("Why?", reason)] return "\n\n".join(f"### {k}\n\n{v or '_No response_'}" for k, v in fields) @@ -387,6 +413,21 @@ def test_change_link_and_description(self): self.assertNotIn("https://alpha.example.com)", text) self.assertIn("```diff", gh.comments[-1]) + def test_add_and_remove_source_link(self): + gh, cfg, _ = self.run_update(update_body(new_source="https://github.com/o/alpha")) + self.assertIn("- [Alpha](https://alpha.example.com) - First tool. ([Source code](https://github.com/o/alpha))", + self.branch_text(gh, cfg)) + with_source = SAMPLE.replace("First tool.", "First tool. ([Source code](https://github.com/o/alpha))") + issue = {"number": 8, "state": "open", "body": update_body(new_source="remove"), "user": {"login": "fixer"}, + "labels": [{"name": "entry update"}]} + gh = FakeGitHub(issue, with_source) + ok = mock.Mock(ok=True, error="", status=200, final_url="", title="", description="") + ok.summary.return_value = "reachable" + with mock.patch.object(commands, "check_url", return_value=ok), \ + mock.patch.object(commands.ai, "assess_change", return_value=ai.Assessment(verdict="approve", confidence=0.9, model="m")): + commands.process_issue(cfg, gh, 8) + self.assertIn("- [Alpha](https://alpha.example.com) - First tool.\n", gh.files[cfg.bot_branch_prefix + "8"]) + def test_move_section(self): gh, cfg, _ = self.run_update(update_body(new_section="Firmware & Controllers")) self.assertEqual(gh.prs[0][0], "Move Alpha to Firmware & Controllers") diff --git a/contributing.md b/contributing.md index 2fc5d21..7f9f9c0 100644 --- a/contributing.md +++ b/contributing.md @@ -39,6 +39,13 @@ Use the **[Fix or remove an entry](https://github.com/blackveilprecision/awesome - Mention "Open-source" or "Commercial" when it's relevant. - Keep it under 160 characters. - Entries are sorted alphabetically within each section. +- If the main link is a website or web app and the project has a separate source repository, add it at the end: + + ```markdown + - [Name](https://app.example.com) - Description. ([Source code](https://github.com/owner/project)) + ``` + + In the form, put the repository in the optional **Source code** field. ## Sending a pull request instead