Support an optional source-code link for web apps and sites

.github/ISSUE_TEMPLATE/add-resource.yml

@@ -25,6 +25,14 @@ body:
       placeholder: https://github.com/Carvera-Community/Carvera_Controller
     validations:
       required: true
+  - type: input
+    id: source
+    attributes:
+      label: Source code
+      description: Optional. If the link above is a website or web app, link its source repository here.
+      placeholder: https://github.com/owner/project
+    validations:
+      required: false
   - type: dropdown
     id: category
     attributes:

.github/ISSUE_TEMPLATE/update-entry.yml

@@ -33,6 +33,13 @@ body:
       description: Leave empty to keep the current link.
     validations:
       required: false
+  - type: input
+    id: new_source
+    attributes:
+      label: New source code link
+      description: Optional. Adds or replaces the entry's source-code link (for web apps and sites). Type "remove" to drop it.
+    validations:
+      required: false
   - type: input
     id: new_name
     attributes:

.github/copilot-instructions.md

@@ -6,7 +6,7 @@ This repository is a curated "awesome list" about Makera desktop CNC machines (C
 
 Check every added or changed entry and flag:
 
-- **Format:** each entry must be exactly `- [Name](https://link) - Description.` with a plain ` - ` separator (not an en or em dash).
+- **Format:** each entry must be exactly `- [Name](https://link) - Description.` with a plain ` - ` separator (not an en or em dash). Entries whose main link is a website or web app may end with an optional source link: `- [Name](https://app) - Description. ([Source code](https://repo))`.
 - **Description style:** one sentence, starts with a capital letter, ends with a period, under 160 characters. It must not start with "A", "An" or the entry's own name, and must not use marketing superlatives ("best", "ultimate", "powerful").
 - **Order:** entries are alphabetical (case-insensitive) within each `##` section.
 - **Relevance:** the resource must be specific to Makera machines: software, controllers, firmware, CAM workflows and post-processors, upgrades, printable mods, tooling, guides, videos or communities. Flag generic CNC content with no Makera angle (it belongs in Awesome CNC), general 3D printing, reseller and off-topic links.

.github/scripts/awesome_bot/ai.py

@@ -37,6 +37,8 @@
   marketplace listing, affiliate link, link shortener, scraped copy, or SEO/content-farm article;
 - it is not a duplicate of an existing entry (compare against the existing names provided);
 - it fits the chosen category (if not, still approve but put the better category in "category").
+When a separate source-code link is given, the main link may be the project's website or web app;
+that is expected.
 Verdict "needs_review" when it is plausible but you are unsure: niche, little information available,
 commercial with unclear value, possible self-promotion of a small or brand-new project, or the page
 could not be read. Verdict "reject" for off-topic content, spam, low-effort pages, or anything harmful.
@@ -61,6 +63,7 @@
 <submission>
 Name: {name}
 Link: {url}
+Source code: {source}
 Category chosen by submitter: {category}
 Pricing: {pricing}
 Submitter's description: {description}
@@ -120,6 +123,7 @@
 Section: {section}
 Name: {name}
 Link: {url}
+Source code: {source}
 Description: {description}
 </entry>
 
@@ -132,6 +136,7 @@
 New section: {new_section}
 New name: {new_name}
 New link: {new_url}
+New source code link: {new_source}
 New description: {new_description}
 Reason given: {reason}
 </change>
@@ -251,6 +256,7 @@ def assess(cfg, submission, categories, existing_names, link, pr_description="")
         categories="\n".join(f"- {c}" for c in categories),
         name=_untrusted(submission.name, 120),
         url=_untrusted(submission.url, 500),
+        source=_untrusted(submission.source, 500) or "(none)",
         category=category or "(none / not sure)",
         pricing=_untrusted(submission.pricing, 60) or "unknown",
         description=_untrusted(submission.description, 600) or "(none)",
@@ -279,11 +285,13 @@ def page(link):
     user = CHANGE_USER_PROMPT.format(
         categories="\n".join(f"- {c}" for c in categories),
         section=old.section, name=_untrusted(old.name, 120), url=_untrusted(old.url, 500),
+        source=_untrusted(old.source, 500) or "(none)",
         description=_untrusted(old.description, 300), old_page=page(old_link),
         action="remove the entry" if new is None else "change the entry",
         new_section=new.section if new else "-",
         new_name=_untrusted(new.name, 120) if new else "-",
         new_url=_untrusted(new.url, 500) if new else "-",
+        new_source=(_untrusted(new.source, 500) or "(none)") if new else "-",
         new_description=_untrusted(request.new_description, 600) or "(unchanged)",
         reason=_untrusted(request.reason, 1200) or "(none given)",
         new_page=page(new_link),

.github/scripts/awesome_bot/commands.py

@@ -132,18 +132,37 @@ def process_issue(cfg, gh, number, force=False):
         problems.append(f"Already listed under **{duplicate.section}** as [{duplicate.name}]({duplicate.url}).")
     if not section and sub.category != forms.NOT_SURE:
         problems.append(f"Unknown **Category** \"{ai.clean_note(sub.category, 80)}\". Pick one from the dropdown.")
-
-    link = None
+    source = sub.source.strip().strip("<>")
+    if not source and (found := readme.find_repo_link(f"{sub.description}\n{sub.why}", exclude=url)):
+        source = found
+        notes.append(f"Using the repository mentioned in the submission as the source-code link: {found}")
+    if source and is_valid_url(url) and normalize_url(source) == normalize_url(url):
+        source = ""  # the main link already is the repository
+    if source:
+        if not is_valid_url(source):
+            problems.append("**Source code** must be a full URL starting with `https://`.")
+        elif (other := doc.find_url(source)) and not duplicate:
+            problems.append(f"That source code is already listed under **{other.section}** as [{other.name}]({other.url}).")
+    sub.source = source
+
+    link = source_link = None
     if not problems:
-        link = check_url(url)
-        if link.ok is False:
-            (notes if force else problems).append(f"The link looks broken ({link.error}).")
-        elif link.ok is None:
-            notes.append(f"Couldn't verify the link automatically ({link.error}), so a maintainer will check it.")
+        for target in filter(None, (url, source)):
+            result = check_url(target)
+            what = "link" if target == url else "source-code link"
+            if result.ok is False:
+                (notes if force else problems).append(f"The {what} looks broken ({result.error}).")
+            elif result.ok is None:
+                notes.append(f"Couldn't verify the {what} automatically ({result.error}), so a maintainer will check it.")
+            if target == url:
+                link = result
+            else:
+                source_link = result
     log.section("Checks", [
         ("Duplicate", f"yes: {duplicate.name} in {duplicate.section}" if duplicate else "none"),
         ("Category", section or f"{sub.category} (Copilot will suggest one)"),
         *log.link_rows("Link", link, url),
+        *(log.link_rows("Source code", source_link, source) if source else []),
         ("Problems", problems),
     ])
 
@@ -167,16 +186,17 @@ def process_issue(cfg, gh, number, force=False):
     if not problems:
         description = (assessment and assessment.description) or readme.normalize_description(
             sub.description, cfg.max_description_length)
-        entry_problems = readme.check_entry(name, url, description, cfg.max_description_length)
+        entry_problems = readme.check_entry(name, url, description, cfg.max_description_length, source)
         problems += [p.message for p in entry_problems if p.level == "error"]
         notes += [p.message for p in entry_problems if p.level == "warning"]
-        proposal = readme.Entry(name, url, description, -1, section)
+        proposal = readme.Entry(name, url, description, -1, section, source)
 
     if problems:
         status = "needs_changes"
     elif force:
         status = "approved"
     elif (assessment.verdict == "approve" and not assessment.error and link and link.ok
+          and (source_link is None or source_link.ok)
           and assessment.confidence >= cfg.auto_approve_min_confidence):
         status = "ai_approved"
     elif assessment.verdict == "reject":
@@ -361,13 +381,28 @@ def process_update(cfg, gh, issue, labels, force, notes):
                 problems.append(f"That link is already on the list as [{other.name}]({other.url}).")
         description = (readme.normalize_description(request.new_description, cfg.max_description_length)
                        if request.new_description.strip() else old.description)
-        new = readme.Entry(readme.sanitize_name(request.new_name) or old.name, url, description, -1, section or old.section)
+        source = old.source
+        if new_source := request.new_source.strip().strip("<>"):
+            if new_source.casefold() in ("remove", "none", "-"):
+                source = ""
+            elif not is_valid_url(new_source):
+                problems.append("**New source code link** must be a full URL starting with `https://`, or \"remove\".")
+            elif (other := doc.find_url(new_source)) and other.line != old.line:
+                problems.append(f"That source code is already on the list as [{other.name}]({other.url}).")
+            else:
+                source = new_source
+        new = readme.Entry(readme.sanitize_name(request.new_name) or old.name, url, description, -1,
+                           section or old.section, source)
         if new.render() == old.render() and new.section == old.section:
             problems.append("Nothing would change. Fill in at least one of the **New** fields.")
 
     old_link = new_link = None
     if old and not problems:
         old_link = check_url(old.url)
+        if new and new.source and new.source != old.source:
+            source_check = check_url(new.source)
+            if source_check.ok is False:
+                (notes if force else problems).append(f"The new source-code link looks broken ({source_check.error}).")
         if new and new.url != old.url:
             new_link = check_url(new.url)
             if new_link.ok is False:
@@ -395,7 +430,7 @@ def process_update(cfg, gh, issue, labels, force, notes):
         if new and assessment.category and assessment.category != new.section:
             notes.append(f"Automated review suggests **{assessment.category}** for this entry.")
         if new:
-            entry_problems = readme.check_entry(new.name, new.url, new.description, cfg.max_description_length)
+            entry_problems = readme.check_entry(new.name, new.url, new.description, cfg.max_description_length, new.source)
             problems += [p.message for p in entry_problems if p.level == "error"]
             notes += [p.message for p in entry_problems if p.level == "warning"]
 
@@ -555,7 +590,7 @@ def review_pr(cfg, gh, number):
         link = check_url(entry.url)
         section = head_doc.section(entry.section)
         sub = forms.Submission(
-            name=entry.name, url=entry.url, category=entry.section, description=entry.description,
+            name=entry.name, url=entry.url, source=entry.source, category=entry.section, description=entry.description,
             affiliation="unknown",
         )
         existing = [e.name for e in section.entries if e is not entry]

.github/scripts/awesome_bot/forms.py

@@ -7,6 +7,7 @@
 FIELD_LABELS = {
     "name": "Name",
     "url": "Link",
+    "source": "Source code",
     "category": "Category",
     "description": "Description",
     "pricing": "Pricing",
@@ -19,6 +20,7 @@
     "entry": "Entry",
     "action": "What should happen?",
     "new_url": "New link",
+    "new_source": "New source code link",
     "new_name": "New name",
     "new_description": "New description",
     "new_section": "New section",
@@ -35,6 +37,7 @@
 class Submission:
     name: str = ""
     url: str = ""
+    source: str = ""
     category: str = ""
     description: str = ""
     pricing: str = ""
@@ -63,6 +66,7 @@ class UpdateRequest:
     entry: str = ""
     action: str = ""
     new_url: str = ""
+    new_source: str = ""
     new_name: str = ""
     new_description: str = ""
     new_section: str = ""

.github/scripts/awesome_bot/readme.py

@@ -14,6 +14,10 @@
 ENTRY_RE = re.compile(
     r"^- \[(?P<name>[^\[\]]+)\]\((?P<url>(?:[^()\s]|\([^()\s]*\))+)\) - (?P<desc>\S.*)$"
 )
+# Optional second link for entries whose main link is a website or web app: "... ([Source code](url))".
+SOURCE_RE = re.compile(r"^(?P<desc>.*\S)\s+\(\[Source code\]\((?P<source>[^()\s]+)\)\)$")
+REPO_LINK_RE = re.compile(
+    r"https?://(?:www\.)?(?:github\.com|gitlab\.com|codeberg\.org|bitbucket\.org|git\.sr\.ht)/[^\s)>\]]+", re.I)
 H2_RE = re.compile(r"^## (?P<title>.+?)\s*$")
 # Mirrors awesome-lint's ToC rule: these sections never appear in the table of contents.
 TOC_EXCLUDED = {"Contents", "Contributing", "Footnotes", "Related Lists"}
@@ -28,9 +32,14 @@ class Entry:
     description: str
     line: int  # 0-based index into Readme.lines
     section: str
+    source: str = ""  # optional source-code link
 
     def render(self):
-        return format_entry(self.name, self.url, self.description)
+        return format_entry(self.name, self.url, self.description, self.source)
+
+    @property
+    def urls(self):
+        return [self.url, self.source] if self.source else [self.url]
 
 
 @dataclass
@@ -74,7 +83,10 @@ def _parse(self):
                     section.subheadings.append(i)
                 elif line.startswith("- ") or line.startswith("* "):
                     if m := ENTRY_RE.match(line):
-                        section.entries.append(Entry(m["name"], m["url"], m["desc"], i, title))
+                        desc, source = m["desc"], ""
+                        if s := SOURCE_RE.match(desc):
+                            desc, source = s["desc"], s["source"]
+                        section.entries.append(Entry(m["name"], m["url"], desc, i, title, source))
                     else:
                         section.bad_lines.append((i, line))
             sections.append(section)
@@ -98,12 +110,23 @@ def entries(self):
         return [e for s in self.categories for e in s.entries]
 
     def find_url(self, url):
+        """The entry using this link, as its main link or its source-code link."""
         key = normalize_url(url)
-        return next((e for e in self.entries() if normalize_url(e.url) == key), None)
+        return next((e for e in self.entries() if key in map(normalize_url, e.urls)), None)
+
+
+def format_entry(name, url, description, source=""):
+    line = f"- [{name}]({url}) - {description}"
+    return f"{line} ([Source code]({source}))" if source else line
 
 
-def format_entry(name, url, description):
-    return f"- [{name}]({url}) - {description}"
+def find_repo_link(text, exclude=""):
+    """First source-repository URL mentioned in free text (for submitters who paste it in the description)."""
+    for match in REPO_LINK_RE.finditer(text or ""):
+        url = match.group(0).rstrip(".,;:!?'\"")
+        if not exclude or normalize_url(url) != normalize_url(exclude):
+            return url
+    return ""
 
 
 def sort_key(name):
@@ -125,6 +148,8 @@ def sanitize_name(name):
 def normalize_description(text, max_length):
     """Best-effort cleanup of free-form text into awesome-list description style."""
     text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text or "")  # markdown links -> text
+    text = re.sub(r"\(?\s*https?://\S+?\s*\)?(?=\s|$)", " ", text)  # bare URLs belong in the link fields
+    text = re.sub(r"\s+([.,;:!?])", r"\1", text)
     text = re.sub(r"[`*_<>|\[\]\\]", "", text)
     text = re.sub(r"\s+", " ", text).strip().lstrip("-–— ").strip()
     text = re.sub(r"^(an?|the)\s+(?=\w)", "", text, flags=re.I)
@@ -142,15 +167,18 @@ def normalize_description(text, max_length):
     return text
 
 
-def check_entry(name, url, description, max_length):
+def check_entry(name, url, description, max_length, source=""):
     """Problems with a single entry, independent of its position in the README."""
     problems = []
     if not name.strip():
         problems.append(Problem("Entry name is empty."))
-    if not is_valid_url(url):
-        problems.append(Problem(f"Link `{url}` is not an absolute http(s) URL."))
-    elif url.startswith("http://"):
-        problems.append(Problem(f"Prefer an `https://` link for `{url}` if the site supports it.", level="warning"))
+    for link in filter(None, (url, source)):
+        if not is_valid_url(link):
+            problems.append(Problem(f"Link `{link}` is not an absolute http(s) URL."))
+        elif link.startswith("http://"):
+            problems.append(Problem(f"Prefer an `https://` link for `{link}` if the site supports it.", level="warning"))
+    if source and normalize_url(source) == normalize_url(url):
+        problems.append(Problem(f"The source-code link of `{name}` is the same as its main link; leave it out."))
     if not description:
         problems.append(Problem(f"`{name}` needs a description."))
         return problems
@@ -209,14 +237,14 @@ def validate(text, max_length, meta_sections):
             problems.append(Problem(f"Entries must look like {ENTRY_FORMAT}", line + 1))
         names, previous = set(), None
         for entry in section.entries:
-            for p in check_entry(entry.name, entry.url, entry.description, max_length):
+            for p in check_entry(entry.name, entry.url, entry.description, max_length, entry.source):
                 p.line = entry.line + 1
                 problems.append(p)
-            key = normalize_url(entry.url)
-            if key in seen:
-                problems.append(Problem(f"Duplicate link: `{entry.name}` points to the same page as `{seen[key]}`.", entry.line + 1))
-            else:
-                seen[key] = entry.name
+            for key in {normalize_url(u) for u in entry.urls}:
+                if key in seen:
+                    problems.append(Problem(f"Duplicate link: `{entry.name}` points to the same page as `{seen[key]}`.", entry.line + 1))
+                else:
+                    seen[key] = entry.name
             if entry.name.casefold() in names:
                 problems.append(Problem(f"`{entry.name}` appears twice in `{section.title}`.", entry.line + 1))
             names.add(entry.name.casefold())
@@ -243,13 +271,13 @@ def new_errors(before_text, after_text, max_length, meta_sections):
     return fresh
 
 
-def insert_entry(text, section_title, name, url, description, meta_sections):
+def insert_entry(text, section_title, name, url, description, meta_sections, source=""):
     readme = Readme(text, meta_sections)
     section = readme.section(section_title)
     if section is None or section.title in readme.meta:
         raise ValueError(f"Unknown section: {section_title}")
     lines = list(readme.lines)
-    new_line = format_entry(name, url, description)
+    new_line = format_entry(name, url, description, source)
     if section.entries:
         key = sort_key(name)
         after = next((e for e in section.entries if sort_key(e.name) > key), None)
@@ -299,7 +327,7 @@ def find_entries(readme, query):
         query = m["url"]
     if is_valid_url(query):
         key = normalize_url(query)
-        return [e for e in readme.entries() if normalize_url(e.url) == key]
+        return [e for e in readme.entries() if key in map(normalize_url, e.urls)]
     name = sanitize_name(query).casefold()
     return [e for e in readme.entries() if e.name.casefold() == name] if name else []
 
@@ -335,5 +363,5 @@ def apply_entry_diff(text, removed, added, meta_sections):
     text = "\n".join(lines)
     for entry in added:
         if not Readme(text, meta_sections).find_url(entry.url):
-            text = insert_entry(text, entry.section, entry.name, entry.url, entry.description, meta_sections)
+            text = insert_entry(text, entry.section, entry.name, entry.url, entry.description, meta_sections, entry.source)
     return text, missing