remove upstart, move to systemd

r2/r2/lib/translation.py

@@ -33,11 +33,11 @@
 from pylons.i18n.translation import LanguageError, NullTranslations, translation
 
 try:
-    import reddit_i18n
+    import tippr_i18n
 except ImportError:
     I18N_PATH = ''
 else:
-    I18N_PATH = os.path.dirname(reddit_i18n.__file__)
+    I18N_PATH = os.path.dirname(tippr_i18n.__file__)
 
 # Different from the default lang (as defined in the ini file)
 # Source language is what is in the source code

scripts/manage-consumers

@@ -1,27 +1,118 @@
 #!/bin/bash
+# Manage tippr queue consumers via systemd
+#
+# Usage: manage-consumers [start|stop|restart|status] [consumer_name]
+#
+# This script reads consumer configuration from $TIPPR_CONSUMER_CONFIG
+# and manages the appropriate number of systemd service instances.
 
-command=${UPSTART_JOB#tippr-consumers-}
-for consumerpath in $TIPPR_CONSUMER_CONFIG/*; do
-    consumer=$(basename $consumerpath)
+COMMAND="${1:-start}"
+TARGET="${2:-all}"
 
-    # allow targeting which consumer the event is meant for (defaulting to 'all')
-    if [ ! -z "$TARGET" -a "x$TARGET" != "xall" -a "x$TARGET" != "x$consumer" ]; then
+# Source the tippr environment
+if [ -f /etc/default/tippr ]; then
+    . /etc/default/tippr
+fi
+
+CONSUMER_CONFIG="${TIPPR_CONSUMER_CONFIG:-/home/tippr/consumer-count.d}"
+
+if [ ! -d "$CONSUMER_CONFIG" ]; then
+    echo "Consumer config directory not found: $CONSUMER_CONFIG" >&2
+    exit 0  # Exit cleanly if no consumers configured
+fi
+
+for consumerpath in "$CONSUMER_CONFIG"/*; do
+    # Skip if no files match the glob
+    [ -e "$consumerpath" ] || continue
+
+    # Only process regular files (skip directories, symlinks to directories, etc.)
+    [ -f "$consumerpath" ] || continue
+
+    consumer=$(basename "$consumerpath")
+
+    # Skip hidden files and backup files
+    case "$consumer" in
+        .*|*~|*.bak|*.orig)
+            continue
+            ;;
+    esac
+
+    # Allow targeting a specific consumer
+    if [ "$TARGET" != "all" ] && [ "$TARGET" != "$consumer" ]; then
         continue
     fi
 
-    if [ -d $consumerpath ]; then
-        types=$consumerpath/*
-    else
-        types=$consumerpath
+    # Skip executable files
+    if [ -x "$consumerpath" ]; then
+        continue
+    fi
+
+    # Skip files containing NUL bytes or binary content (avoid ELF/binary inputs)
+    if grep -q $'\x00' "$consumerpath" 2>/dev/null; then
+        echo "Skipping binary file: $consumerpath" >&2
+        continue
     fi
 
-    for typepath in $types; do
-        instance_count=$(cat $typepath)
-        type_=$(basename $typepath)
+    # Read instance count - read only first line, strip control characters, take first word only
+    raw=$(head -n1 "$consumerpath" 2>/dev/null | tr -d '\000-\037\177' | awk '{print $1}')
+
+    # Validate it's a number and within reasonable range (1-100)
+    if ! [[ "$raw" =~ ^[0-9]+$ ]] || [ -z "$raw" ]; then
+        echo "Skipping $consumer (invalid count in $consumerpath)" >&2
+        continue
+    fi
+
+    instance_count="$raw"
+
+    # Additional safety check: ensure instance_count is actually numeric before using in seq
+    if ! [ "$instance_count" -eq "$instance_count" ] 2>/dev/null; then
+        echo "Skipping $consumer (non-numeric count)" >&2
+        continue
+    fi
+
+    # Skip if no instances configured or if count is unreasonably high
+    if [ "$instance_count" -eq 0 ]; then
+        echo "Skipping $consumer (0 instances configured)"
+        continue
+    fi
+
+    if [ "$instance_count" -gt 100 ]; then
+        echo "Skipping $consumer (instance count $instance_count exceeds maximum of 100)" >&2
+        continue
+    fi
 
-        for i in $(seq 1 "$instance_count"); do
-            "/sbin/$command" "tippr-consumer-$consumer" "type=$type_" "x=$i"
-        done
+    # Manage each instance
+    for i in $(seq 1 "$instance_count"); do
+        unit="tippr-consumer@${consumer}${i}.service"
+        case "$COMMAND" in
+            start)
+                echo "Starting $unit"
+                systemctl start "$unit" --plain 2>&1 || true
+                ;;
+            stop)
+                echo "Stopping $unit"
+                systemctl stop "$unit" --plain 2>&1 || true
+                ;;
+            restart)
+                echo "Restarting $unit"
+                systemctl restart "$unit" --plain 2>&1 || true
+                ;;
+            status)
+                systemctl status "$unit" --plain --no-pager 2>&1 || true
+                ;;
+            enable)
+                echo "Enabling $unit"
+                systemctl enable "$unit" --plain 2>&1 || true
+                ;;
+            disable)
+                echo "Disabling $unit"
+                systemctl disable "$unit" --plain 2>&1 || true
+                ;;
+            *)
+                echo "Unknown command: $COMMAND" >&2
+                echo "Usage: $0 [start|stop|restart|status|enable|disable] [consumer_name]" >&2
+                exit 1
+                ;;
+        esac
     done
 done
-

scripts/run-consumer

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Run a tippr queue consumer
+# Usage: run-consumer <consumer_name>
+#
+# This script is called by systemd to run queue consumers.
+# The consumer name determines which queue processing function to call.
+
+set -e
+
+CONSUMER="$1"
+if [ -z "$CONSUMER" ]; then
+    echo "Usage: $0 <consumer_name>" >&2
+    exit 1
+fi
+
+# Source the tippr environment
+if [ -f /etc/default/tippr ]; then
+    . /etc/default/tippr
+fi
+
+# Defaults assume TIPPR_USER=tippr; these are overridden by /etc/default/tippr
+TIPPR_INI=${TIPPR_INI:-/home/tippr/src/tippr/r2/run.ini}
+VENV=${VIRTUAL_ENV:-/home/tippr/venv}
+
+cd "${TIPPR_ROOT:-/home/tippr/src/tippr/r2}"
+
+# Map consumer names to their Python commands
+case "$CONSUMER" in
+    vote_link_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.voting import consume_link_vote_queue; consume_link_vote_queue()"
+        ;;
+    vote_comment_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.voting import consume_comment_vote_queue; consume_comment_vote_queue()"
+        ;;
+    commentstree_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_commentstree_q; run_commentstree_q()"
+        ;;
+    newcomments_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_newcomments_q; run_newcomments_q()"
+        ;;
+    scraper_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.scraper import run_scraper_q; run_scraper_q()"
+        ;;
+    markread_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_markread_q; run_markread_q()"
+        ;;
+    del_account_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_del_account_q; run_del_account_q()"
+        ;;
+    author_query_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_author_query_q; run_author_query_q()"
+        ;;
+    subreddit_query_q*|subtippr_query_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_subreddit_query_q; run_subreddit_query_q()"
+        ;;
+    domain_query_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.db.queries import run_domain_query_q; run_domain_query_q()"
+        ;;
+    butler_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.butler import run_butler_q; run_butler_q()"
+        ;;
+    automoderator_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.automoderator import run_automoderator_q; run_automoderator_q()"
+        ;;
+    search_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.search import run_search_q; run_search_q()"
+        ;;
+    event_collector_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.eventcollector import run_event_collector_q; run_event_collector_q()"
+        ;;
+    modmail_email_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.emailer import run_modmail_email_q; run_modmail_email_q()"
+        ;;
+    sitemaps_q*)
+        exec "$VENV/bin/paster" run --proctitle "$CONSUMER" "$TIPPR_INI" -c \
+            "from r2.lib.sitemaps import run_sitemaps_q; run_sitemaps_q()"
+        ;;
+    *)
+        echo "Unknown consumer: $CONSUMER" >&2
+        exit 1
+        ;;
+esac

scripts/wrap-job

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 """
 Wrap a command, setuid/setgid, change directory to the tippr
 code, and send output to syslog.
@@ -34,36 +34,38 @@ import subprocess
 CONSUMER_PREFIX = "tippr-consumer-"
 
 
-# drop permissions
+# drop permissions (only if running as root)
 user = os.environ.get("TIPPR_USER", "tippr")
 group = os.environ.get("TIPPR_GROUP", user)
-uid = pwd.getpwnam(user).pw_uid
-gid = grp.getgrnam(group).gr_gid
-os.setgroups([])
-os.setgid(gid)
-os.setuid(uid)
+
+if os.getuid() == 0:
+    uid = pwd.getpwnam(user).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+    os.setgroups([])
+    os.setgid(gid)
+    os.setuid(uid)
 
 # change directory to the tippr code root
-root = os.environ.get("TIPPR_ROOT", "/opt/tippr/lib/public/r2")
+root = os.environ.get("TIPPR_ROOT", "/home/tippr/src/tippr/r2")
 os.chdir(root)
 
-# configure syslog
-job_name = os.environ.get("UPSTART_JOB", "-".join(sys.argv[1:]))
+# configure syslog - use SYSTEMD_UNIT if available, fall back to command args
+job_name = os.environ.get("SYSTEMD_UNIT", os.environ.get("INVOCATION_ID", "-".join(sys.argv[1:])))
 if job_name.startswith(CONSUMER_PREFIX):
     # consumers are a bit different from crons, while crons want an
     # ident of tippr-job-JOBNAME, we want consumers to have an ident
     # of CONSUMERNAME_INSTANCE
-    job_name = (job_name[len(CONSUMER_PREFIX):] +
-                "_" +
-                os.environ.get("UPSTART_INSTANCE", ""))
+    job_name = job_name[len(CONSUMER_PREFIX):]
+
 facility = getattr(syslog, "LOG_" + os.environ.get("TIPPR_LOG_FACILITY", "CRON"))
 syslog.openlog(ident=job_name, facility=facility)
 
 # run the wrapped command
 child = subprocess.Popen(sys.argv[1:],
                          stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT,
-                         bufsize=1)
+                         bufsize=1,
+                         text=True)
 
 # write out to syslog
 while True:
@@ -74,7 +76,7 @@ while True:
 
     line = line.rstrip('\n')
     syslog.syslog(syslog.LOG_NOTICE, line)
-    print line
+    print(line)
 
 # our success depends on our child's success
 child.wait()