From c9e7cf7b147140367dc26b3993395f7f703f7dbc Mon Sep 17 00:00:00 2001 From: Alex Dehnert Date: Fri, 17 Oct 2025 16:40:54 -0400 Subject: [PATCH 1/3] docs: Update and improve Sphinx/ReadTheDocs setup - Update to a newer ReadTheDocs build OS - Update to Sphinx 8.2 (or newer), which adds the `:collapsible:` option which is used by upcoming content changes - Build PDF and epub docs, mostly because it's easy - Bump the copyright date - Add a Sphinx extension for Django - Add the deps from INSTALLED_APPS to `docs/requirements.txt`, since Sphinx now tries to do Django setup - Similarly, create `local.py` if it doesn't exist already so that settings can be imported - Mock some more files that don't get processed properly - `squaresdb.urls` and `squaresdb.wsgi` both have some Django-related errors, and while those might be fixable, there's not much in those files to document, so skipping them seems easiest - Fix non-mock Sphinx warnings - Set `language` - Disable `html_static_path` since we don't use it --- .readthedocs.yaml | 14 +++++++------- docs/conf.py | 24 ++++++++++++++++++++---- docs/requirements.txt | 7 +++++++ setup.py | 2 +- squaresdb/membership/models.py | 3 +-- squaresdb/settings/saml_decode.py | 7 ++----- 6 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dd2aa46..6019014 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,7 @@ version: 2 # Set the OS, Python version and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3.12" # You can also specify other tool versions: @@ -23,13 +23,13 @@ sphinx: # fail_on_warning: true # Optionally build your docs in additional formats such as PDF and ePub -# formats: -# - pdf -# - epub +formats: + - pdf + - epub # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/conf.py b/docs/conf.py index b7df39c..2c9dd32 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,11 +14,19 @@ import sys import os +import os.path # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# Include directory that contains SquaresDB so that spinxcontrib_django can +# import our settings module +_djsettings = '../squaresdb/settings/local.py' +if not os.path.exists(_djsettings): + os.symlink('local.dev-template.py', _djsettings) +else: + print("local.py already exists") +sys.path.insert(0, os.path.join(os.path.abspath('.'), '..')) # -- General configuration ------------------------------------------------ @@ -30,9 +38,12 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + # This appears to hang locally but run on ReadTheDocs 'sphinx.ext.autosummary', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', + # https://github.com/sphinx-doc/sphinxcontrib-django + 'sphinxcontrib_django', ] # Add any paths that contain templates here, relative to this directory. @@ -51,7 +62,7 @@ # General information about the project. project = u'Tech Squares Member DB' -copyright = u'2018, Alex Dehnert, Tech Squares' +copyright = u'2025, Alex Dehnert, Tech Squares' author = u'Alex Dehnert, Tech Squares' # The version info for the project you're documenting, acts as replacement for @@ -68,7 +79,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -112,6 +123,10 @@ # Somehow, commenting out django leads to lots of error messages, # but not commenting it out leads to hangs?? #"django", + #"django.db.models", # this also causes hangs + "manage", + "squaresdb.urls", + "squaresdb.wsgi", "reversion", "social_core", "social_django", @@ -131,6 +146,7 @@ #settings.configure(INSTALLED_APPS=INSTALLED_APPS) #import django #django.setup() +django_settings = "squaresdb.settings" # #autosummary_generate = True @@ -175,7 +191,7 @@ def setup(app): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..6c9489a --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +sphinxcontrib-django +# sphinxcontrib-django tries to set up Django, so we need everything +# in INSTALLED_APPS +django-bootstrap-static>=5,<6 +django-select2 +django-reversion +social-auth-app-django diff --git a/setup.py b/setup.py index 4430b03..c90278d 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'pylint', 'pylint-django', # lint 'mypy', 'django-stubs', # type checking ], - 'doc': ['sphinx'], + 'doc': ['sphinx>8.2', 'sphinxcontrib-django'], }, author = "Tech Squares webapp team", diff --git a/squaresdb/membership/models.py b/squaresdb/membership/models.py index ff44b40..5f718c8 100644 --- a/squaresdb/membership/models.py +++ b/squaresdb/membership/models.py @@ -189,8 +189,7 @@ def get_link(cls, secret, request_ip): Returns a tuple, with elements: (1) Validity: True (valid) or False (invalid) - (2) Object: PersonAuthLink object if the secret was found and None - otherwise + (2) Object: PersonAuthLink object if the secret was found and None otherwise If validity is False but object is non-None, one should next typically call send_new_auth_link to generate a replacement. diff --git a/squaresdb/settings/saml_decode.py b/squaresdb/settings/saml_decode.py index 3d369ad..a0ce415 100755 --- a/squaresdb/settings/saml_decode.py +++ b/squaresdb/settings/saml_decode.py @@ -10,11 +10,8 @@ def decode_authn_request(authn_request): """ AuthnRequest is always deflated, base64 encoded and url-escaped. - :Parameters: - -`authn_request`: AuthnRequest - - :Return: - The decoded AuthnRequest if successful else empty string. + :parameter authn_request: AuthnRequest + :return: The decoded AuthnRequest if successful else empty string. """ decoded = '' a = urllib.unquote(authn_request) From 3d904e8610aacc2f4356e37e882d97091ea1e936 Mon Sep 17 00:00:00 2001 From: Alex Dehnert Date: Fri, 17 Oct 2025 16:59:30 -0400 Subject: [PATCH 2/3] utils: Make `install.py` more portable We had a couple people trying to set SquaresDB up on Windows, and this fixes a bunch of the issues they were running into. - Use `#!/usr/bin/env python`, which matches most of our other shebang lines, instead of `#!/usr/bin/env python3`, which would require people to have both `python` and `python3` to run all the scripts (people generally do, so it's mostly fine, but it seems a little silly) - Use `getpass.getuser()`[1] to find the current user, instead of checking `LOGNAME` and the `pwd` module -- `getuser` implements a very similar "check some env vars, then optionally `pwd`" algorithm but with more env vars, more concisely, and more portably (`pwd` is only available on Linux, not Windows, while `getpass` is available on almost all platforms) - Explicitly prefix `python` when running `manage.py`, instead of depending on the shebang line -- AFAICT shebang lines don't work on Windows [1] https://docs.python.org/3/library/getpass.html#getpass.getuser --- squaresdb/utils/install.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/squaresdb/utils/install.py b/squaresdb/utils/install.py index 9e1f969..ad8ae7a 100755 --- a/squaresdb/utils/install.py +++ b/squaresdb/utils/install.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import argparse -import pwd +import getpass import os import os.path import random @@ -16,8 +16,7 @@ def parse_args(): help="Report planned changes but don't make them") parser.add_argument('--scripts', action='store_true', help="Set up web_scripts index.fcgi and media symlinks") - user = os.environ.get("LOGNAME", pwd.getpwuid(os.getuid())[0]) - parser.add_argument('--locker', type=str, default=user, + parser.add_argument('--locker', type=str, default=getpass.getuser(), help="Locker name to use on scripts") parser.add_argument('--instance', type=str, help="Name of install instance to use. Used to name " @@ -169,8 +168,8 @@ def call(cmd_args): if args.scripts: call(['/mit/scripts/sql/bin/create-database', args.instance]) manage = os.path.join(BASE_DIR, "manage.py") - call([manage, "migrate"]) - call([manage, "createinitialrevisions", + call(["python", manage, "migrate"]) + call(["python", manage, "createinitialrevisions", "--comment=Initial revision (in setup script)", "membership"]) From 6d7acd43284224a4b420756c90a2f348d6ce22cb Mon Sep 17 00:00:00 2001 From: Alex Dehnert Date: Fri, 17 Oct 2025 17:08:48 -0400 Subject: [PATCH 3/3] docs: Update install docs - Add some guidance on how to install on Windows - More clearly call out what needs to be done by hand and what's context or handled automatically - Hide the scripts.mit.edu install docs since they don't work - Note that this uses the `:collapsible:` option, which is new in Sphinx 8.2, which we recently upgraded to - Dropdowns from sphinx-design (https://sphinx-design.readthedocs.io/en/latest/dropdowns.html) would also work, but requiring new Sphinx seems preferable to requiring a third-party extension (though I think the built-in feature looks worse) - Format the warning text more clearly - More clearly state that SSO isn't needed for a dev server --- docs/installing.rst | 66 +++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/docs/installing.rst b/docs/installing.rst index 362733a..b1d93aa 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -4,14 +4,17 @@ Installing SquaresDB Dependencies ------------ -Most SquaresDB dependencies will be installed automatically by ``pip``, but some need to be installed already: +Make sure you have Python 3, ``virtualenv``, and ``pip`` installed. These are used for Python package management, and will be able to install most of the other software for you. + +On Linux, in order for ``pip`` to build packages, the following need to be installed separately: - python (3), with dev headers (Debian: ``apt install python3-dev``) - xmlsec1 (Debian: ``apt install libxmlsec1-dev``; Fedora: ``xmlsec1-devel``) - pkg-config or equivalent (Debian: ``apt install pkgconf``) -Here's a list of some of the key pip-installable dependencies, and what they're -used for: +On Windows, these don't seem to be required (presumably ``pip`` pulls in pre-built wheels for the relevant packages). + +SquaresDB depends on a number of additional packages. Installing them manually **is not required** - ``pip`` will install them for you. Some of the key dependencies (along with what they're used for) are: - ``django``: web framework (`Django docs`_) - ``django-reversion``: version controlling objects in the DB (`reversion @@ -40,39 +43,47 @@ able to get it running with:: pip install -e git+https://github.com/tech-squares/squaresdb.git#egg=squaresdb cd $VENV/src/squaresdb/squaresdb/ pip install -e ..[dev] - utils/install.py --email whatever + utils/install.py --email your.email@example.com + +Note that you do **not** need to clone the source repo - the ``pip install -e`` will clone it for you (as well as installing dependencies like Django). See also https://diswww.mit.edu/pergamon/squares-webapps/21 (requires MIT certs). -Installing on Scripts ---------------------- +Installing on scripts.mit.edu +----------------------------- -.. warning:: The DB no longer runs on Fedora 20 scripts.mit.edu, which as Feb 2020 is the default. It used to run under the Fedora 30 pool (Python 3.7) with some effort, and setup instructions are documented below, but as of probably March 2022 it probably requires at least Python 3.8. +.. admonition:: Legacy instructions + :collapsible: closed -Much the same instructions should work. There's some tweaks -- the summary is:: + .. warning:: + SquaresDB uses Django 5.1, which requires at least Python 3.10, which is not available on scripts.mit.edu. Fedora 20 scripts.mit.edu (the default as of 2025) has Python 3.3, and Fedora 30 has Python 3.7. - VENV=venv-name - virtualenv $VENV - . $VENV/bin/activate - ln -s /usr/lib64/python3.7/site-packages/xmlsec.cpython-37m-x86_64-linux-gnu.so /usr/lib64/python3.7/site-packages/xmlsec-1.3.3-py3.7.egg-info . - pip install -e git+https://github.com/tech-squares/squaresdb.git@main#egg=squaresdb[scripts] - cd $VENV/src/squaresdb/squaresdb/ - utils/install.py --email whatever --scripts + The last working directions, for Fedora 30, are documented below, but **will no longer work** because the DB now uses a newer Django. Dev (and deployment) need to be done locally or on some other non-scripts.mit.edu platform now. + + The install process on scripts.mit.edu is similar to a regular Linux machine, but with some tweaks. The summary is:: -.. note:: The extension ``xmlsec`` (for SAML support, including Touchstone) requires headers that aren't installed to compile, so we link it into the virtualenv. We used to recommend passing ``--system-site-packages``, but the system Django is too old, so that doesn't work. (I think we might be able to install a newer one, but ``django-babel`` conflicts with newer Django, so it fails.) Sadly, this makes it very slow, because of running everything out of AFS. + VENV=venv-name + virtualenv $VENV + . $VENV/bin/activate + ln -s /usr/lib64/python3.7/site-packages/xmlsec.cpython-37m-x86_64-linux-gnu.so /usr/lib64/python3.7/site-packages/xmlsec-1.3.3-py3.7.egg-info . + pip install -e git+https://github.com/tech-squares/squaresdb.git@main#egg=squaresdb[scripts] + cd $VENV/src/squaresdb/squaresdb/ + utils/install.py --email whatever --scripts -The ``--scripts`` option will make the installer do various scripts-specific -things: + .. note:: The extension ``xmlsec`` (for SAML support, including Touchstone) requires headers that aren't installed to compile, so we link it into the virtualenv. We used to recommend passing ``--system-site-packages``, but the system Django is too old, so that doesn't work. (I think we might be able to install a newer one, but ``django-babel`` conflicts with newer Django, so it fails.) Sadly, this makes it very slow, because of running everything out of AFS. -- configure ``DATABASES`` to use sql.mit.edu (not sqlite), and creates the database -- configure ``ALLOWED_HOSTS`` to include tech-squares.mit.edu, - locker.scripts.mit.edu, and s-a.mit.edu (a specific scripts host, useful for - using ``manage.py runserver``) -- configure admin media to use shared scripts copies as applicable -- configure various other settings -- create the directory in ``web_scripts``, with appropriate FastCGI and ``.htaccess`` config + The ``--scripts`` option will make the installer do various scripts-specific + things: -.. warning:: sql.mit.edu runs MySQL 5.1, and Django `requires `_ 5.6+, so you'll actually need to switch back (currently manually) to sqlite. + - configure ``DATABASES`` to use sql.mit.edu (not sqlite), and creates the database + - configure ``ALLOWED_HOSTS`` to include tech-squares.mit.edu, + locker.scripts.mit.edu, and s-a.mit.edu (a specific scripts host, useful for + using ``manage.py runserver``) + - configure admin media to use shared scripts copies as applicable + - configure various other settings + - create the directory in ``web_scripts``, with appropriate FastCGI and ``.htaccess`` config + + .. warning:: sql.mit.edu runs MySQL 5.1, and Django `requires `_ 5.6+ (as of 3.0; MySQL 8.0 as of Django 5.2), so you'll actually need to switch back (currently manually) to sqlite. Configuring Google and MIT auth @@ -81,6 +92,9 @@ Configuring Google and MIT auth SquaresDB supports using python-social-auth_ to authenticate using Google and MIT Shibboleth. +.. note:: + A typical development install doesn't need this, and can exclusively use Django's built-in auth. This whole section can be safely ignored if not setting up a prod DB or testing auth. + .. _python-social-auth: https://python-social-auth.readthedocs.io/en/latest/index.html Google auth