From 31407d50c14816d3aeb2302d27731245d3b84976 Mon Sep 17 00:00:00 2001 From: Krzysztof Lewczuk Date: Sat, 29 Aug 2026 00:08:46 +0200 Subject: [PATCH 1/2] Add galaxy-web: a browsable/searchable web UI for galaxy archives The existing web server (web.cpp) only supports looking up a single message by Message-ID, with no way to browse groups or threads. This adds a new tool that serves a proper local browsing UI on top of a galaxy: a newsgroup listing, paginated per-group thread lists, a threaded message view, message permalinks, and full-text search (within one group or across the whole galaxy) using each archive's already-embedded lexicon, so no extra indexing step is required. Message rendering (header/quote/signature styling) reuses the same MessageLines-based approach as web.cpp. News: links in quoted text now resolve through a /msgid/ route that looks the message up via the galaxy and redirects to its proper group+message URL. --- CMakeLists.txt | 4 + README.md | 3 +- galaxy-web/config.ini | 6 + galaxy-web/galaxy-web.cpp | 689 ++++++++++++++++++++++++++++++++++++++ man/uat-galaxy-web.1 | 38 +++ man/uat.1 | 1 + 6 files changed, 740 insertions(+), 1 deletion(-) create mode 100644 galaxy-web/config.ini create mode 100644 galaxy-web/galaxy-web.cpp create mode 100644 man/uat-galaxy-web.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cf4fc3..49a194d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,9 @@ target_link_libraries(filter-spam PRIVATE terminator common zstd lz4) add_executable(galaxy-util galaxy-util/galaxy-util.cpp) target_link_libraries(galaxy-util PRIVATE common zstd libuat Tracy::TracyClient) +add_executable(galaxy-web galaxy-web/galaxy-web.cpp) +target_link_libraries(galaxy-web PRIVATE mongoose ini common libuat zstd) + add_executable(google-groups google-groups/google-groups.cpp) target_link_libraries(google-groups PRIVATE curl common ssl) @@ -361,6 +364,7 @@ install(FILES man/uat-filter-newsgroups.1 man/uat-filter-spam.1 man/uat-galaxy-util.1 + man/uat-galaxy-web.1 man/uat-google-groups.1 man/uat-import-source-maildir.1 man/uat-import-source-maildir-7z.1 diff --git a/README.md b/README.md index 4f0edc2..82da33f 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ These tools provide access to archive data: ### End-user Utilities - tbrowser --- Curses-based text mode browser of archives. +- web --- Web server for looking up a single message by Message-ID. +- galaxy-web --- Local web server for browsing and searching a whole galaxy: newsgroup list, paginated thread lists, threaded message view, and full-text search within a group or across all groups. ## Future work ideas @@ -123,7 +125,6 @@ Here are some viable ideas that I'm not really planning to do any time soon, but - Implement messages extractor, for example in mbox format. Would need to properly encode headers and add content encoding information (UTF-8 everywhere). - Implement a read-only NNTP server. Would need to properly encode headers and add content encoding information. 7-bit cleanness probably would be nice, so also encode as quoted-printable. Some headers may need to be rewritten (eg. "Lines", which most probably won't be true, due to MIME processing). Message sorting by date may be necessary to put some sense into internal message numbers, which currently have no meaning at all. -- Implement pan-group search mechanism. ## Workflow diff --git a/galaxy-web/config.ini b/galaxy-web/config.ini new file mode 100644 index 0000000..af80c64 --- /dev/null +++ b/galaxy-web/config.ini @@ -0,0 +1,6 @@ +[server] +bind = 127.0.0.1 +port = 8120 + +[galaxy] +path = news/galaxy diff --git a/galaxy-web/galaxy-web.cpp b/galaxy-web/galaxy-web.cpp new file mode 100644 index 0000000..4c7c808 --- /dev/null +++ b/galaxy-web/galaxy-web.cpp @@ -0,0 +1,689 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../contrib/ini/ini.h" +#include "../contrib/mongoose/mongoose.h" +#ifdef GetMessage +# undef GetMessage +#endif +#include "../libuat/Archive.hpp" +#include "../libuat/Galaxy.hpp" +#include "../libuat/SearchEngine.hpp" +#include "../common/KillRe.hpp" +#include "../common/MessageLines.hpp" +#include "../common/MessageLogic.hpp" +#include "../common/UTF8.hpp" + +static void TryIni( const char*& value, ini_t* config, const char* section, const char* key ) +{ + auto tmp = ini_get( config, section, key ); + if( tmp ) value = tmp; +} + +static ExpandingBuffer eb; +static MessageLines ml; +static KillRe killre; +static std::unique_ptr galaxy; +static std::map groupIndex; + +static const size_t ThreadListPageSize = 50; +static const size_t SearchPerGroupCap = 20; +static const size_t SearchGlobalCap = 200; + +struct ResultRow +{ + std::string group; + uint32_t idx; + float rank; +}; + +static const char* CSS = R"CSS( +body { background:#111111; color:#cccccc; font-family: 'Roboto Mono', Menlo, Consolas, monospace; margin:0; padding:0; } +a { color:#3a96dd; text-decoration:none; } +a:hover { text-decoration:underline; } +header { background:#1b1b1b; padding:0.8em 1.5em; border-bottom:1px solid #333333; display:flex; justify-content:space-between; align-items:center; flex-wrap:wrap; gap:0.5em; } +header a.brand { color:#ffffff; font-weight:bold; font-size:1.1em; } +form.search input[type=text] { background:#222222; border:1px solid #444444; color:#cccccc; padding:0.3em 0.6em; font-family: inherit; } +form.search input[type=submit] { background:#2a2a2a; border:1px solid #444444; color:#cccccc; padding:0.3em 0.8em; cursor:pointer; font-family: inherit; } +main { max-width:1024px; margin: 1.5em auto; padding: 0 1em 3em 1em; } +h1 { font-size:1.4em; } +table { width:100%; border-collapse:collapse; } +th, td { text-align:left; padding:0.4em 0.6em; border-bottom:1px solid #222222; vertical-align:top; } +th { color:#888888; font-weight:normal; font-size:0.8em; text-transform:uppercase; } +tr:hover { background:#181818; } +.count { color:#888888; font-size:0.85em; } +.pager { margin-top:1em; display:flex; gap:1.5em; align-items:center; } +.thread-msg { background:#1a1a1a; border:1px solid #262626; border-radius:4px; margin: 0.6em 0; padding:0.8em; } +.msg-hdr { color:#888888; font-size:0.85em; margin-bottom:0.5em; } +.msg-hdr .from { color:#13a10e; } +.msg-hdr .subj { color:#3a96dd; } +.msg-body { white-space:pre-wrap; font-size:0.92em; } +.hdrName { color:#13a10e } +.hdrBody { color:#3a96dd } +.q1 { color:#c50f1f } +.q2 { color:#881798 } +.q3 { color:#3b78ff } +.q4 { color:#13a10e } +.q5 { color:#c19c00 } +.signature { color:#767676 } +.hide { display:none } +.rank { color:#666666; font-size:0.8em; } +)CSS"; + +static std::string Encode( const char* txt, const char* end ) +{ + std::string enc; + while( txt < end ) + { + if( *txt == '&' ) + { + enc += "&"; + txt++; + } + else if( *txt == '<' ) + { + enc += "<"; + txt++; + } + else if( *txt == '>' ) + { + enc += ">"; + txt++; + } + else if( *txt == '"' ) + { + enc += """; + txt++; + } + else + { + const auto len = codepointlen( *txt ); + enc.append( txt, txt+len ); + txt += len; + } + } + return enc; +} + +static std::string Encode( const char* txt ) +{ + if( !txt ) return std::string(); + return Encode( txt, txt + strlen( txt ) ); +} + +static std::string Encode( const std::string& txt ) +{ + return Encode( txt.c_str(), txt.c_str() + txt.size() ); +} + +static int HexVal( char c ) +{ + if( c >= '0' && c <= '9' ) return c - '0'; + c = (char)tolower( (unsigned char)c ); + if( c >= 'a' && c <= 'f' ) return c - 'a' + 10; + return -1; +} + +static std::string UriDecode( const char* txt, size_t sz ) +{ + std::string str; + str.reserve( sz ); + size_t i = 0; + while( i < sz ) + { + char c = txt[i]; + if( c == '%' && i+2 < sz ) + { + int h1 = HexVal( txt[i+1] ); + int h2 = HexVal( txt[i+2] ); + if( h1 >= 0 && h2 >= 0 ) + { + str.push_back( (char)( ( h1 << 4 ) | h2 ) ); + i += 3; + continue; + } + } + if( c == '+' ) str.push_back( ' ' ); + else str.push_back( c ); + i++; + } + return str; +} + +static std::string UriPathEncode( const std::string& s ) +{ + static const char* hex = "0123456789ABCDEF"; + std::string out; + out.reserve( s.size() * 3 ); + for( unsigned char c : s ) + { + if( isalnum( c ) || c == '-' || c == '_' || c == '.' || c == '~' ) + { + out.push_back( (char)c ); + } + else + { + out.push_back( '%' ); + out.push_back( hex[c >> 4] ); + out.push_back( hex[c & 0xF] ); + } + } + return out; +} + +static std::vector SplitPath( const std::string& uri ) +{ + std::vector parts; + size_t i = uri.empty() ? 0 : 1; + while( i < uri.size() ) + { + auto j = uri.find( '/', i ); + if( j == std::string::npos ) j = uri.size(); + if( j > i ) parts.emplace_back( UriDecode( uri.c_str() + i, j - i ) ); + i = j + 1; + } + return parts; +} + +static std::string GetQueryVar( struct mg_str qs, const char* name ) +{ + char buf[4096]; + int n = mg_get_http_var( &qs, name, buf, sizeof( buf ) ); + if( n > 0 ) return std::string( buf, n ); + return std::string(); +} + +static std::string FormatDate( uint32_t unixTime ) +{ + time_t t = (time_t)unixTime; + char buf[64]; + strftime( buf, sizeof( buf ), "%Y-%m-%d %H:%M", localtime( &t ) ); + return buf; +} + +static std::string FormatRank( float r ) +{ + char buf[32]; + snprintf( buf, sizeof( buf ), "%.2f", r ); + return buf; +} + +static std::string PageHeader( const std::string& title ) +{ + std::string s = "\n\n\n\n"; + s += "\n"; + s += "\n"; + s += ""; + s += Encode( title.c_str() ); + s += " - Usenet Archive\n\n\n\n"; + s += "
Usenet Archive"; + s += "
"; + s += "
\n
\n"; + return s; +} + +static std::string PageFooter() +{ + return "\n
\n\n\n"; +} + +static std::string RenderMessage( const char* message ) +{ + ml.PrepareLines( message, false ); + std::string out; + auto& lines = ml.Lines(); + auto& parts = ml.Parts(); + for( auto& line : lines ) + { + const bool isHeader = line.parts > 0 && parts[line.idx].flags == MessageLines::L_HeaderName; + if( isHeader ) + { + out += ""; + } + for( int i=0; i"; + else if( part.flags == MessageLines::L_HeaderBody ) out += ""; + else if( part.flags == MessageLines::L_Quote0 ) noSpan = true; + else if( part.flags == MessageLines::L_Quote1 ) out += ""; + else if( part.flags == MessageLines::L_Quote2 ) out += ""; + else if( part.flags == MessageLines::L_Quote3 ) out += ""; + else if( part.flags == MessageLines::L_Quote4 ) out += ""; + else if( part.flags == MessageLines::L_Quote5 ) out += ""; + else if( part.flags == MessageLines::L_Signature ) out += ""; + else noSpan = true; + + const bool du = part.deco == MessageLines::D_Underline; + const bool di = part.deco == MessageLines::D_Italics; + const bool db = part.deco == MessageLines::D_Bold; + const bool dl = part.deco == MessageLines::D_Url; + if( du ) out += ""; + else if( di ) out += ""; + else if( db ) out += ""; + else if( dl ) + { + if( part.len > 5 && memcmp( message + part.offset, "news:", 5 ) == 0 ) + { + std::string mid( message + part.offset + 5, message + part.offset + part.len ); + out += ""; + } + else + { + out += ""; + } + } + + out += Encode( message + part.offset, message + part.offset + part.len ); + + if( du ) out += ""; + else if( di ) out += ""; + else if( db ) out += ""; + else if( dl ) out += ""; + + if( !noSpan ) out += ""; + } + out += "\n"; + if( isHeader ) out += ""; + } + ml.Reset(); + return out; +} + +static std::string RenderGroupList() +{ + std::string body = "

Newsgroups

"; + auto avail = galaxy->GetAvailableArchives(); + std::sort( avail.begin(), avail.end(), []( int a, int b ) { + return strcmp( galaxy->GetArchiveName( a ), galaxy->GetArchiveName( b ) ) < 0; + } ); + body += "

" + std::to_string( avail.size() ) + " groups

"; + body += ""; + for( int idx : avail ) + { + const char* name = galaxy->GetArchiveName( idx ); + const char* desc = galaxy->GetArchiveDescription( idx ); + body += ""; + } + body += "
NameDescriptionMessagesThreads
" + Encode( name ) + "" + Encode( desc ) + "" + + std::to_string( galaxy->NumberOfMessages( idx ) ) + "" + std::to_string( galaxy->NumberOfTopLevel( idx ) ) + "
"; + return body; +} + +static std::string RenderThreadList( const std::string& name, int archiveIdx, size_t page ) +{ + auto& archive = *galaxy->GetArchive( archiveIdx, false ); + auto topLevel = archive.GetTopLevel(); + size_t total = topLevel.size; + size_t totalPages = ( total + ThreadListPageSize - 1 ) / ThreadListPageSize; + if( totalPages == 0 ) totalPages = 1; + if( page >= totalPages ) page = totalPages - 1; + size_t start = page * ThreadListPageSize; + size_t end = std::min( total, start + ThreadListPageSize ); + + std::string body = "

" + Encode( name ) + "

"; + auto desc = archive.GetShortDescription(); + if( desc.second ) body += "

" + Encode( desc.first, desc.first + desc.second ) + "

"; + body += "

" + std::to_string( archive.NumberOfMessages() ) + " messages, " + std::to_string( total ) + " threads

"; + body += "
"; + + body += ""; + for( size_t i=start; i 0 ? cnt - 1 : 0; + body += ""; + } + body += "
SubjectFromDateReplies
" + Encode( killre.Kill( subj ) ) + "" + + Encode( from ) + "" + FormatDate( date ) + "" + std::to_string( replies ) + "
"; + + body += "
"; + if( page > 0 ) body += "« Prev"; + body += "Page " + std::to_string( page+1 ) + " of " + std::to_string( totalPages ) + ""; + if( page+1 < totalPages ) body += "Next »"; + body += "
"; + + return body; +} + +static void RenderThreadRec( Archive& archive, const std::string& name, uint32_t idx, int depth, std::string& body ) +{ + const char* msg = archive.GetMessage( idx, eb ); + if( msg ) + { + const char* subj = archive.GetSubject( idx ); + const char* from = archive.GetRealName( idx ); + uint32_t date = archive.GetDate( idx ); + body += "
"; + body += "
" + Encode( from ) + " — " + + Encode( killre.Kill( subj ) ) + " — " + FormatDate( date ) + " — #
"; + body += "
" + RenderMessage( msg ) + "
"; + body += "
"; + } + auto children = archive.GetChildren( idx ); + for( uint64_t i=0; iGetArchive( archiveIdx, false ); + uint32_t root = FindRoot( archive, anchorIdx ); + std::string body = "

« " + Encode( name ) + "

"; + RenderThreadRec( archive, name, root, 0, body ); + return body; +} + +static std::string RenderSingleMessage( const std::string& name, int archiveIdx, uint32_t idx ) +{ + auto& archive = *galaxy->GetArchive( archiveIdx, false ); + const char* msg = archive.GetMessage( idx, eb ); + if( !msg ) return std::string(); + + uint32_t root = FindRoot( archive, idx ); + std::string body = "

« " + Encode( name ) + " · View full thread

"; + + const char* subj = archive.GetSubject( idx ); + const char* from = archive.GetRealName( idx ); + uint32_t date = archive.GetDate( idx ); + body += "
"; + body += "
" + Encode( from ) + " — " + + Encode( killre.Kill( subj ) ) + " — " + FormatDate( date ) + "
"; + body += "
" + RenderMessage( msg ) + "
"; + body += "
"; + return body; +} + +static std::vector SearchArchive( Archive& archive, const std::string& groupName, const std::string& query, size_t cap ) +{ + std::vector out; + if( query.empty() ) return out; + SearchEngine se( archive ); + auto data = se.Search( query.c_str(), SearchEngine::SF_AdjacentWords ); + auto& results = data.results; + std::sort( results.begin(), results.end(), []( const SearchResult& a, const SearchResult& b ) { return a.rank > b.rank; } ); + size_t n = std::min( results.size(), cap ); + out.reserve( n ); + for( size_t i=0; i& rows, const std::string& query, const std::string& action, bool truncated ) +{ + std::string body = "

Search results

"; + body += "
"; + if( query.empty() ) return body; + if( rows.empty() ) + { + body += "

No results.

"; + return body; + } + body += "

" + std::to_string( rows.size() ) + ( truncated ? "+" : "" ) + " results

"; + body += ""; + for( auto& r : rows ) + { + auto it = groupIndex.find( r.group ); + if( it == groupIndex.end() ) continue; + auto& archive = *galaxy->GetArchive( it->second, false ); + const char* subj = archive.GetSubject( r.idx ); + const char* from = archive.GetRealName( r.idx ); + uint32_t date = archive.GetDate( r.idx ); + body += ""; + } + body += "
GroupSubjectFromDateRank
" + Encode( r.group ) + "" + Encode( killre.Kill( subj ) ) + "" + + Encode( from ) + "" + FormatDate( date ) + "" + FormatRank( r.rank ) + "
"; + return body; +} + +static void Handler( struct mg_connection* nc, int ev, void* data ) +{ + if( ev != MG_EV_HTTP_REQUEST ) return; + auto hm = (struct http_message*)data; + char remoteAddr[100]; + mg_sock_to_str( nc->sock, remoteAddr, sizeof(remoteAddr), MG_SOCK_STRINGIFY_REMOTE | MG_SOCK_STRINGIFY_IP | MG_SOCK_STRINGIFY_PORT ); + std::string uri( hm->uri.p, hm->uri.len ); + auto parts = SplitPath( uri ); + + int code = 200; + std::string title, body; + + if( parts.size() == 2 && parts[0] == "msgid" ) + { + const std::string& midStr = parts[1]; + bool found = false; + if( IsMsgId( midStr.c_str(), midStr.c_str() + midStr.size() ) ) + { + uint8_t packed[4096]; + galaxy->PackMsgId( midStr.c_str(), packed ); + auto gidx = galaxy->GetMessageIndex( packed ); + if( gidx >= 0 ) + { + auto groups = galaxy->GetGroups( gidx ); + for( uint64_t i=0; iIsArchiveAvailable( aidx ) ) + { + auto& archive = *galaxy->GetArchive( aidx, false ); + uint8_t archivePacked[4096]; + archive.RepackMsgId( packed, archivePacked, galaxy->GetCompress() ); + int localIdx = archive.GetMessageIndex( archivePacked ); + if( localIdx >= 0 ) + { + std::string loc = "/group/" + UriPathEncode( galaxy->GetArchiveName( aidx ) ) + "/msg/" + std::to_string( localIdx ); + mg_http_send_redirect( nc, 302, mg_mk_str( loc.c_str() ), mg_mk_str( "" ) ); + found = true; + } + } + } + } + } + if( !found ) mg_http_send_error( nc, 404, nullptr ); + printf( "%s \"%.*s %.*s\" %i\n", remoteAddr, (int)hm->method.len, hm->method.p, (int)hm->uri.len, hm->uri.p, found ? 302 : 404 ); + fflush( stdout ); + return; + } + + if( parts.empty() ) + { + title = "Groups"; + body = RenderGroupList(); + } + else if( parts[0] == "search" && parts.size() == 1 ) + { + std::string q = GetQueryVar( hm->query_string, "q" ); + title = "Search"; + if( q.empty() ) + { + body = RenderResults( {}, q, "/search", false ); + } + else + { + std::vector rows; + for( int idx : galaxy->GetAvailableArchives() ) + { + auto& archive = *galaxy->GetArchive( idx, false ); + auto part = SearchArchive( archive, galaxy->GetArchiveName( idx ), q, SearchPerGroupCap ); + rows.insert( rows.end(), part.begin(), part.end() ); + } + std::sort( rows.begin(), rows.end(), []( const ResultRow& a, const ResultRow& b ) { return a.rank > b.rank; } ); + bool truncated = rows.size() > SearchGlobalCap; + if( truncated ) rows.resize( SearchGlobalCap ); + body = RenderResults( rows, q, "/search", truncated ); + } + } + else if( parts[0] == "group" && parts.size() >= 2 ) + { + const std::string& name = parts[1]; + auto it = groupIndex.find( name ); + if( it == groupIndex.end() ) + { + code = 404; + } + else + { + int archiveIdx = it->second; + auto& archive = *galaxy->GetArchive( archiveIdx, false ); + + if( parts.size() == 2 ) + { + std::string pageStr = GetQueryVar( hm->query_string, "page" ); + size_t page = pageStr.empty() ? 0 : (size_t)strtoul( pageStr.c_str(), nullptr, 10 ); + title = name; + body = RenderThreadList( name, archiveIdx, page ); + } + else if( parts.size() == 3 && parts[2] == "search" ) + { + std::string q = GetQueryVar( hm->query_string, "q" ); + title = name + " search"; + auto rows = SearchArchive( archive, name, q, SearchGlobalCap ); + body = RenderResults( rows, q, "/group/" + Encode( name ) + "/search", false ); + } + else if( parts.size() == 4 && parts[2] == "thread" ) + { + uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), nullptr, 10 ); + if( idx >= archive.NumberOfMessages() ) code = 404; + else + { + title = name; + body = RenderThread( name, archiveIdx, idx ); + } + } + else if( parts.size() == 4 && parts[2] == "msg" ) + { + uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), nullptr, 10 ); + if( idx >= archive.NumberOfMessages() ) code = 404; + else + { + title = name; + body = RenderSingleMessage( name, archiveIdx, idx ); + if( body.empty() ) code = 404; + } + } + else + { + code = 404; + } + } + } + else + { + code = 404; + } + + int size = 0; + if( code == 404 ) + { + mg_http_send_error( nc, 404, nullptr ); + } + else + { + std::string page = PageHeader( title.empty() ? "Usenet Archive" : title ) + body + PageFooter(); + size = (int)page.size(); + mg_send_head( nc, 200, size, "Content-Type: text/html; charset=utf-8" ); + mg_printf( nc, "%.*s", size, page.c_str() ); + } + + printf( "%s \"%.*s %.*s\" %i %i\n", remoteAddr, (int)hm->method.len, hm->method.p, (int)hm->uri.len, hm->uri.p, code, size ); + fflush( stdout ); +} + +int main( int argc, char** argv ) +{ + if( argc != 2 ) + { + fprintf( stderr, "Usage: %s /path/to/config.ini\n", argv[0] ); + fflush( stderr ); + return 1; + } + + auto config = ini_load( argv[1] ); + if( !config ) + { + fprintf( stderr, "Cannot open config file %s!\n", argv[1] ); + fflush( stderr ); + return 2; + } + + const char* bind = "127.0.0.1"; + const char* port = "8120"; + const char* galaxyPath = "news/galaxy"; + + TryIni( bind, config, "server", "bind" ); + TryIni( port, config, "server", "port" ); + TryIni( galaxyPath, config, "galaxy", "path" ); + + galaxy.reset( Galaxy::Open( galaxyPath ) ); + if( !galaxy ) + { + fprintf( stderr, "Cannot access galaxy at %s!\n", galaxyPath ); + fflush( stderr ); + ini_free( config ); + return 3; + } + + for( int idx : galaxy->GetAvailableArchives() ) + { + groupIndex[galaxy->GetArchiveName( idx )] = idx; + } + fprintf( stderr, "Loaded %zu groups.\n", groupIndex.size() ); + + char address[1024]; + snprintf( address, 1024, "%s:%s", bind, port ); + + struct mg_mgr mgr; + mg_mgr_init( &mgr, nullptr ); + auto conn = mg_bind( &mgr, address, Handler ); + if( !conn ) + { + fprintf( stderr, "Cannot bind to %s!\n", address ); + fflush( stderr ); + ini_free( config ); + return 4; + } + mg_set_protocol_http_websocket( conn ); + + printf( "Listening on %s...\n", address ); + fflush( stdout ); + for(;;) + { + mg_mgr_poll( &mgr, std::numeric_limits::max() ); + } + + ini_free( config ); + return 0; +} diff --git a/man/uat-galaxy-web.1 b/man/uat-galaxy-web.1 new file mode 100644 index 0000000..b169fcd --- /dev/null +++ b/man/uat-galaxy-web.1 @@ -0,0 +1,38 @@ +.TH UAT 1 2026-08-29 UAT "Usenet Archive Toolkit" +.SH NAME +uat-galaxy-web \- web server for browsing and searching a galaxy +.SH SYNOPSIS +.I uat-galaxy-web + +.SH DESCRIPTION +This utility starts a local web server for browsing a galaxy (a collection +of archives, see +.BR \%uat-galaxy-util (1)) +as a set of linked pages: a newsgroup listing, per-group paginated thread +lists, full threaded message views, and single-message permalinks. +Full-text search is available both within a single group and across the +whole galaxy, using each archive's embedded lexicon +(see +.BR \%uat-lexicon (1)), +so no extra indexing step is required beyond having built the galaxy. +.SH OPTIONS +.TP +.BR config.ini +Path to configuration file. +.SH CONFIGURATION +Configuration file is in INI format with the following options: +.TP +.BR [server]\ bind +IP address to bind to (default: 127.0.0.1). +.TP +.BR [server]\ port +Port to listen on (default: 8120). +.TP +.BR [galaxy]\ path +Path to galaxy directory (default: news/galaxy). +.SH "SEE ALSO" +.ad l +.nh +.BR \%uat (1), +.BR \%uat-galaxy-util (1), +.BR \%uat-web (1) diff --git a/man/uat.1 b/man/uat.1 index deba276..69a27b6 100644 --- a/man/uat.1 +++ b/man/uat.1 @@ -99,6 +99,7 @@ License, or (at your option) any later version. .BR \%uat-filter-newsgroups (1), .BR \%uat-filter-spam (1), .BR \%uat-galaxy-util (1), +.BR \%uat-galaxy-web (1), .BR \%uat-google-groups (1), .BR \%uat-import-source-maildir (1), .BR \%uat-import-source-maildir-7z (1), From 3a19f368f46f5ab3705dcc7553444392503b58c4 Mon Sep 17 00:00:00 2001 From: Krzysztof Lewczuk Date: Sat, 29 Aug 2026 11:12:42 +0200 Subject: [PATCH 2/2] Address review of galaxy-web PR - Fix stack buffer overflow in PackMsgId (/msgid/): cap message-ID length before packing. - Install galaxy-web binary and register it in the uat dispatcher. - Revert unrelated README removal of "pan-group search" future-work item. - 404 on non-numeric /msg and /thread indices instead of silently resolving to message 0. - Stop UriDecode from turning '+' into space in path segments (broke msgids containing literal '+'). - Report accurate truncation on group-level search results. - Add cycle guard to FindRoot against corrupt connectivity data. - Restrict D_Url-decorated links to a scheme allowlist, blocking javascript: URLs from message bodies. - Cap each group's contribution to global search at SearchGlobalCap (was 20) so a group's real top matches aren't dropped before the merge, and normalize per-group ranks to [0,1] before merging so raw TF/IDF magnitudes from different-sized corpora aren't compared directly. - Grow GetQueryVar's buffer to MG_MAX_HTTP_REQUEST_SIZE so long query values can no longer be silently dropped to empty. --- CMakeLists.txt | 1 + README.md | 1 + galaxy-web/galaxy-web.cpp | 61 +++++++++++++++++++++++++++++---------- uat/uat.cpp | 1 + 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 49a194d..8d2896f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,6 +328,7 @@ install(TARGETS filter-newsgroups filter-spam galaxy-util + galaxy-web google-groups import-source-maildir import-source-maildir-7z diff --git a/README.md b/README.md index 82da33f..e72bdcb 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Here are some viable ideas that I'm not really planning to do any time soon, but - Implement messages extractor, for example in mbox format. Would need to properly encode headers and add content encoding information (UTF-8 everywhere). - Implement a read-only NNTP server. Would need to properly encode headers and add content encoding information. 7-bit cleanness probably would be nice, so also encode as quoted-printable. Some headers may need to be rewritten (eg. "Lines", which most probably won't be true, due to MIME processing). Message sorting by date may be necessary to put some sense into internal message numbers, which currently have no meaning at all. +- Implement pan-group search mechanism. ## Workflow diff --git a/galaxy-web/galaxy-web.cpp b/galaxy-web/galaxy-web.cpp index 4c7c808..951a3f2 100644 --- a/galaxy-web/galaxy-web.cpp +++ b/galaxy-web/galaxy-web.cpp @@ -35,7 +35,6 @@ static std::unique_ptr galaxy; static std::map groupIndex; static const size_t ThreadListPageSize = 50; -static const size_t SearchPerGroupCap = 20; static const size_t SearchGlobalCap = 200; struct ResultRow @@ -151,8 +150,7 @@ static std::string UriDecode( const char* txt, size_t sz ) continue; } } - if( c == '+' ) str.push_back( ' ' ); - else str.push_back( c ); + str.push_back( c ); i++; } return str; @@ -195,7 +193,11 @@ static std::vector SplitPath( const std::string& uri ) static std::string GetQueryVar( struct mg_str qs, const char* name ) { - char buf[4096]; + // MG_MAX_HTTP_REQUEST_SIZE bounds the entire request line, so the decoded + // value of any query variable can never exceed it. A smaller buffer here + // would let mg_get_http_var silently drop long-but-valid values (returns + // -3) instead of reading them. + char buf[MG_MAX_HTTP_REQUEST_SIZE]; int n = mg_get_http_var( &qs, name, buf, sizeof( buf ) ); if( n > 0 ) return std::string( buf, n ); return std::string(); @@ -237,6 +239,17 @@ static std::string PageFooter() return "\n\n\n\n"; } +static bool IsSafeScheme( const char* txt, size_t len ) +{ + static const char* allowed[] = { "http://", "https://", "ftp://", "mailto:", "news:" }; + for( auto scheme : allowed ) + { + size_t sl = strlen( scheme ); + if( len >= sl && memcmp( txt, scheme, sl ) == 0 ) return true; + } + return false; +} + static std::string RenderMessage( const char* message ) { ml.PrepareLines( message, false ); @@ -271,6 +284,7 @@ static std::string RenderMessage( const char* message ) const bool du = part.deco == MessageLines::D_Underline; const bool di = part.deco == MessageLines::D_Italics; const bool db = part.deco == MessageLines::D_Bold; + bool dlSafe = false; const bool dl = part.deco == MessageLines::D_Url; if( du ) out += ""; else if( di ) out += ""; @@ -281,10 +295,12 @@ static std::string RenderMessage( const char* message ) { std::string mid( message + part.offset + 5, message + part.offset + part.len ); out += ""; + dlSafe = true; } - else + else if( IsSafeScheme( message + part.offset, part.len ) ) { out += ""; + dlSafe = true; } } @@ -293,7 +309,7 @@ static std::string RenderMessage( const char* message ) if( du ) out += ""; else if( di ) out += ""; else if( db ) out += ""; - else if( dl ) out += ""; + else if( dlSafe ) out += ""; if( !noSpan ) out += ""; } @@ -388,10 +404,12 @@ static void RenderThreadRec( Archive& archive, const std::string& name, uint32_t static uint32_t FindRoot( Archive& archive, uint32_t idx ) { + uint32_t steps = 0; + const uint32_t maxSteps = archive.NumberOfMessages() + 1; for(;;) { int32_t p = archive.GetParent( idx ); - if( p < 0 ) return idx; + if( p < 0 || ++steps > maxSteps ) return idx; idx = (uint32_t)p; } } @@ -433,10 +451,17 @@ static std::vector SearchArchive( Archive& archive, const std::string SearchEngine se( archive ); auto data = se.Search( query.c_str(), SearchEngine::SF_AdjacentWords ); auto& results = data.results; + if( results.empty() ) return out; std::sort( results.begin(), results.end(), []( const SearchResult& a, const SearchResult& b ) { return a.rank > b.rank; } ); size_t n = std::min( results.size(), cap ); out.reserve( n ); - for( size_t i=0; i 0.f ? 1.f / maxRank : 1.f; + for( size_t i=0; iPackMsgId( midStr.c_str(), packed ); @@ -537,7 +562,7 @@ static void Handler( struct mg_connection* nc, int ev, void* data ) for( int idx : galaxy->GetAvailableArchives() ) { auto& archive = *galaxy->GetArchive( idx, false ); - auto part = SearchArchive( archive, galaxy->GetArchiveName( idx ), q, SearchPerGroupCap ); + auto part = SearchArchive( archive, galaxy->GetArchiveName( idx ), q, SearchGlobalCap ); rows.insert( rows.end(), part.begin(), part.end() ); } std::sort( rows.begin(), rows.end(), []( const ResultRow& a, const ResultRow& b ) { return a.rank > b.rank; } ); @@ -570,13 +595,16 @@ static void Handler( struct mg_connection* nc, int ev, void* data ) { std::string q = GetQueryVar( hm->query_string, "q" ); title = name + " search"; - auto rows = SearchArchive( archive, name, q, SearchGlobalCap ); - body = RenderResults( rows, q, "/group/" + Encode( name ) + "/search", false ); + auto rows = SearchArchive( archive, name, q, SearchGlobalCap + 1 ); + bool truncated = rows.size() > SearchGlobalCap; + if( truncated ) rows.resize( SearchGlobalCap ); + body = RenderResults( rows, q, "/group/" + Encode( name ) + "/search", truncated ); } else if( parts.size() == 4 && parts[2] == "thread" ) { - uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), nullptr, 10 ); - if( idx >= archive.NumberOfMessages() ) code = 404; + char* end; + uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), &end, 10 ); + if( *end != '\0' || end == parts[3].c_str() || idx >= archive.NumberOfMessages() ) code = 404; else { title = name; @@ -585,8 +613,9 @@ static void Handler( struct mg_connection* nc, int ev, void* data ) } else if( parts.size() == 4 && parts[2] == "msg" ) { - uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), nullptr, 10 ); - if( idx >= archive.NumberOfMessages() ) code = 404; + char* end; + uint32_t idx = (uint32_t)strtoul( parts[3].c_str(), &end, 10 ); + if( *end != '\0' || end == parts[3].c_str() || idx >= archive.NumberOfMessages() ) code = 404; else { title = name; diff --git a/uat/uat.cpp b/uat/uat.cpp index a3e1208..d6a2b20 100644 --- a/uat/uat.cpp +++ b/uat/uat.cpp @@ -24,6 +24,7 @@ static const Tool tools[] = { { "filter-newsgroups", "Remove messages with wrong \"Newsgroups\" field." }, { "filter-spam", "Learn which messages are spam and remove them." }, { "galaxy-util", "Generate archive galaxy data." }, + { "galaxy-web", "Start web server for browsing a galaxy." }, { "google-groups", "Crawl google groups and save in maildir tree." }, { "import-source-maildir", "Import messages from a directory tree." }, { "import-source-maildir-7z", "Import messages from a compressed directory tree." },