Pdf.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using PdfSharp.Pdf;
using PdfSharp.Pdf.IO;
using RT.Json;
using RT.Servers;
using RT.TagSoup;
using RT.Util;
using RT.Util.ExtensionMethods;

namespace KtaneWeb
{
    public sealed partial class KtanePropellerModule
    {
        private HttpResponse pdf(HttpRequest req)
        {
            if (!req.Url.Path.StartsWith("/PDF/", StringComparison.InvariantCultureIgnoreCase))
                return null;

            var filename = req.Url.Path[5..];
            if (filename.Length < 1 || filename.Contains('/'))
                return null;
            try { filename = filename.UrlUnescape(); }
            catch (Exception) { return HttpResponse.Empty(HttpStatusCode._400_BadRequest); }

            // If the PDF file already exists in the PDF folder, use that
            var existingPdfPath = Path.Combine(_config.BaseDir, "PDF", filename);
            if (File.Exists(existingPdfPath))
                return null;

            // See if an equivalent HTML file exists, even with a wildcard match or incorrect filename capitilization
            var htmlFile = new DirectoryInfo(Path.Combine(_config.BaseDir, "HTML")).GetFiles(Path.GetFileNameWithoutExtension(filename) + ".html").Select(fs => fs.FullName).FirstOrDefault();
            if (htmlFile == null)
                return null;

            // Check if the PDF filename is exactly correct and redirect if it isn’t
            var pdfUrl = $"/PDF/{Path.GetFileNameWithoutExtension(htmlFile)}.pdf";
            if (!Regex.IsMatch(pdfUrl, $"^{Regex.Escape("/PDF/" + filename).Replace("\\*", ".*")}$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant))
                return null;
            if (pdfUrl != req.Url.Path[..5] + filename)
                return HttpResponse.Redirect(req.Url.WithPath(pdfUrl));

            // Turns out an HTML file corresponding to the requested PDF file exists, so we will try to generate the PDF automatically by invoking Google Chrome
            KtaneModuleInfo module = null;
            try { module = _moduleInfoCache.Modules.First(mod => mod.FileName == Path.GetFileNameWithoutExtension(filename)); }
            catch { }
            return HttpResponse.File(generatePdf(htmlFile, false, module?.PageRenderTime ?? defaultRenderDelay).filename, "application/pdf");
        }

        private const int defaultRenderDelay = 500;
        /// <summary>
        ///     If <paramref name="avoidGenerating"/> is <c>false</c>, generates a PDF for the specified HTML file. If
        ///     <paramref name="avoidGenerating"/> is <c>true</c>, examines whether a PDF would need to be generated.</summary>
        /// <param name="htmlFile">
        ///     Full path to the HTML file.</param>
        /// <param name="avoidGenerating">
        ///     If <c>true</c>, no PDFs are generated.</param>
        /// <returns>
        ///     The filename of the generated (or would-be generated) PDF, and a boolean specifying whether the PDF was
        ///     generated/would be generated (<c>true</c>) or was already available (<c>false</c>).</returns>
        private (string filename, bool wasGenerated) generatePdf(string htmlFile, bool avoidGenerating = false, int renderDelay = defaultRenderDelay)
        {
            var cache = _moduleInfoCache;
            lock (cache.AutogeneratedPdfs)
            {
                if (cache.AutogeneratedPdfs.TryGetValue(htmlFile, out var pdfFile) && File.Exists(pdfFile))
                {
                    File.SetLastWriteTimeUtc(pdfFile, DateTime.UtcNow);
                    return (pdfFile, false);
                }

                var tempFilename = $"{MD5.HashData(File.ReadAllBytes(htmlFile)).ToHex()}.pdf";
                var tempFilepath = Path.Combine(_config.PdfTempPath ?? Path.GetTempPath(), tempFilename);
                var didGenerate = false;
                if (!File.Exists(tempFilepath))
                {
                    if (avoidGenerating)
                        return (tempFilepath, true);
                    var runner = new CommandRunner();
                    runner.Command = $@"cmd.exe /S /C """"{_config.ChromePath}"" --headless --disable-gpu ""--print-to-pdf={tempFilepath}"" ""--virtual-time-budget={renderDelay}"" --run-all-compositor-stages-before-draw --no-margins ""file:///{htmlFile.Replace('\\', '/').UrlEscape()}""""";
                    runner.StartAndWait();
                    didGenerate = true;
                }
                else
                    File.SetLastAccessTimeUtc(tempFilepath, DateTime.UtcNow);

                cache.AutogeneratedPdfs[htmlFile] = tempFilepath;
                return (tempFilepath, didGenerate);
            }
        }

        private HttpResponse mergePdfs(HttpRequest req)
        {
            var lastExaminedPdfFile = "<none>";

            var language = req.Headers.Cookie.Get("lang", null)?.Value ?? "en";
            var langName = TranslationInfo.LanguageCodeToName.Get(language, "English");

            try
            {
                if (req.Method != HttpMethod.Post)
                    return HttpResponse.Redirect(req.Url.WithPathParent().WithPath(""));

                var messages = new StringBuilder();
                var json = JsonValue.Parse(req.Post["json"].Value);
                json.AppendIndented(messages);
                var keywords = json["search"].GetString().Length == 0 ? null : json["search"].GetString().Split([' '], StringSplitOptions.RemoveEmptyEntries);
                var searchOptions = json["searchOptions"].GetList().Select(j => j.GetString()).ToArray();
                var filterEnabledByProfile = json["filterEnabledByProfile"].GetBool();
                var filterVetoedByProfile = json["filterVetoedByProfile"].GetBool();
                var profileVetoList = (filterEnabledByProfile == filterVetoedByProfile) ? null : json["profileVetoList"]?.GetList().Where(j => j != null).Select(j => j.GetString()).ToArray();
                var searchBySymbol = json["searchBySymbol"].GetBoolSafe() ?? false;
                var searchBySteamID = json["searchBySteamID"].GetBoolSafe() ?? false;
                var searchByModuleID = json["searchByModuleID"].GetBoolSafe() ?? false;
                var displayAllContributors = json["dispAllContr"].GetBoolSafe() ?? false;
                var displayDesc = json["displayDesc"].GetBoolSafe() ?? false;
                var displayTags = json["displayTags"].GetBoolSafe() ?? false;
                var restrictedManuals = json["restrictedManuals"].GetList().Select(j => j.GetString()).ToArray();

                static string unifyString(string str) => str.Normalize(NormalizationForm.FormD).RegexReplace(@"[\u0300-\u036f]", "").Replace("grey", "gray").Replace("colour", "color");

                // Filter
                var matchingModules = _moduleInfoCache.Modules.Where(m =>
                {
                    // TEMPORARY: Currently there is no easy way to find the correct filename for the manual of a translated module, so we’re excluding those from the merged PDF entirely.
                    // A desirable fix would be to discover the correct PDF filename for the translated manual and include it.
                    if (m.TranslationOf != null)
                        return false;

                    if (profileVetoList != null && !(profileVetoList.Contains(m.ModuleID) ? filterVetoedByProfile : filterEnabledByProfile))
                        return false;

                    foreach (var filter in TranslationInfo.Default.Filters1)
                        if (!filter.Matches(m, json["filter"].Safe[filter.PropName].GetDictSafe()))
                            return false;
                    foreach (var filter in TranslationInfo.Default.Filters2)
                        if (!filter.Matches(m, json["filter"].Safe[filter.PropName].GetDictSafe()))
                            return false;

                    if (keywords == null)
                        return true;

                    var searchWhat = searchBySteamID ? (m.SteamID ?? "") : "";
                    if (searchByModuleID)
                        searchWhat += " " + m.ModuleID.ToLowerInvariant();
                    if (searchOptions.Contains("names"))
                        searchWhat += " " + m.Name.ToLowerInvariant() + " " + m.SortKey.ToLowerInvariant();
                    if (searchOptions.Contains("authors") && (m.Author != null || m.Contributors != null))
                        if (displayAllContributors)
                            searchWhat += " " + (m.Author ?? m.Contributors.ToAllAuthorString()).ToLowerInvariant();
                        else
                            searchWhat += " " + (m.Author ?? m.Contributors.ToAuthorString()).ToLowerInvariant();

                    if (searchOptions.Contains("descriptions"))
                    {
                        var descr = m.Descriptions.FirstOrDefault(d => d.Language == langName);
                        if (descr != null && displayDesc)
                            searchWhat += ' ' + descr?.Description.ToLowerInvariant();
                        if (descr != null && displayTags && !string.IsNullOrWhiteSpace(descr.Tags))
                            searchWhat += ' ' + descr?.Tags.ToLowerInvariant();
                    }

                    if (searchBySymbol && m.Symbol != null)
                        searchWhat += " " + m.Symbol.ToLowerInvariant();

                    return keywords.All(unifyString(searchWhat).ContainsIgnoreCase);
                });

                // Sort
                switch (json["sort"].GetString())
                {
                    case "name": matchingModules = matchingModules.OrderBy(m => m.SortKey); break;
                    case "defdiff": matchingModules = matchingModules.OrderBy(m => m.DefuserDifficulty); break;
                    case "expdiff": matchingModules = matchingModules.OrderBy(m => m.ExpertDifficulty); break;
                    case "twitchscore": matchingModules = matchingModules.OrderBy(m => m.TwitchPlaysScore ?? 0); break;
                    case "timemodescore": matchingModules = matchingModules.OrderBy(m => m.TimeMode?.Score ?? 0); break;
                    case "published": matchingModules = matchingModules.OrderByDescending(m => m.Published); break;
                }

                var pdfFiles = new List<string>();
                var generated = 0;
                var notGenerated = new List<string>();
                var startTime = DateTime.UtcNow;

                foreach (var module in matchingModules)
                {
                    var filename = $"{module.FileName}.pdf";
                    lastExaminedPdfFile = filename;
                    string fullPath = null;
                    if (json["preferredManuals"].ContainsKey(module.Name))
                    {
                        var pref = json["preferredManuals"][module.Name].GetString();
                        var hasMatch = pref.RegexMatch(@"^(.*) \((PDF|HTML)\)$", out var match);
                        var fullname = $"{module.Name} {match.Groups[1].Value}";
                        var unrestricted = !restrictedManuals.Contains(fullname);

                        // PDF file exists
                        if (hasMatch && unrestricted && match.Groups[2].Value == "PDF"
                                && Path.Combine(_config.BaseDir, "PDF", $"{fullname.Replace(module.Name, module.FileName)}.pdf") is { } path
                                && File.Exists(path))
                        {
                            messages.AppendLine($"{pref} (pref) ⇒ {path}");
                            fullPath = path;
                        }
                        // HTML file exists, regardless if HTML or PDF is selected
                        else if (hasMatch && unrestricted
                                && Path.Combine(_config.BaseDir, "HTML", $"{fullname.Replace(module.Name, module.FileName)}.html") is { } htmlPath
                                && File.Exists(htmlPath))
                        {
                            messages.AppendLine($"{pref} (pref) ⇒ {htmlPath}");
                            fullPath = htmlPath;
                        }
                    }

                    if (fullPath == null)
                    {
                        fullPath = Path.Combine(_config.BaseDir, _config.PdfDir, filename);
                        if (!File.Exists(fullPath))
                            fullPath = Path.Combine(_config.BaseDir, "HTML", $"{module.FileName}.html");
                        if (!File.Exists(fullPath))
                            return HttpResponse.PlainText($"Cannot find {filename}.", HttpStatusCode._500_InternalServerError);
                        messages.AppendLine($"{module.Name} (no pref) ⇒ {fullPath}");
                    }

                    // Generate PDFs
                    if (fullPath.EndsWith(".html") && File.Exists(fullPath))
                    {
                        var avoidGenerating = (DateTime.UtcNow - startTime).TotalSeconds > 5;
                        var (pdfFilename, pdfGenerated) = generatePdf(fullPath, avoidGenerating, module.PageRenderTime ?? defaultRenderDelay);
                        messages.AppendLine($"{module.Name} avoid={avoidGenerating} gen={pdfGenerated} file={pdfFilename}");
                        if (avoidGenerating && pdfGenerated)
                        {
                            notGenerated.Add(Path.GetFileNameWithoutExtension(fullPath));
                            messages.AppendLine($"{module.Name} added to not-generated list");
                        }
                        else if (!avoidGenerating)
                        {
                            if (pdfGenerated)
                                generated++;
                            fullPath = pdfFilename;
                            messages.AppendLine($"{module.Name} (no pref) ⇒ {fullPath} ({(pdfGenerated ? "generated" : "from cache")})");
                        }
                    }

                    if (File.Exists(fullPath))
                        pdfFiles.Add(fullPath);
                    else
                        messages.AppendLine($" — {fullPath} does not exist");
                }

                if (pdfFiles.Count == 0)
                    return HttpResponse.PlainText($"Error: no matching manuals found.\n\n{messages}", HttpStatusCode._500_InternalServerError);

                if (notGenerated.Count > 0)
                    return HttpResponse.Html($"Looks like you’re asking me to generate a lot of PDF files from HTML. I’ve just generated {generated} and I will need to generate the following {notGenerated.Count} more. Please refresh this page once every minute to incrementally have your PDFs generated. Please do not overload the server with excessive requests for merged PDFs, or this feature will need to be disabled.<ul>{notGenerated.Select(g => $"<li>{g.HtmlEscape()}</li>").JoinString()}</ul><!--\n\n{messages}\n\n-->", HttpStatusCode._202_Accepted);

                var list = pdfFiles.JoinString("\n");
                using var mem = new MemoryStream(list.ToUtf8());
                using var sha1hash = SHA1.Create();
                var sha1 = sha1hash.ComputeHash(mem).ToHex();
                var pdfPath = Path.Combine(_config.BaseDir, _config.MergedPdfsDir, $"{sha1}.pdf");
                if (!File.Exists(pdfPath))
                    lock (this)
                        if (!File.Exists(pdfPath))
                        {
                            var mergedPdf = new PdfDocument();
                            foreach (var pdfFile in pdfFiles)
                            {
                                lastExaminedPdfFile = pdfFile;
                                var pdf = PdfReader.Open(Path.Combine(_config.BaseDir, _config.PdfDir, pdfFile), PdfDocumentOpenMode.Import);
                                var count = pdf.PageCount;
                                for (var idx = 0; idx < count; idx++)
                                    mergedPdf.AddPage(pdf.Pages[idx]);
                            }
                            using var f = File.OpenWrite(pdfPath);
                            mergedPdf.Save(f);
                        }
                return HttpResponse.Redirect(req.Url.WithPathParent().WithPathOnly($"/MergedPdfs/{sha1}.pdf"));
            }
            catch (Exception e)
            {
                var exc = e;
                var sb = new StringBuilder();
                while (exc != null)
                {
                    sb.AppendLine($"Error processing PDFs:\r\n{e.GetType().FullName}\r\n{e.Message}\r\nPossible culprit: {lastExaminedPdfFile}\r\n\r\n{e.StackTrace}\r\n\r\n");
                    exc = exc.InnerException;
                }
                return HttpResponse.PlainText(sb.ToString(), HttpStatusCode._500_InternalServerError);
            }
        }
    }
}