From 47da6d824b578381fa8bbd33ce1c3ab3d9bbc0b3 Mon Sep 17 00:00:00 2001 From: GothwadTech <295316515+GothwadTech@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:49:05 +0000 Subject: [PATCH] fix(fdroid): stream index-v1.jar with mirror failover, IzzyOnDroid, disk cache, and reactive UI merge Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .../java/com/gothwad/store/core/AppGraph.kt | 6 +- .../store/data/remote/FdroidCatalog.kt | 575 ++++++++++- .../gothwad/store/data/remote/FdroidIndex.kt | 970 ++++++++++++++++-- .../store/data/remote/StoreRepository.kt | 208 +++- .../store/service/ApkDownloadService.kt | 141 +-- .../com/gothwad/store/ui/StoreViewModel.kt | 191 +++- .../gothwad/store/ui/screens/HomeScreen.kt | 18 +- .../store/ui/screens/ListingScreens.kt | 5 +- .../java/com/gothwad/store/FdroidIndexTest.kt | 184 +++- 9 files changed, 2036 insertions(+), 262 deletions(-) diff --git a/app/src/main/java/com/gothwad/store/core/AppGraph.kt b/app/src/main/java/com/gothwad/store/core/AppGraph.kt index 65d472d..0194d45 100644 --- a/app/src/main/java/com/gothwad/store/core/AppGraph.kt +++ b/app/src/main/java/com/gothwad/store/core/AppGraph.kt @@ -7,6 +7,7 @@ import com.gothwad.store.data.download.DownloadStore import com.gothwad.store.data.install.InstallEngine import com.gothwad.store.data.install.ShizukuBridge import com.gothwad.store.data.library.LibraryRepository +import com.gothwad.store.data.remote.FdroidCatalog import com.gothwad.store.data.remote.StoreNetwork import com.gothwad.store.data.remote.StoreRepository import com.gothwad.store.data.settings.SettingsRepository @@ -18,6 +19,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import okhttp3.OkHttpClient +import java.io.File /** * Hand-rolled dependency graph. Small enough that a DI framework would only add @@ -58,7 +60,9 @@ object AppGraph { if (ready) return true appContext = application settings = SettingsRepository(application) - store = StoreRepository() + store = StoreRepository( + fdroid = FdroidCatalog(cacheDir = File(application.cacheDir, "fdroid")), + ) auth = AuthManager(application) downloads = DownloadStore(application) installer = InstallEngine(application) { downloads.apkDir } diff --git a/app/src/main/java/com/gothwad/store/data/remote/FdroidCatalog.kt b/app/src/main/java/com/gothwad/store/data/remote/FdroidCatalog.kt index 58f8e42..454f90a 100644 --- a/app/src/main/java/com/gothwad/store/data/remote/FdroidCatalog.kt +++ b/app/src/main/java/com/gothwad/store/data/remote/FdroidCatalog.kt @@ -1,42 +1,84 @@ package com.gothwad.store.data.remote import android.os.Build +import android.util.Log import com.gothwad.store.BuildConfig import com.gothwad.store.data.model.AppDetailResponse import com.gothwad.store.data.model.CatalogResponse -import com.squareup.moshi.JsonReader +import com.gothwad.store.data.model.StoreApp +import com.gothwad.store.data.model.StoreRelease import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request +import okio.buffer +import okio.sink +import java.io.File +import java.io.IOException import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger /** - * Read-only client for F-Droid's public repository index. APK bytes stay on - * F-Droid; only the metadata is mirrored into the app. + * Read-only client for F-Droid's public repository index (and IzzyOnDroid, the + * standard companion F-Droid repository bundled in Droid-ify). APK bytes stay + * on F-Droid / IzzyOnDroid; only the metadata is mirrored into the app. * - * The index is roughly 8 MB, so it is streamed through Moshi instead of being - * materialised as one giant string — otherwise the app runs out of heap on - * mid-range devices and the catalogue silently disappears. + * Key design choices modeled directly after Droid-ify (`IndexDownloader.kt`, + * `RepositoryUpdater.kt`, `V1Syncable.kt`, `Repository.kt`) and the official + * F-Droid client (`IndexV1Updater.kt`, `KnownRepos.kt`): + * 1. Downloads `index-v1.jar` (compressed JAR/ZIP archive, ~8.8 MB for F-Droid + * and ~1.6 MB for IzzyOnDroid) instead of raw `index-v1.json` (~40+ MB), + * with automatic fallback to `index-v2.json` / `index-v1.json`. + * 2. Races official HTTPS F-Droid mirrors (`f-droid.org`, `de.freedif.org`, + * `mirror.cyberbits.eu`, `ftp.fau.de`, `mirrors.dotsrc.org`, etc.) using a + * staggered Happy-Eyeballs strategy so a slow or throttled primary server + * never blocks loading. + * 3. Runs downloads in a dedicated [SupervisorJob] scope so switching tabs or + * cancelling a single screen's coroutine never aborts an in-flight sync. + * 4. Caches downloaded `.jar` indexes on disk and emits progressively through + * [catalogFlow] so the UI populates as soon as the fastest repo finishes + * and seamlessly upgrades when the full catalogue arrives. */ class FdroidCatalog( private val http: OkHttpClient = OkHttpClient.Builder() - .connectTimeout(20, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .callTimeout(120, TimeUnit.SECONDS) + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(45, TimeUnit.SECONDS) + // Do not set a hard 120s callTimeout that kills multi-megabyte index downloads + // mid-stream on mobile networks. + .callTimeout(0, TimeUnit.SECONDS) + .followRedirects(true) + .followSslRedirects(true) + .retryOnConnectionFailure(true) .build(), + private val cacheDir: File? = null, + private val indexUrl: String = INDEX_URL, + private val repoUrl: String = REPO_URL, + private val izzyRepoUrl: String = IZZY_REPO_URL, ) { - @Volatile private var cached: CatalogResponse? = null - @Volatile private var cachedAt: Long = 0 + private val catalogScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val stateMutex = Mutex() - /** One download at a time: home, apps and games all ask for the index. */ - private val mutex = Mutex() + @Volatile private var cached: CatalogResponse? = null + @Volatile private var fdroidCached: CatalogResponse? = null + @Volatile private var izzyCached: CatalogResponse? = null + @Volatile private var cachedAt: Long = 0L + @Volatile private var activeSyncDeferred: Deferred? = null + @Volatile private var diskCacheChecked: Boolean = false private val _lastError = MutableStateFlow(null) @@ -46,56 +88,499 @@ class FdroidCatalog( */ val lastError: StateFlow = _lastError.asStateFlow() - suspend fun catalog(): CatalogResponse = withContext(Dispatchers.IO) { - cached?.takeIf { System.currentTimeMillis() - cachedAt < CACHE_MILLIS }?.let { return@withContext it } - mutex.withLock { - // Another caller may have finished the download while we waited. - cached?.takeIf { System.currentTimeMillis() - cachedAt < CACHE_MILLIS } - ?: try { - download() - } catch (cancellation: CancellationException) { - throw cancellation - } catch (error: Exception) { - _lastError.value = error.message ?: "F-Droid is unavailable right now." - throw error + private val _catalogFlow = MutableStateFlow(null) + + /** + * Emits whenever the F-Droid catalogue is loaded from disk cache, updated + * by IzzyOnDroid, or updated by the full F-Droid repository. + */ + val catalogFlow: StateFlow = _catalogFlow.asStateFlow() + + private val effectiveCacheDir: File by lazy { + val dir = cacheDir ?: File(System.getProperty("java.io.tmpdir", "/tmp"), "kiwistore-fdroid") + runCatching { dir.mkdirs() } + dir + } + + private val fdroidDiskFile: File + get() = File(effectiveCacheDir, "fdroid_index_v1.jar") + + private val izzyDiskFile: File + get() = File(effectiveCacheDir, "izzy_index_v1.jar") + + suspend fun catalog(forceRefresh: Boolean = false): CatalogResponse { + val now = System.currentTimeMillis() + if (!forceRefresh) { + cached?.let { existing -> + if (now - cachedAt < CACHE_MILLIS && existing.apps.isNotEmpty()) { + return existing } + } + if (!diskCacheChecked) { + val diskLoaded = loadFromDiskIfPresent() + if (diskLoaded != null && diskLoaded.apps.isNotEmpty()) { + val isFresh = now - cachedAt < CACHE_MILLIS && fdroidCached != null + if (!isFresh) { + ensureSyncStarted(forceRefresh = false) + } + return diskLoaded + } + } + } + + val deferred = ensureSyncStarted(forceRefresh = forceRefresh) + return try { + deferred.await() + } catch (cancellation: CancellationException) { + throw cancellation + } catch (error: Exception) { + cached?.takeIf { it.apps.isNotEmpty() } ?: throw error } } suspend fun find(packageName: String): AppDetailResponse? { - val catalog = catalog() - val app = catalog.apps.firstOrNull { it.packageName == packageName } ?: return null + val raw = packageName.removePrefix("fdroid:") + val catalog = runCatching { catalog() }.getOrNull() ?: cached ?: return null + val app = catalog.apps.firstOrNull { + it.packageName.equals(raw, ignoreCase = true) || + it.id.equals(packageName, ignoreCase = true) + } ?: return null + val liveRelease = catalog.releases.firstOrNull { it.appId == app.id } return AppDetailResponse( app = app, - liveRelease = catalog.releases.firstOrNull { it.appId == app.id }, + screenshots = app.screenshots, + liveRelease = liveRelease, + releases = listOfNotNull(liveRelease), ) } - private fun download(): CatalogResponse { - val request = Request.Builder() - .url(INDEX_URL) - .header("Accept", "application/json") - .header("User-Agent", "KiwiStore-Android/${BuildConfig.VERSION_NAME}") - .build() - http.newCall(request).execute().use { response -> - if (!response.isSuccessful) error("F-Droid repository returned HTTP ${response.code}") - val body = response.body ?: error("F-Droid repository returned an empty index.") - val index = FdroidIndexParser.parse(JsonReader.of(body.source()), Build.VERSION.SDK_INT) - val catalog = index.toCatalog(REPO_URL) - check(catalog.apps.isNotEmpty()) { - "F-Droid index was read, but it has no APK compatible with Android ${Build.VERSION.SDK_INT}." + private suspend fun loadFromDiskIfPresent(): CatalogResponse? = stateMutex.withLock { + if (diskCacheChecked) return@withLock cached + diskCacheChecked = true + + val abis = deviceAbis() + val sdk = Build.VERSION.SDK_INT + + val diskFdroid = fdroidDiskFile.takeIf { it.exists() && it.length() > 1024L }?.let { file -> + runCatching { + FdroidIndexParser.parseFile(file, sdk, abis).toCatalog(repoUrl) + }.onFailure { e -> + Log.w(TAG, "Deleting corrupted cached F-Droid index ${file.name}: ${e.message}") + runCatching { file.delete() } + }.getOrNull() + } + + val diskIzzy = izzyDiskFile.takeIf { it.exists() && it.length() > 1024L }?.let { file -> + runCatching { + FdroidIndexParser.parseFile(file, sdk, abis).toCatalog(izzyRepoUrl) + }.onFailure { e -> + Log.w(TAG, "Deleting corrupted cached IzzyOnDroid index ${file.name}: ${e.message}") + runCatching { file.delete() } + }.getOrNull() + } + + if (diskFdroid == null && diskIzzy == null) return@withLock null + + if (diskFdroid != null) fdroidCached = diskFdroid + if (diskIzzy != null) izzyCached = diskIzzy + + val merged = mergeCatalogs(fdroidCached, izzyCached) + if (merged.apps.isNotEmpty()) { + cached = merged + val fdroidMod = fdroidDiskFile.takeIf { it.exists() }?.lastModified() ?: 0L + val izzyMod = izzyDiskFile.takeIf { it.exists() }?.lastModified() ?: 0L + cachedAt = when { + fdroidMod > 0L && izzyMod > 0L -> minOf(fdroidMod, izzyMod) + fdroidMod > 0L -> fdroidMod + else -> izzyMod + } + _lastError.value = null + _catalogFlow.value = merged + Log.i(TAG, "Loaded ${merged.apps.size} F-Droid apps from disk cache.") + } + cached + } + + private suspend fun ensureSyncStarted(forceRefresh: Boolean): Deferred = + stateMutex.withLock { + val current = activeSyncDeferred + if (!forceRefresh && current != null && current.isActive) { + return@withLock current + } + + val firstReady = CompletableDeferred() + activeSyncDeferred = firstReady + + catalogScope.launch { + val errors = mutableListOf() + try { + coroutineScope { + val izzyJob = launch { + runCatching { fetchIzzyRepo() } + .onSuccess { izzy -> + publishRepoUpdate(newFdroid = null, newIzzy = izzy, firstReady = firstReady) + } + .onFailure { e -> + if (e is CancellationException) throw e + Log.w(TAG, "IzzyOnDroid sync failed: ${e.message}") + synchronized(errors) { + errors += "IzzyOnDroid: ${e.message ?: e::class.java.simpleName}" + } + } + } + + val fdroidJob = launch { + runCatching { fetchOfficialFdroidRepo() } + .onSuccess { fdroid -> + publishRepoUpdate(newFdroid = fdroid, newIzzy = null, firstReady = firstReady) + } + .onFailure { e -> + if (e is CancellationException) throw e + Log.e(TAG, "Official F-Droid sync failed across mirrors: ${e.message}", e) + synchronized(errors) { + errors += "F-Droid: ${e.message ?: e::class.java.simpleName}" + } + } + } + + izzyJob.join() + fdroidJob.join() + } + + val finalCatalog = cached + if (finalCatalog != null && finalCatalog.apps.isNotEmpty()) { + cachedAt = System.currentTimeMillis() + _lastError.value = null + if (!firstReady.isCompleted) { + firstReady.complete(finalCatalog) + } + } else { + val combinedMessage = synchronized(errors) { + errors.joinToString(" | ").ifBlank { "F-Droid is unavailable right now." } + } + _lastError.value = combinedMessage + if (!firstReady.isCompleted) { + firstReady.completeExceptionally(IOException(combinedMessage)) + } + } + } catch (e: Exception) { + if (e !is CancellationException) { + val msg = e.message ?: "F-Droid is unavailable right now." + _lastError.value = msg + if (!firstReady.isCompleted) { + firstReady.completeExceptionally(e) + } + } + } finally { + stateMutex.withLock { + if (activeSyncDeferred === firstReady) { + activeSyncDeferred = null + } + } + } + } + + firstReady + } + + private suspend fun publishRepoUpdate( + newFdroid: CatalogResponse?, + newIzzy: CatalogResponse?, + firstReady: CompletableDeferred, + ) { + stateMutex.withLock { + if (newFdroid != null && newFdroid.apps.isNotEmpty()) { + fdroidCached = newFdroid + } + if (newIzzy != null && newIzzy.apps.isNotEmpty()) { + izzyCached = newIzzy } - return catalog.also { parsed -> - cached = parsed + val merged = mergeCatalogs(fdroidCached, izzyCached) + if (merged.apps.isNotEmpty()) { + cached = merged cachedAt = System.currentTimeMillis() _lastError.value = null + _catalogFlow.value = merged + if (!firstReady.isCompleted) { + firstReady.complete(merged) + } } } } + private suspend fun fetchOfficialFdroidRepo(): CatalogResponse { + val customBase = repoUrl.trimEnd('/') + val mirrors = buildList { + add(customBase) + for (m in FDROID_MIRRORS) { + val clean = m.trimEnd('/') + if (clean !in this) add(clean) + } + } + + val jarUrls = buildList { + if (indexUrl.endsWith(".jar", ignoreCase = true)) { + add(indexUrl) + } + for (m in mirrors) { + val u = "$m/index-v1.jar" + if (u !in this) add(u) + } + } + + val jarError = try { + return raceMirrorsForIndex( + urls = jarUrls, + targetCacheFile = fdroidDiskFile, + canonicalRepoUrl = repoUrl, + staggerMillis = MIRROR_STAGGER_MS, + maxConcurrent = 3, + ) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (e: Exception) { + e + } + + val fallbackUrls = buildList { + if (!indexUrl.endsWith(".jar", ignoreCase = true)) add(indexUrl) + for (m in mirrors.take(4)) { + add("$m/index-v2.json") + add("$m/index-v1.json") + } + } + return try { + raceMirrorsForIndex( + urls = fallbackUrls, + targetCacheFile = fdroidDiskFile, + canonicalRepoUrl = repoUrl, + staggerMillis = MIRROR_STAGGER_MS, + maxConcurrent = 2, + ) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (_: Exception) { + throw jarError + } + } + + private suspend fun fetchIzzyRepo(): CatalogResponse { + val base = izzyRepoUrl.trimEnd('/') + val urls = listOf( + "$base/index-v1.jar", + "$base/index-v2.json", + "$base/index-v1.json", + ) + return raceMirrorsForIndex( + urls = urls, + targetCacheFile = izzyDiskFile, + canonicalRepoUrl = izzyRepoUrl, + staggerMillis = 4_000L, + maxConcurrent = 2, + ) + } + + private suspend fun raceMirrorsForIndex( + urls: List, + targetCacheFile: File, + canonicalRepoUrl: String, + staggerMillis: Long, + maxConcurrent: Int, + ): CatalogResponse = coroutineScope { + val resultChannel = Channel>(Channel.UNLIMITED) + val inFlight = AtomicInteger(0) + val parentScope = this + val attemptCounter = AtomicInteger(0) + + val launcherJob = launch { + for ((idx, url) in urls.withIndex()) { + if (idx > 0) { + delay(staggerMillis) + } + while (inFlight.get() >= maxConcurrent) { + delay(250L) + } + inFlight.incrementAndGet() + val attemptId = attemptCounter.incrementAndGet() + parentScope.launch { + try { + val response = downloadAndParseIndex( + url = url, + targetCacheFile = targetCacheFile, + canonicalRepoUrl = canonicalRepoUrl, + attemptId = attemptId, + ) + resultChannel.send(Result.success(response)) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (e: Exception) { + Log.w(TAG, "Mirror attempt failed ($url): ${e.message}") + resultChannel.send(Result.failure(e)) + } finally { + inFlight.decrementAndGet() + } + } + } + } + + var lastError: Exception? = null + var failures = 0 + val total = urls.size + while (failures < total) { + val next = resultChannel.receive() + if (next.isSuccess) { + val winner = next.getOrThrow() + launcherJob.cancel() + coroutineContext[Job]?.cancelChildren() + return@coroutineScope winner + } else { + failures++ + lastError = (next.exceptionOrNull() as? Exception) ?: lastError + } + } + throw lastError ?: IOException("All F-Droid index mirrors failed.") + } + + private fun downloadAndParseIndex( + url: String, + targetCacheFile: File, + canonicalRepoUrl: String, + attemptId: Int, + ): CatalogResponse { + val tempFile = File( + effectiveCacheDir, + "${targetCacheFile.name}.tmp.$attemptId.${Thread.currentThread().id}", + ) + try { + val request = Request.Builder() + .url(url) + .header("Accept", "application/java-archive, application/zip, application/json, */*") + .header("User-Agent", "KiwiStore-Android/${BuildConfig.VERSION_NAME}") + .build() + + http.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("F-Droid repository returned HTTP ${response.code} for $url") + } + val body = response.body ?: throw IOException("F-Droid repository returned an empty index.") + tempFile.parentFile?.mkdirs() + tempFile.sink().buffer().use { sink -> + sink.writeAll(body.source()) + } + } + + if (!tempFile.exists() || tempFile.length() < 256L) { + throw IOException("F-Droid index download from $url was truncated (${tempFile.length()} bytes).") + } + + val index = FdroidIndexParser.parseFile( + file = tempFile, + sdkInt = Build.VERSION.SDK_INT, + supportedAbis = deviceAbis(), + ) + val catalog = index.toCatalog(canonicalRepoUrl) + check(catalog.apps.isNotEmpty()) { + "F-Droid index was read, but it has no APK compatible with Android ${Build.VERSION.SDK_INT}." + } + + synchronized(this) { + runCatching { + if (!tempFile.renameTo(targetCacheFile)) { + tempFile.copyTo(targetCacheFile, overwrite = true) + tempFile.delete() + } + } + } + + Log.i(TAG, "Loaded ${catalog.apps.size} apps from $url") + return catalog + } finally { + runCatching { if (tempFile.exists()) tempFile.delete() } + } + } + + private fun mergeCatalogs( + primary: CatalogResponse?, + secondary: CatalogResponse?, + ): CatalogResponse { + if (primary == null && secondary == null) return CatalogResponse(source = "F-Droid") + if (primary == null || primary.apps.isEmpty()) return secondary ?: CatalogResponse(source = "F-Droid") + if (secondary == null || secondary.apps.isEmpty()) return primary + + val seenPackages = HashSet(primary.apps.size + secondary.apps.size) + val mergedApps = ArrayList(primary.apps.size + secondary.apps.size) + val validAppIds = HashSet(primary.apps.size + secondary.apps.size) + + for (app in primary.apps) { + if (seenPackages.add(app.packageName)) { + mergedApps += app + validAppIds += app.id + } + } + for (app in secondary.apps) { + if (seenPackages.add(app.packageName)) { + mergedApps += app + validAppIds += app.id + } + } + mergedApps.sortByDescending { it.updatedAt } + + val releaseByAppId = LinkedHashMap(validAppIds.size) + for (rel in secondary.releases) { + if (rel.appId in validAppIds) releaseByAppId[rel.appId] = rel + } + for (rel in primary.releases) { + if (rel.appId in validAppIds) releaseByAppId[rel.appId] = rel + } + + return CatalogResponse( + apps = mergedApps, + releases = releaseByAppId.values.toList(), + source = "F-Droid", + ) + } + + private fun deviceAbis(): Set = + runCatching { Build.SUPPORTED_ABIS?.filter { it.isNotBlank() }?.toSet().orEmpty() } + .getOrDefault(emptySet()) + companion object { - const val INDEX_URL = "https://f-droid.org/repo/index-v1.json" + private const val TAG = "FdroidCatalog" + const val INDEX_URL = "https://f-droid.org/repo/index-v1.jar" const val REPO_URL = "https://f-droid.org/repo" + const val IZZY_INDEX_URL = "https://apt.izzysoft.de/fdroid/repo/index-v1.jar" + const val IZZY_REPO_URL = "https://apt.izzysoft.de/fdroid/repo" private const val CACHE_MILLIS = 6 * 60 * 60 * 1000L + private const val MIRROR_STAGGER_MS = 3_000L + + /** + * Official HTTPS mirrors announced in F-Droid's `repo.mirrors` metadata + * and used by Droid-ify / F-Droid client. + */ + val FDROID_MIRRORS: List = listOf( + "https://f-droid.org/repo", + "https://de.freedif.org/fdroid/repo", + "https://mirror.cyberbits.eu/fdroid/repo", + "https://ftp.fau.de/fdroid/repo", + "https://mirrors.dotsrc.org/fdroid/repo", + "https://ftp.agdsn.de/fdroid/repo", + "https://mirror.fcix.net/fdroid/repo", + "https://mirror.freedif.org/fdroid/repo", + "https://ftp.gwdg.de/pub/android/fdroid/repo", + "https://ftp.lysator.liu.se/pub/fdroid/repo", + ) + + /** + * Returns candidate download URLs for an APK or asset URL. If [url] + * points to `https://f-droid.org/repo/...`, includes the official + * HTTPS mirrors as automatic failover targets. + */ + fun candidateUrls(url: String): List { + val clean = url.trim() + val prefix = "$REPO_URL/" + if (!clean.startsWith(prefix)) return listOf(clean) + val relativePath = clean.removePrefix(prefix) + return FDROID_MIRRORS.map { mirror -> "${mirror.trimEnd('/')}/$relativePath" } + } } } diff --git a/app/src/main/java/com/gothwad/store/data/remote/FdroidIndex.kt b/app/src/main/java/com/gothwad/store/data/remote/FdroidIndex.kt index 34c1904..7731016 100644 --- a/app/src/main/java/com/gothwad/store/data/remote/FdroidIndex.kt +++ b/app/src/main/java/com/gothwad/store/data/remote/FdroidIndex.kt @@ -4,14 +4,32 @@ import com.gothwad.store.data.model.CatalogResponse import com.gothwad.store.data.model.StoreApp import com.gothwad.store.data.model.StoreRelease import com.squareup.moshi.JsonReader +import okio.BufferedSource +import okio.GzipSource +import okio.buffer +import okio.source +import java.io.File +import java.io.RandomAccessFile +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone +import java.util.zip.ZipFile +import java.util.zip.ZipInputStream /** - * The small subset of F-Droid's v1 index that Kiwi Store retains. + * The subset of F-Droid's v1 (`index-v1.jar` / `index-v1.json`) and v2 + * (`index-v2.json`) repository indexes that Kiwi Store retains. * - * `apps` in index-v1.json is an array, while `packages` is an object keyed by - * package name. The parser below streams both sections and keeps only the - * newest APK compatible with this device, rather than building a second huge - * copy of the several-megabyte index in memory. + * Modeled after Droid-ify (`IndexV1Parser.kt`, `IndexConverter.kt`) and the + * official F-Droid client (`IndexV1Updater.kt`, `AppV1.kt`): + * - `apps` in `index-v1.json` is an array while `packages` is an object keyed + * by package name. + * - Supports streaming directly from `index-v1.jar` (ZIP/JAR archive containing + * `index-v1.json`), which is how both Droid-ify and official F-Droid fetch + * the index (~8.8 MB vs ~40+ MB uncompressed). + * - Retains only the best compatible APK per package (checking `minSdkVersion`, + * `maxSdkVersion`, and device `nativecode` ABIs). */ data class FdroidIndex( val apps: List, @@ -25,11 +43,17 @@ data class FdroidApp( val description: String? = null, val icon: String? = null, val webSite: String? = null, + val sourceCode: String? = null, + val license: String? = null, val authorName: String? = null, val authorEmail: String? = null, val categories: List = emptyList(), /** English localization, or the best available fallback localization. */ val localized: FdroidLocalized? = null, + val localizedLocale: String = "en-US", + val suggestedVersionCode: Long? = null, + val added: Long? = null, + val lastUpdated: Long? = null, ) data class FdroidLocalized( @@ -37,6 +61,9 @@ data class FdroidLocalized( val summary: String? = null, val description: String? = null, val icon: String? = null, + val featureGraphic: String? = null, + val phoneScreenshots: List = emptyList(), + val whatsNew: String? = null, ) data class FdroidApk( @@ -49,32 +76,184 @@ data class FdroidApk( val minSdkVersion: Int? = null, val maxSdkVersion: Int? = null, val targetSdkVersion: Int? = null, + val permissions: List = emptyList(), + val nativecode: List = emptyList(), + val added: Long? = null, + val whatsNew: String? = null, ) -/** Streaming parser for the official F-Droid index-v1 wire format. */ +/** + * Streaming parser for the official F-Droid `index-v1.jar` / `index-v1.json` + * and `index-v2.json` wire formats. + */ object FdroidIndexParser { + /** + * Parses a `.jar` / `.zip` archive (extracting `index-v1.json` or + * `index-v2.json` via [ZipFile]) or a plain `.json` file from disk. + */ + fun parseFile( + file: File, + sdkInt: Int, + supportedAbis: Set = emptySet(), + ): FdroidIndex { + if (isZipFile(file)) { + ZipFile(file).use { zip -> + val entry = zip.getEntry("index-v1.json") + ?: zip.getEntry("index-v2.json") + ?: zip.entries().asSequence().firstOrNull { + !it.isDirectory && it.name.endsWith(".json", ignoreCase = true) && + !it.name.endsWith("entry.json", ignoreCase = true) + } + ?: error("F-Droid archive ${file.name} does not contain index-v1.json or index-v2.json.") + zip.getInputStream(entry).source().buffer().use { buffered -> + return parse(JsonReader.of(buffered), sdkInt, supportedAbis) + } + } + } + file.source().buffer().use { buffered -> + return parseSource(buffered, sdkInt, supportedAbis) + } + } + + /** + * Automatically detects whether [source] is a `.jar` / `.zip` stream + * (`PK\x03\x04`), a `.gz` stream (`\x1f\x8b`), or raw JSON, and streams + * the inner JSON index through [parse]. + */ + fun parseSource( + source: BufferedSource, + sdkInt: Int, + supportedAbis: Set = emptySet(), + ): FdroidIndex { + val peek = source.peek() + if (peek.request(4)) { + val b0 = peek.readByte().toInt() and 0xFF + val b1 = peek.readByte().toInt() and 0xFF + val b2 = peek.readByte().toInt() and 0xFF + val b3 = peek.readByte().toInt() and 0xFF + // ZIP / JAR magic header: PK\x03\x04 + if (b0 == 0x50 && b1 == 0x4B && b2 == 0x03 && b3 == 0x04) { + val zis = ZipInputStream(source.inputStream()) + while (true) { + val entry = zis.nextEntry ?: break + if (!entry.isDirectory && + (entry.name == "index-v1.json" || + entry.name == "index-v2.json" || + (entry.name.endsWith(".json", ignoreCase = true) && + !entry.name.endsWith("entry.json", ignoreCase = true))) + ) { + return parse(JsonReader.of(zis.source().buffer()), sdkInt, supportedAbis) + } + } + error("F-Droid JAR stream did not contain index-v1.json or index-v2.json.") + } + // GZIP magic header: 0x1F 0x8B + if (b0 == 0x1F && b1 == 0x8B) { + GzipSource(source).buffer().use { unzipped -> + return parseSource(unzipped, sdkInt, supportedAbis) + } + } + } + return parse(JsonReader.of(source), sdkInt, supportedAbis) + } + /** Reads the large index without materialising its full JSON tree. */ - fun parse(reader: JsonReader, sdkInt: Int): FdroidIndex { + fun parse( + reader: JsonReader, + sdkInt: Int, + supportedAbis: Set = emptySet(), + ): FdroidIndex { val apps = LinkedHashMap() + val rawPackageCandidates = LinkedHashMap>() val compatibleApks = LinkedHashMap() + check(reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + "Expected JSON object at root of F-Droid index, found ${reader.peek()}" + } reader.beginObject() while (reader.hasNext()) { when (reader.nextName()) { "apps" -> readApps(reader, apps) - "packages" -> readPackages(reader, sdkInt, compatibleApks) + "packages" -> readPackages(reader, apps, rawPackageCandidates) else -> reader.skipValue() } } reader.endObject() + for ((packageName, candidates) in rawPackageCandidates) { + val suggestedCode = apps[packageName]?.suggestedVersionCode + selectBestApk(candidates, sdkInt, supportedAbis, suggestedCode)?.let { best -> + compatibleApks[packageName] = best + } + } + check(apps.isNotEmpty()) { "F-Droid index did not contain any app metadata (expected an 'apps' array)." } return FdroidIndex(apps.values.toList(), compatibleApks) } + private fun isZipFile(file: File): Boolean { + if (!file.exists() || file.length() < 4L) return false + return runCatching { + RandomAccessFile(file, "r").use { raf -> + val b0 = raf.read() + val b1 = raf.read() + val b2 = raf.read() + val b3 = raf.read() + b0 == 0x50 && b1 == 0x4B && b2 == 0x03 && b3 == 0x04 + } + }.getOrDefault(false) + } + + internal fun selectBestApk( + candidates: List, + sdkInt: Int, + supportedAbis: Set, + suggestedVersionCode: Long?, + ): FdroidApk? { + val sdkCompatible = candidates.filter { apk -> + val minSdk = apk.minSdkVersion ?: 1 + val maxSdk = apk.maxSdkVersion + !apk.apkName.isNullOrBlank() && + minSdk <= sdkInt && + (maxSdk == null || maxSdk <= 0 || maxSdk >= sdkInt) + } + if (sdkCompatible.isEmpty()) return null + + val abiOrder = supportedAbis.toList() + val abiFiltered = if (abiOrder.isEmpty()) { + sdkCompatible + } else { + sdkCompatible.filter { apk -> + apk.nativecode.isEmpty() || apk.nativecode.any { it in supportedAbis } + }.ifEmpty { sdkCompatible } + } + + fun abiScore(apk: FdroidApk): Int { + if (abiOrder.isEmpty() || apk.nativecode.isEmpty()) return 1 + var best = 0 + for (abi in apk.nativecode) { + val idx = abiOrder.indexOf(abi) + if (idx >= 0) { + val score = (abiOrder.size - idx) + 1 + if (score > best) best = score + } + } + return best + } + + return abiFiltered.maxWithOrNull( + compareBy { it.versionCode ?: 0L } + .thenBy { apk -> + if (suggestedVersionCode != null && apk.versionCode == suggestedVersionCode) 1 else 0 + } + .thenBy { apk -> abiScore(apk) } + .thenBy { apk -> if (apk.nativecode.isEmpty()) 1 else 0 }, + ) + } + private fun readApps(reader: JsonReader, apps: MutableMap) { check(reader.peek() == JsonReader.Token.BEGIN_ARRAY) { "F-Droid index 'apps' field must be a JSON array." @@ -98,10 +277,16 @@ object FdroidIndexParser { var description: String? = null var icon: String? = null var website: String? = null + var sourceCode: String? = null + var license: String? = null var authorName: String? = null var authorEmail: String? = null var categories: List = emptyList() var localized: FdroidLocalized? = null + var localizedLocale = "en-US" + var suggestedVersionCode: Long? = null + var added: Long? = null + var lastUpdated: Long? = null reader.beginObject() while (reader.hasNext()) { @@ -112,10 +297,21 @@ object FdroidIndexParser { "description" -> description = reader.readStringValue() "icon" -> icon = reader.readStringValue() "webSite" -> website = reader.readStringValue() + "sourceCode" -> sourceCode = reader.readStringValue() + "license" -> license = reader.readStringValue() "authorName" -> authorName = reader.readStringValue() "authorEmail" -> authorEmail = reader.readStringValue() "categories" -> categories = readStringArray(reader) - "localized" -> localized = readLocalized(reader) + "suggestedVersionCode" -> suggestedVersionCode = reader.readLongValue() + "added" -> added = reader.readLongValue() + "lastUpdated" -> lastUpdated = reader.readLongValue() + "localized" -> { + val pair = readLocalized(reader) + if (pair != null) { + localizedLocale = pair.first + localized = pair.second + } + } else -> reader.skipValue() } } @@ -130,36 +326,106 @@ object FdroidIndexParser { description = description, icon = icon, webSite = website, + sourceCode = sourceCode, + license = license, authorName = authorName, authorEmail = authorEmail, categories = categories, localized = localized, + localizedLocale = localizedLocale, + suggestedVersionCode = suggestedVersionCode, + added = added, + lastUpdated = lastUpdated, ) } - private fun readLocalized(reader: JsonReader): FdroidLocalized? { + private fun readLocalized(reader: JsonReader): Pair? { if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { reader.skipValue() return null } - var english: FdroidLocalized? = null - var fallback: FdroidLocalized? = null + val byLocale = LinkedHashMap() reader.beginObject() while (reader.hasNext()) { val locale = reader.nextName() val value = readLocalizedEntry(reader) ?: continue - if (locale.equals("en-US", ignoreCase = true)) { - english = value - } else if (fallback == null && value.hasMetadata()) { - fallback = value + if (value.hasMetadata()) { + byLocale[locale] = value } } reader.endObject() - // Prefer each English field, falling back to another translation when - // the English localization is incomplete. - return mergeLocalized(english, fallback) + if (byLocale.isEmpty()) return null + + val preferredLocales = listOf("en-US", "en", "en-GB", "en_US", "en-AU", "en-CA") + val primaryLocale = byLocale.keys.firstOrNull { key -> + preferredLocales.any { it.equals(key, ignoreCase = true) } + } ?: byLocale.keys.first() + + val ordered = buildList { + for (pref in preferredLocales) { + byLocale.entries.firstOrNull { it.key.equals(pref, ignoreCase = true) }?.let { + if (it.toPair() !in this) add(it.toPair()) + } + } + for ((loc, entry) in byLocale) { + val p = loc to entry + if (p !in this) add(p) + } + } + + fun pickString(selector: (FdroidLocalized) -> String?): Pair? { + for ((loc, entry) in ordered) { + val v = selector(entry)?.takeIf { it.isNotBlank() } + if (v != null) return loc to v + } + return null + } + + val namePair = pickString { it.name } + val summaryPair = pickString { it.summary } + val descPair = pickString { it.description } + val whatsNewPair = pickString { it.whatsNew } + val iconPair = pickString { it.icon } + val fgPair = pickString { it.featureGraphic } + val shotsPair = ordered.firstNotNullOfOrNull { (loc, entry) -> + entry.phoneScreenshots.takeIf { it.isNotEmpty() }?.let { loc to it } + } + + val resolvedIcon = iconPair?.let { (loc, iconName) -> + if (loc.equals(primaryLocale, ignoreCase = true) || iconName.contains("/") || iconName.startsWith("/")) { + iconName + } else { + "$loc/$iconName" + } + } + val resolvedFg = fgPair?.let { (loc, fgName) -> + if (loc.equals(primaryLocale, ignoreCase = true) || fgName.contains("/") || fgName.startsWith("/")) { + fgName + } else { + "$loc/$fgName" + } + } + val resolvedShots = shotsPair?.let { (loc, shots) -> + if (loc.equals(primaryLocale, ignoreCase = true)) { + shots + } else { + shots.map { s -> + if (s.contains("/") || s.startsWith("/")) s else "$loc/phoneScreenshots/$s" + } + } + }.orEmpty() + + return primaryLocale to FdroidLocalized( + name = namePair?.second, + summary = summaryPair?.second, + description = descPair?.second, + icon = resolvedIcon, + featureGraphic = resolvedFg, + phoneScreenshots = resolvedShots, + whatsNew = whatsNewPair?.second, + ) } private fun readLocalizedEntry(reader: JsonReader): FdroidLocalized? { @@ -171,6 +437,10 @@ object FdroidIndexParser { var summary: String? = null var description: String? = null var icon: String? = null + var featureGraphic: String? = null + val phoneScreenshots = ArrayList() + val tabletScreenshots = ArrayList() + var whatsNew: String? = null reader.beginObject() while (reader.hasNext()) { when (reader.nextName()) { @@ -178,31 +448,39 @@ object FdroidIndexParser { "summary" -> summary = reader.readStringValue() "description" -> description = reader.readStringValue() "icon" -> icon = reader.readStringValue() + "featureGraphic" -> featureGraphic = reader.readStringValue() + "phoneScreenshots" -> phoneScreenshots.addAll(readStringArray(reader)) + "sevenInchScreenshots" -> { + readStringArray(reader).forEach { s -> + tabletScreenshots += if (s.contains("/")) s else "sevenInchScreenshots/$s" + } + } + "tenInchScreenshots" -> { + readStringArray(reader).forEach { s -> + tabletScreenshots += if (s.contains("/")) s else "tenInchScreenshots/$s" + } + } + "whatsNew" -> whatsNew = reader.readStringValue() else -> reader.skipValue() } } reader.endObject() - return FdroidLocalized(name, summary, description, icon) + return FdroidLocalized( + name = name, + summary = summary, + description = description, + icon = icon, + featureGraphic = featureGraphic, + phoneScreenshots = phoneScreenshots.ifEmpty { tabletScreenshots }, + whatsNew = whatsNew, + ) } private fun FdroidLocalized.hasMetadata(): Boolean = !name.isNullOrBlank() || !summary.isNullOrBlank() || - !description.isNullOrBlank() || !icon.isNullOrBlank() - - private fun mergeLocalized( - preferred: FdroidLocalized?, - fallback: FdroidLocalized?, - ): FdroidLocalized? { - if (preferred == null) return fallback - if (fallback == null) return preferred - fun choose(first: String?, second: String?): String? = first?.takeIf { it.isNotBlank() } ?: second - return FdroidLocalized( - name = choose(preferred.name, fallback.name), - summary = choose(preferred.summary, fallback.summary), - description = choose(preferred.description, fallback.description), - icon = choose(preferred.icon, fallback.icon), - ) - } + !description.isNullOrBlank() || !icon.isNullOrBlank() || + !featureGraphic.isNullOrBlank() || phoneScreenshots.isNotEmpty() || + !whatsNew.isNullOrBlank() private fun readStringArray(reader: JsonReader): List { if (reader.peek() != JsonReader.Token.BEGIN_ARRAY) { @@ -220,8 +498,8 @@ object FdroidIndexParser { private fun readPackages( reader: JsonReader, - sdkInt: Int, - compatibleApks: MutableMap, + apps: MutableMap, + rawPackages: MutableMap>, ) { if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { reader.skipValue() @@ -230,30 +508,36 @@ object FdroidIndexParser { reader.beginObject() while (reader.hasNext()) { val packageName = reader.nextName() - readPackageVersions(reader, sdkInt)?.let { compatibleApks[packageName] = it } + when (reader.peek()) { + JsonReader.Token.BEGIN_ARRAY -> { + val versions = readPackageVersionsV1(reader) + if (versions.isNotEmpty()) { + rawPackages[packageName] = versions + } + } + JsonReader.Token.BEGIN_OBJECT -> { + readPackageEntryV2(reader, packageName, apps, rawPackages) + } + else -> reader.skipValue() + } } reader.endObject() } - private fun readPackageVersions(reader: JsonReader, sdkInt: Int): FdroidApk? { - if (reader.peek() != JsonReader.Token.BEGIN_ARRAY) { - reader.skipValue() - return null - } - var best: FdroidApk? = null + private fun readPackageVersionsV1(reader: JsonReader): List { + val list = ArrayList(4) reader.beginArray() while (reader.hasNext()) { - val apk = readApk(reader) ?: continue - val minSdk = apk.minSdkVersion ?: 1 - val maxSdk = apk.maxSdkVersion - if (minSdk > sdkInt || (maxSdk != null && maxSdk < sdkInt) || apk.apkName.isNullOrBlank()) continue - if (best == null || (apk.versionCode ?: 0L) > (best.versionCode ?: 0L)) best = apk + val apk = readApkV1(reader) ?: continue + if (!apk.apkName.isNullOrBlank()) { + list += apk + } } reader.endArray() - return best + return list } - private fun readApk(reader: JsonReader): FdroidApk? { + private fun readApkV1(reader: JsonReader): FdroidApk? { if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { reader.skipValue() return null @@ -267,6 +551,9 @@ object FdroidIndexParser { var minSdkVersion: Int? = null var maxSdkVersion: Int? = null var targetSdkVersion: Int? = null + var added: Long? = null + val permissions = LinkedHashSet() + val nativecode = ArrayList(4) reader.beginObject() while (reader.hasNext()) { when (reader.nextName()) { @@ -279,6 +566,9 @@ object FdroidIndexParser { "minSdkVersion" -> minSdkVersion = reader.readLongValue()?.toIntOrNull() "maxSdkVersion" -> maxSdkVersion = reader.readLongValue()?.toIntOrNull() "targetSdkVersion" -> targetSdkVersion = reader.readLongValue()?.toIntOrNull() + "added" -> added = reader.readLongValue() + "nativecode" -> nativecode.addAll(readStringArray(reader)) + "uses-permission", "uses-permission-sdk-23" -> readPermissionsV1(reader, permissions) else -> reader.skipValue() } } @@ -293,15 +583,377 @@ object FdroidIndexParser { minSdkVersion = minSdkVersion, maxSdkVersion = maxSdkVersion, targetSdkVersion = targetSdkVersion, + permissions = permissions.toList(), + nativecode = nativecode, + added = added, ) } + private fun readPermissionsV1(reader: JsonReader, into: MutableSet) { + if (reader.peek() != JsonReader.Token.BEGIN_ARRAY) { + reader.skipValue() + return + } + reader.beginArray() + while (reader.hasNext()) { + when (reader.peek()) { + JsonReader.Token.BEGIN_ARRAY -> { + reader.beginArray() + val perm = if (reader.hasNext()) reader.readStringValue() else null + while (reader.hasNext()) reader.skipValue() + reader.endArray() + if (!perm.isNullOrBlank()) into += perm + } + JsonReader.Token.STRING -> { + val perm = reader.readStringValue() + if (!perm.isNullOrBlank()) into += perm + } + else -> reader.skipValue() + } + } + reader.endArray() + } + + /** + * Parses a package entry from F-Droid `index-v2.json` (`packages[pkg] = { metadata, versions }`). + */ + private fun readPackageEntryV2( + reader: JsonReader, + packageName: String, + apps: MutableMap, + rawPackages: MutableMap>, + ) { + reader.beginObject() + var parsedMeta: FdroidApp? = null + val versions = ArrayList(4) + while (reader.hasNext()) { + when (reader.nextName()) { + "metadata" -> parsedMeta = readMetadataV2(reader, packageName) + "versions" -> readVersionsMapV2(reader, versions) + else -> reader.skipValue() + } + } + reader.endObject() + + if (parsedMeta != null) { + val latestWhatsNew = versions.maxByOrNull { it.versionCode ?: 0L }?.whatsNew + val updatedMeta = if (!latestWhatsNew.isNullOrBlank() && parsedMeta.localized?.whatsNew.isNullOrBlank()) { + val baseLoc = parsedMeta.localized ?: FdroidLocalized() + parsedMeta.copy(localized = baseLoc.copy(whatsNew = latestWhatsNew)) + } else { + parsedMeta + } + apps[packageName] = updatedMeta + } + if (versions.isNotEmpty()) { + rawPackages[packageName] = versions + } + } + + private fun readMetadataV2(reader: JsonReader, packageName: String): FdroidApp? { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return null + } + var name: String? = null + var summary: String? = null + var description: String? = null + var icon: String? = null + var featureGraphic: String? = null + var authorName: String? = null + var authorEmail: String? = null + var webSite: String? = null + var sourceCode: String? = null + var license: String? = null + var categories: List = emptyList() + var added: Long? = null + var lastUpdated: Long? = null + val screenshots = ArrayList() + + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "name" -> name = reader.readLocalizedStringV2() + "summary" -> summary = reader.readLocalizedStringV2() + "description" -> description = reader.readLocalizedStringV2() + "icon" -> icon = readLocalizedFileV2(reader) + "featureGraphic" -> featureGraphic = readLocalizedFileV2(reader) + "authorName" -> authorName = reader.readStringValue() + "authorEmail" -> authorEmail = reader.readStringValue() + "webSite" -> webSite = reader.readStringValue() + "sourceCode" -> sourceCode = reader.readStringValue() + "license" -> license = reader.readStringValue() + "categories" -> categories = readStringArray(reader) + "added" -> added = reader.readLongValue() + "lastUpdated" -> lastUpdated = reader.readLongValue() + "screenshots" -> readScreenshotsV2(reader, screenshots) + else -> reader.skipValue() + } + } + reader.endObject() + + return FdroidApp( + packageName = packageName, + name = name, + summary = summary, + description = description, + icon = icon, + webSite = webSite, + sourceCode = sourceCode, + license = license, + authorName = authorName, + authorEmail = authorEmail, + categories = categories, + localized = FdroidLocalized( + name = name, + summary = summary, + description = description, + icon = icon, + featureGraphic = featureGraphic, + phoneScreenshots = screenshots, + ), + added = added, + lastUpdated = lastUpdated, + ) + } + + private fun readLocalizedFileV2(reader: JsonReader): String? { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return null + } + val byLocale = LinkedHashMap() + reader.beginObject() + while (reader.hasNext()) { + val locale = reader.nextName() + val filePath = readFileObjectPathV2(reader) + if (!filePath.isNullOrBlank()) { + byLocale[locale] = filePath + } + } + reader.endObject() + val preferred = listOf("en-US", "en", "en-GB", "en_US") + return preferred.firstNotNullOfOrNull { byLocale[it] } ?: byLocale.values.firstOrNull() + } + + private fun readFileObjectPathV2(reader: JsonReader): String? { + if (reader.peek() == JsonReader.Token.STRING) return reader.readStringValue() + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return null + } + var name: String? = null + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "name" -> name = reader.readStringValue() + else -> reader.skipValue() + } + } + reader.endObject() + return name + } + + private fun readScreenshotsV2(reader: JsonReader, into: MutableList) { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return + } + val phoneList = ArrayList() + val otherList = ArrayList() + reader.beginObject() + while (reader.hasNext()) { + val deviceType = reader.nextName() + val target = if (deviceType == "phone") phoneList else otherList + if (reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + val byLocale = LinkedHashMap>() + reader.beginObject() + while (reader.hasNext()) { + val locale = reader.nextName() + if (reader.peek() == JsonReader.Token.BEGIN_ARRAY) { + val files = ArrayList() + reader.beginArray() + while (reader.hasNext()) { + readFileObjectPathV2(reader)?.let(files::add) + } + reader.endArray() + if (files.isNotEmpty()) byLocale[locale] = files + } else { + reader.skipValue() + } + } + reader.endObject() + val preferred = listOf("en-US", "en", "en-GB", "en_US") + val chosen = preferred.firstNotNullOfOrNull { byLocale[it] } ?: byLocale.values.firstOrNull() + if (chosen != null) target.addAll(chosen) + } else { + reader.skipValue() + } + } + reader.endObject() + into.addAll(phoneList.ifEmpty { otherList }) + } + + private fun readVersionsMapV2(reader: JsonReader, into: MutableList) { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return + } + reader.beginObject() + while (reader.hasNext()) { + reader.nextName() + readApkV2(reader)?.let(into::add) + } + reader.endObject() + } + + private fun readApkV2(reader: JsonReader): FdroidApk? { + if (reader.peek() != JsonReader.Token.BEGIN_OBJECT) { + reader.skipValue() + return null + } + var added: Long? = null + var apkName: String? = null + var sha256: String? = null + var size: Long? = null + var versionName: String? = null + var versionCode: Long? = null + var minSdk: Int? = null + var targetSdk: Int? = null + var maxSdk: Int? = null + var whatsNew: String? = null + val permissions = LinkedHashSet() + val nativecode = ArrayList(4) + + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "added" -> added = reader.readLongValue() + "whatsNew" -> whatsNew = reader.readLocalizedStringV2() + "file" -> { + if (reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "name" -> apkName = reader.readStringValue()?.removePrefix("/") + "sha256" -> sha256 = reader.readStringValue() + "size" -> size = reader.readLongValue() + else -> reader.skipValue() + } + } + reader.endObject() + } else { + reader.skipValue() + } + } + "manifest" -> { + if (reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "versionName" -> versionName = reader.readStringValue() + "versionCode" -> versionCode = reader.readLongValue() + "maxSdkVersion" -> maxSdk = reader.readLongValue()?.toIntOrNull() + "nativecode" -> nativecode.addAll(readStringArray(reader)) + "usesSdk" -> { + if (reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "minSdkVersion" -> minSdk = reader.readLongValue()?.toIntOrNull() + "targetSdkVersion" -> targetSdk = reader.readLongValue()?.toIntOrNull() + else -> reader.skipValue() + } + } + reader.endObject() + } else { + reader.skipValue() + } + } + "usesPermission", "usesPermissionSdk23" -> readPermissionsV2(reader, permissions) + else -> reader.skipValue() + } + } + reader.endObject() + } else { + reader.skipValue() + } + } + else -> reader.skipValue() + } + } + reader.endObject() + + if (apkName.isNullOrBlank() || versionCode == null) return null + return FdroidApk( + apkName = apkName, + versionCode = versionCode, + versionName = versionName, + size = size, + hash = sha256, + hashType = "sha256", + minSdkVersion = minSdk, + maxSdkVersion = maxSdk, + targetSdkVersion = targetSdk, + permissions = permissions.toList(), + nativecode = nativecode, + added = added, + whatsNew = whatsNew, + ) + } + + private fun readPermissionsV2(reader: JsonReader, into: MutableSet) { + if (reader.peek() != JsonReader.Token.BEGIN_ARRAY) { + reader.skipValue() + return + } + reader.beginArray() + while (reader.hasNext()) { + if (reader.peek() == JsonReader.Token.BEGIN_OBJECT) { + var perm: String? = null + reader.beginObject() + while (reader.hasNext()) { + when (reader.nextName()) { + "name" -> perm = reader.readStringValue() + else -> reader.skipValue() + } + } + reader.endObject() + if (!perm.isNullOrBlank()) into += perm + } else { + reader.skipValue() + } + } + reader.endArray() + } + + private fun JsonReader.readLocalizedStringV2(): String? = when (peek()) { + JsonReader.Token.STRING, JsonReader.Token.NUMBER -> nextString() + JsonReader.Token.BEGIN_OBJECT -> { + val byLocale = LinkedHashMap() + beginObject() + while (hasNext()) { + val loc = nextName() + val v = readStringValue() + if (!v.isNullOrBlank()) byLocale[loc] = v + } + endObject() + val preferred = listOf("en-US", "en", "en-GB", "en_US") + preferred.firstNotNullOfOrNull { byLocale[it] } ?: byLocale.values.firstOrNull() + } + else -> { + skipValue() + null + } + } + private fun JsonReader.readStringValue(): String? = when (peek()) { JsonReader.Token.NULL -> { nextNull() null } JsonReader.Token.STRING, JsonReader.Token.NUMBER -> nextString() + JsonReader.Token.BOOLEAN -> nextBoolean().toString() else -> { skipValue() null @@ -313,7 +965,9 @@ object FdroidIndexParser { nextNull() null } - JsonReader.Token.NUMBER -> nextLong() + JsonReader.Token.NUMBER -> runCatching { nextLong() }.getOrElse { + runCatching { nextDouble().toLong() }.getOrNull() + } JsonReader.Token.STRING -> nextString().toLongOrNull() else -> { skipValue() @@ -349,7 +1003,10 @@ object FdroidCategories { "theming" to "tools", "privacy" to "tools", "security" to "tools", + "navigation" to "tools", "navigation & maps" to "tools", + "automation" to "tools", + "xposed" to "tools", "weather" to "tools", "remote" to "tools", "tethering" to "tools", @@ -357,6 +1014,10 @@ object FdroidCategories { "quick settings" to "tools", "screen lock" to "tools", "medical" to "tools", + "writing" to "productivity", + "reading" to "productivity", + "office" to "productivity", + "time" to "productivity", "writing & notes" to "productivity", "text & notes" to "productivity", "productivity" to "productivity", @@ -366,6 +1027,7 @@ object FdroidCategories { "task" to "productivity", "science & education" to "education", "education" to "education", + "religion" to "education", "book" to "education", "reference" to "education", "kids" to "education", @@ -373,6 +1035,7 @@ object FdroidCategories { "music & audio" to "music", "audio" to "music", "video" to "entertainment", + "graphics" to "photo", "photography" to "photo", "images" to "photo", "wallpapers" to "photo", @@ -383,6 +1046,7 @@ object FdroidCategories { "news & magazines" to "entertainment", "readers & news" to "entertainment", "sports & health" to "entertainment", + "food" to "entertainment", "money" to "finance", "finance" to "finance", ) @@ -407,37 +1071,148 @@ object FdroidCategories { /** F-Droid keeps games in a category, not in a kind. */ fun isGame(categories: List): Boolean = categories.any { it.trim().contains("game", ignoreCase = true) } + + /** + * Collects all normalized category slugs and aliases for [categories] so + * that every category tile on Home and Categories (e.g. `games`, + * `photography`, `books`, `health`, `video`, `news`, `social`, `maps`, + * `food`, `communication`, `tools`) matches the relevant F-Droid apps. + */ + fun tagsFor(categories: List): List { + if (categories.isEmpty()) return listOf(DEFAULT_SLUG) + val out = LinkedHashSet() + for (raw in categories) { + val trimmed = raw.trim() + if (trimmed.isEmpty()) continue + out += trimmed + val lower = trimmed.lowercase() + slug(trimmed)?.let { out += it } + when (lower) { + "games" -> out.addAll(listOf("games", "game", "entertainment")) + "internet" -> out.addAll(listOf("browser", "communication", "social", "news", "internet")) + "connectivity" -> out.addAll(listOf("tools", "communication", "connectivity")) + "messaging", "phone & sms" -> out.addAll(listOf("communication", "social", "messaging")) + "multimedia" -> out.addAll(listOf("music", "video", "entertainment", "photography", "multimedia")) + "graphics", "photography" -> out.addAll(listOf("photo", "photography", "graphics")) + "reading", "writing" -> out.addAll(listOf("productivity", "books", "news", "education", lower)) + "office", "time" -> out.addAll(listOf("productivity", "business", "tools", lower)) + "science & education", "religion" -> out.addAll(listOf("education", "books")) + "money" -> out.addAll(listOf("finance", "shopping", "business", "money")) + "sports & health" -> out.addAll(listOf("health", "lifestyle", "entertainment")) + "food" -> out.addAll(listOf("food", "lifestyle", "health", "entertainment")) + "navigation", "navigation & maps" -> out.addAll(listOf("maps", "travel", "navigation", "tools")) + "security", "system", "development", "theming", "automation", "xposed" -> { + out.addAll(listOf("tools", "security", lower)) + } + else -> { + if (lower.contains("game")) out.addAll(listOf("games", "game", "entertainment")) + } + } + } + if (out.isEmpty()) out += DEFAULT_SLUG + return out.toList() + } +} + +internal object FdroidDates { + private val isoFormat: ThreadLocal = object : ThreadLocal() { + override fun initialValue(): SimpleDateFormat = + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + } + + fun toIso(epochMillis: Long?): String { + if (epochMillis == null || epochMillis <= 0L) return "" + return isoFormat.get()?.format(Date(epochMillis)).orEmpty() + } +} + +internal object FdroidHtml { + private val brTagRegex = Regex("(?i)<\\s*br\\s*/?\\s*>") + private val blockEndRegex = Regex("(?i)") + private val liStartRegex = Regex("(?i)<\\s*li[^>]*>") + private val anyTagRegex = Regex("<[^>]+>") + private val multiBlankLines = Regex("\n{3,}") + private val multiSpaces = Regex("[ \t]{2,}") + + fun clean(raw: String): String { + if (raw.isEmpty()) return "" + if (!raw.contains('<') && !raw.contains('&')) return raw.trim() + var s = raw + .replace(brTagRegex, "\n") + .replace(blockEndRegex, "\n\n") + .replace(liStartRegex, "• ") + .replace(anyTagRegex, "") + s = s + .replace(" ", " ") + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("'", "'") + .replace("—", "—") + .replace("–", "–") + .replace("…", "…") + s = s.replace(multiSpaces, " ") + s = s.lines().joinToString("\n") { it.trim() } + s = s.replace(multiBlankLines, "\n\n") + return s.trim() + } } /** Converts the streamed index into the store's models. */ fun FdroidIndex.toCatalog(repoUrl: String): CatalogResponse { + val base = repoUrl.trimEnd('/') val releasesByPackage = packages val listings = ArrayList(apps.size) val releases = ArrayList(apps.size) - for (meta in apps) { + // Sort apps by most recently updated first (matching Droid-ify SortOrder.UPDATED) + // so actively maintained apps surface at the top of shelves instead of + // abandoned apps whose names start with digits or 'A'. + val sortedApps = apps.sortedByDescending { meta -> + val apk = releasesByPackage[meta.packageName] + maxOf(meta.lastUpdated ?: 0L, apk?.added ?: 0L, meta.added ?: 0L) + } + + for (meta in sortedApps) { val packageName = meta.packageName val apk = releasesByPackage[packageName] ?: continue - val apkName = apk.apkName?.trim().orEmpty() + val apkName = apk.apkName?.trim()?.removePrefix("/").orEmpty() if (apkName.isBlank()) continue val versionCode = apk.versionCode ?: 0L val localized = meta.localized + val locale = meta.localizedLocale.ifBlank { "en-US" } val name = localized?.name?.trim().orEmpty() .ifBlank { meta.name?.trim().orEmpty() } .ifBlank { packageName.substringAfterLast('.') } - val summary = localized?.summary?.trim().orEmpty() + val rawSummary = localized?.summary?.trim().orEmpty() .ifBlank { meta.summary?.trim().orEmpty() } .ifBlank { localized?.description?.trim().orEmpty() } .ifBlank { meta.description?.trim().orEmpty() } - val description = localized?.description?.trim().orEmpty() + val rawDescription = localized?.description?.trim().orEmpty() .ifBlank { meta.description?.trim().orEmpty() } - .ifBlank { summary } + .ifBlank { rawSummary } + val summary = FdroidHtml.clean(rawSummary) + val description = FdroidHtml.clean(rawDescription).ifBlank { summary } val categories = meta.categories.mapNotNull { category -> category.trim().takeIf { it.isNotEmpty() } } val id = "fdroid:$packageName" + val screenshots = localized?.phoneScreenshots + ?.mapNotNull { shot -> fdroidScreenshotUrl(packageName, locale, shot, base) } + .orEmpty() + val featureGraphicUrl = fdroidFeatureGraphicUrl(meta, packageName, locale, base) + val createdAtIso = FdroidDates.toIso(meta.added ?: apk.added) + val updatedAtIso = FdroidDates.toIso(meta.lastUpdated ?: apk.added ?: meta.added) + val changelog = localized?.whatsNew?.trim()?.takeIf { it.isNotEmpty() } + ?.let(FdroidHtml::clean) + .orEmpty() listings += StoreApp( id = id, @@ -445,14 +1220,23 @@ fun FdroidIndex.toCatalog(repoUrl: String): CatalogResponse { name = name, developerName = meta.authorName?.trim().orEmpty() .ifBlank { meta.authorEmail?.trim().orEmpty() } + .ifBlank { meta.license?.trim()?.takeIf { it.isNotEmpty() }?.let { "Open-source ($it)" }.orEmpty() } .ifBlank { "Open-source developer" }, category = FdroidCategories.slugFor(categories), kind = if (FdroidCategories.isGame(categories)) "game" else "app", status = "published", shortDesc = summary, fullDesc = description, - iconUrl = fdroidIconUrl(meta, packageName, repoUrl), - website = meta.webSite?.trim()?.takeIf { it.isNotBlank() }, + iconUrl = fdroidIconUrl(meta, packageName, locale, base), + featureGraphicUrl = featureGraphicUrl, + screenshots = screenshots, + tags = FdroidCategories.tagsFor(categories), + supportEmail = meta.authorEmail?.trim().orEmpty(), + website = meta.webSite?.trim()?.takeIf { it.isNotBlank() } + ?: meta.sourceCode?.trim()?.takeIf { it.isNotBlank() }, + privacyPolicyUrl = meta.sourceCode?.trim()?.takeIf { it.isNotBlank() }, + createdAt = createdAtIso, + updatedAt = updatedAtIso, source = "F-Droid", ) @@ -463,15 +1247,18 @@ fun FdroidIndex.toCatalog(repoUrl: String): CatalogResponse { versionName = apk.versionName?.trim().orEmpty().ifBlank { versionCode.toString() }, apkProvider = "fdroid", apkKey = apkName, - apkUrl = "$repoUrl/$apkName", + apkUrl = "$base/$apkName", apkSizeBytes = apk.size ?: 0L, apkSha256 = apk.hash?.trim().orEmpty().takeIf { it.isNotBlank() && apk.hashType.orEmpty().ifBlank { "sha256" }.equals("sha256", ignoreCase = true) }, minSdk = apk.minSdkVersion ?: 1, targetSdk = apk.targetSdkVersion?.takeIf { it > 0 }, + permissions = apk.permissions, + changelog = changelog, reviewStatus = "approved", isLive = true, + createdAt = FdroidDates.toIso(apk.added ?: meta.lastUpdated ?: meta.added), ) } @@ -479,11 +1266,60 @@ fun FdroidIndex.toCatalog(repoUrl: String): CatalogResponse { } /** - * The index's app icon is the versioned image in icons-640/. The localized - * icon is a fallback hosted under the package's locale directory. + * The index's app icon is the versioned image in `icons-640/`. The localized + * icon is a fallback hosted under the package's locale directory + * (`///`), matching Droid-ify and official + * F-Droid (`AppV1.kt`). */ -private fun fdroidIconUrl(meta: FdroidApp, packageName: String, repoUrl: String): String { - meta.icon?.trim()?.takeIf { it.isNotBlank() }?.let { return "$repoUrl/icons-640/$it" } - meta.localized?.icon?.trim()?.takeIf { it.isNotBlank() }?.let { return "$repoUrl/$packageName/en-US/$it" } - return "" +private fun fdroidIconUrl( + meta: FdroidApp, + packageName: String, + locale: String, + repoUrl: String, +): String { + val topIcon = meta.icon?.trim()?.takeIf { it.isNotBlank() && !it.endsWith(".xml", ignoreCase = true) } + if (topIcon != null) { + return when { + topIcon.startsWith("http://") || topIcon.startsWith("https://") -> topIcon + topIcon.startsWith("/") -> "$repoUrl$topIcon" + else -> "$repoUrl/icons-640/$topIcon" + } + } + val locIcon = meta.localized?.icon?.trim()?.takeIf { it.isNotBlank() } ?: return "" + return when { + locIcon.startsWith("http://") || locIcon.startsWith("https://") -> locIcon + locIcon.startsWith("/") -> "$repoUrl$locIcon" + locIcon.contains("/") -> "$repoUrl/$packageName/$locIcon" + else -> "$repoUrl/$packageName/$locale/$locIcon" + } +} + +private fun fdroidFeatureGraphicUrl( + meta: FdroidApp, + packageName: String, + locale: String, + repoUrl: String, +): String? { + val fg = meta.localized?.featureGraphic?.trim()?.takeIf { it.isNotBlank() } ?: return null + return when { + fg.startsWith("http://") || fg.startsWith("https://") -> fg + fg.startsWith("/") -> "$repoUrl$fg" + fg.contains("/") -> "$repoUrl/$packageName/$fg" + else -> "$repoUrl/$packageName/$locale/$fg" + } +} + +private fun fdroidScreenshotUrl( + packageName: String, + locale: String, + shot: String, + repoUrl: String, +): String? { + val clean = shot.trim().takeIf { it.isNotEmpty() } ?: return null + return when { + clean.startsWith("http://") || clean.startsWith("https://") -> clean + clean.startsWith("/") -> "$repoUrl$clean" + clean.contains("/") -> "$repoUrl/$packageName/$clean" + else -> "$repoUrl/$packageName/$locale/phoneScreenshots/$clean" + } } diff --git a/app/src/main/java/com/gothwad/store/data/remote/StoreRepository.kt b/app/src/main/java/com/gothwad/store/data/remote/StoreRepository.kt index 3425b0a..d4b4dd5 100644 --- a/app/src/main/java/com/gothwad/store/data/remote/StoreRepository.kt +++ b/app/src/main/java/com/gothwad/store/data/remote/StoreRepository.kt @@ -2,8 +2,8 @@ package com.gothwad.store.data.remote import android.util.Log import com.gothwad.store.data.model.ApiErrorBody -import com.gothwad.store.data.model.CatalogResponse import com.gothwad.store.data.model.AppDetailResponse +import com.gothwad.store.data.model.CatalogResponse import com.gothwad.store.data.model.LibraryBody import com.gothwad.store.data.model.MeResponse import com.gothwad.store.data.model.ReportBody @@ -41,13 +41,17 @@ class StoreRepository( */ val fdroidError: StateFlow = fdroid.lastError + /** + * Emits whenever the F-Droid + IzzyOnDroid catalogue is loaded from disk + * cache or updated over the network. + */ + val fdroidUpdates: StateFlow = fdroid.catalogFlow + /** * The Kiwi Store catalogue only. * * Split out from [catalog] so the UI can show first-party listings right - * away and add the F-Droid catalogue once its (large) index has been - * downloaded — waiting for ~8 MB before rendering anything leaves the - * user staring at a spinner. + * away and add the F-Droid catalogue once its index has been downloaded. */ suspend fun firstPartyCatalog(kind: String?, category: String?, query: String): CatalogResponse { val trimmed = query.trim() @@ -73,16 +77,69 @@ class StoreRepository( query: String = "", ): CatalogResponse = combine(base, query.trim(), kind, category) + /** + * Synchronously merges [gothwad] first-party listings with an already-loaded + * [external] F-Droid catalogue, applying [kind], [category], and [query] + * filters and relevance ranking for search. + */ + fun mergeWithExternal( + gothwad: CatalogResponse, + external: CatalogResponse, + kind: String?, + category: String?, + query: String = "", + ): CatalogResponse { + if (external.apps.isEmpty()) return gothwad + val ownedPackages = gothwad.apps.mapTo(mutableSetOf()) { it.packageName } + val q = query.trim().lowercase() + + val baseSequence = external.apps + .asSequence() + .filter { app -> + app.packageName !in ownedPackages && + kindMatches(app, kind) && + categoryMatches(app, category) + } + + val candidates = if (q.isEmpty()) { + baseSequence.take(MAX_FDROID_APPS).toList() + } else { + baseSequence + .mapNotNull { app -> + val score = searchScore(app, q) + if (score > 0) app to score else null + } + .sortedWith( + compareByDescending> { it.second } + .thenByDescending { it.first.updatedAt }, + ) + .map { it.first } + .take(MAX_FDROID_APPS) + .toList() + } + + val ids = candidates.mapTo(mutableSetOf()) { it.id } + return gothwad.copy( + // Distinct by id: the UI uses the id as the list key, so a listing + // that appears twice would crash the list. + apps = (gothwad.apps + candidates).distinctBy { it.id }, + releases = gothwad.releases + external.releases.filter { it.appId in ids }, + ) + } + /** The optional F-Droid catalogue on its own. */ - suspend fun fdroidCatalog(): CatalogResponse = fdroid.catalog() + suspend fun fdroidCatalog(forceRefresh: Boolean = false): CatalogResponse = + fdroid.catalog(forceRefresh) suspend fun catalog(kind: String?, category: String?, query: String): CatalogResponse { - val base = firstPartyCatalog(kind, category, query) + val base = runCatching { firstPartyCatalog(kind, category, query) } + .getOrDefault(CatalogResponse()) return combine(base, query.trim(), kind, category) } suspend fun home(kind: String?): CatalogResponse { - val base = firstPartyHome(kind) + val base = runCatching { firstPartyHome(kind) } + .getOrDefault(CatalogResponse()) return combine(base, "", kind, null) } @@ -112,24 +169,28 @@ class StoreRepository( kind: String?, category: String?, ): CatalogResponse { - // The first-party catalogue remains usable if the optional upstream is unavailable. val external = loadFdroid() ?: return gothwad - val ownedPackages = gothwad.apps.mapTo(mutableSetOf()) { it.packageName } - val q = query.trim().lowercase() - val candidates = external.apps.filter { app -> - app.packageName !in ownedPackages && - kindMatches(app, kind) && - categoryMatches(app, category) && - (q.isEmpty() || app.name.contains(q, true) || app.packageName.contains(q, true) || - app.shortDesc.contains(q, true) || app.developerName.contains(q, true)) - } - val ids = candidates.mapTo(mutableSetOf()) { it.id } - return gothwad.copy( - // Distinct by id: the UI uses the id as the list key, so a listing - // that appears twice would crash the list. - apps = (gothwad.apps + candidates).distinctBy { it.id }, - releases = gothwad.releases + external.releases.filter { it.appId in ids }, - ) + return mergeWithExternal(gothwad, external, kind, category, query) + } + + private fun searchScore(app: StoreApp, needle: String): Int { + val nameLower = app.name.lowercase() + val pkgLower = app.packageName.lowercase() + val shortLower = app.shortDesc.lowercase() + val devLower = app.developerName.lowercase() + + if (nameLower == needle || pkgLower == needle) return 100 + if (nameLower.startsWith(needle)) return 85 + val words = nameLower.split(' ', '-', '_', '.', ':') + if (words.any { it == needle }) return 75 + if (words.any { it.startsWith(needle) }) return 65 + if (nameLower.contains(needle)) return 55 + if (pkgLower.contains(needle)) return 45 + if (app.tags.any { it.lowercase().contains(needle) }) return 35 + if (devLower.contains(needle)) return 30 + if (shortLower.contains(needle)) return 20 + if (app.fullDesc.lowercase().contains(needle)) return 10 + return 0 } /** F-Droid keeps games in a category and everything else in "app". */ @@ -141,11 +202,17 @@ class StoreRepository( else -> true } - /** Both catalogues use Kiwi Store slugs, so a plain comparison is enough. */ - private fun categoryMatches(app: StoreApp, category: String?): Boolean { - val wanted = category?.trim().orEmpty() + /** + * Matches an app against a category slug or display name, checking both + * `app.category`, `app.tags`, and the normalized Kiwi Store / F-Droid + * category equivalence groups. + */ + internal fun categoryMatches(app: StoreApp, category: String?): Boolean { + val wanted = normalizeCategory(category) if (wanted.isEmpty()) return true - return app.category.equals(wanted, ignoreCase = true) + val equivalents = CATEGORY_EQUIVALENTS[wanted] ?: setOf(wanted) + if (normalizeCategory(app.category) in equivalents) return true + return app.tags.any { normalizeCategory(it) in equivalents } } /** @@ -172,9 +239,21 @@ class StoreRepository( } suspend fun categories(): List { - val response = call { api.categories() } - assertConfigured(response.source) - return response.categories.sortedBy { it.name.lowercase() } + val upstream = runCatching { + val response = call { api.categories() } + assertConfigured(response.source) + response.categories + }.getOrDefault(emptyList()) + + val existingSlugs = upstream.mapTo(HashSet()) { normalizeCategory(it.slug) } + val merged = ArrayList(upstream.size + DEFAULT_CATEGORIES.size) + merged.addAll(upstream) + for (fallback in DEFAULT_CATEGORIES) { + if (normalizeCategory(fallback.slug) !in existingSlugs) { + merged += fallback + } + } + return merged.sortedBy { it.name.lowercase() } } suspend fun detail(id: String): AppDetailResponse { @@ -182,12 +261,18 @@ class StoreRepository( return fdroid.find(id.removePrefix("fdroid:")) ?: throw StoreException("This F-Droid app is no longer in its repository index.", StoreException.Kind.NOT_FOUND) } - val response = call { api.detail(id) } - if (response.app.status != "published") throw StoreException( - "This app is no longer available on Kiwi Store.", - StoreException.Kind.NOT_PUBLISHED, - ) - return response.copy(reviews = response.reviews.sortedByDescending { it.createdAt }) + val firstParty = runCatching { + val response = call { api.detail(id) } + if (response.app.status != "published") throw StoreException( + "This app is no longer available on Kiwi Store.", + StoreException.Kind.NOT_PUBLISHED, + ) + response.copy(reviews = response.reviews.sortedByDescending { it.createdAt }) + } + firstParty.getOrNull()?.let { return it } + fdroid.find(id)?.let { return it } + throw (firstParty.exceptionOrNull() + ?: StoreException("This app is no longer available.", StoreException.Kind.NOT_FOUND)) } /** Releases this device could install, newest first. */ @@ -268,8 +353,53 @@ class StoreRepository( } } - private companion object { - const val TAG = "StoreRepository" + companion object { + private const val TAG = "StoreRepository" + private const val MAX_FDROID_APPS = 500 + + private val DEFAULT_CATEGORIES = listOf( + StoreCategory(slug = "tools", name = "Tools & Utilities", kind = "app"), + StoreCategory(slug = "browser", name = "Browsers & Internet", kind = "app"), + StoreCategory(slug = "communication", name = "Communication", kind = "app"), + StoreCategory(slug = "productivity", name = "Productivity", kind = "app"), + StoreCategory(slug = "education", name = "Science & Education", kind = "app"), + StoreCategory(slug = "music", name = "Music & Audio", kind = "app"), + StoreCategory(slug = "photo", name = "Photography & Graphics", kind = "app"), + StoreCategory(slug = "finance", name = "Money & Finance", kind = "app"), + StoreCategory(slug = "health", name = "Sports & Health", kind = "app"), + StoreCategory(slug = "maps", name = "Maps & Navigation", kind = "app"), + StoreCategory(slug = "entertainment", name = "Games & Entertainment", kind = "both"), + ) + + private val CATEGORY_EQUIVALENTS: Map> = mapOf( + "games" to setOf("games", "game", "entertainment"), + "game" to setOf("games", "game", "entertainment"), + "entertainment" to setOf("entertainment", "games", "game", "video", "multimedia"), + "browser" to setOf("browser", "internet", "web", "communication"), + "internet" to setOf("internet", "browser", "web", "communication"), + "communication" to setOf("communication", "messaging", "phone-sms", "internet", "social", "connectivity"), + "social" to setOf("social", "communication", "messaging", "internet"), + "tools" to setOf("tools", "system", "security", "development", "theming", "automation", "connectivity", "navigation", "xposed"), + "security" to setOf("security", "tools", "privacy", "system"), + "productivity" to setOf("productivity", "reading", "writing", "office", "time", "calendar-agenda", "note", "task"), + "books" to setOf("books", "reading", "writing", "education", "science-education", "religion"), + "news" to setOf("news", "reading", "internet", "entertainment", "news-magazines", "readers-news"), + "education" to setOf("education", "science-education", "reading", "books", "religion"), + "music" to setOf("music", "multimedia", "audio", "music-audio"), + "video" to setOf("video", "multimedia", "entertainment", "music"), + "photo" to setOf("photo", "photography", "graphics", "images", "wallpapers"), + "photography" to setOf("photography", "photo", "graphics", "images", "wallpapers"), + "finance" to setOf("finance", "money", "shopping", "business"), + "shopping" to setOf("shopping", "money", "finance"), + "health" to setOf("health", "sports-health", "lifestyle", "food", "entertainment"), + "lifestyle" to setOf("lifestyle", "sports-health", "health", "food"), + "food" to setOf("food", "lifestyle", "health", "entertainment"), + "maps" to setOf("maps", "navigation", "navigation-maps", "travel", "tools"), + "travel" to setOf("travel", "navigation", "maps"), + ) + + private fun normalizeCategory(value: String?): String = + value?.trim()?.lowercase()?.replace(Regex("[^a-z0-9]+"), "-")?.trim('-').orEmpty() } } diff --git a/app/src/main/java/com/gothwad/store/service/ApkDownloadService.kt b/app/src/main/java/com/gothwad/store/service/ApkDownloadService.kt index 5345ccd..8759f38 100644 --- a/app/src/main/java/com/gothwad/store/service/ApkDownloadService.kt +++ b/app/src/main/java/com/gothwad/store/service/ApkDownloadService.kt @@ -14,6 +14,7 @@ import com.gothwad.store.data.download.DownloadStatus import com.gothwad.store.data.download.DownloadTask import com.gothwad.store.data.install.InstallPhase import com.gothwad.store.data.install.InstallRequest +import com.gothwad.store.data.remote.FdroidCatalog import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -160,73 +161,93 @@ class ApkDownloadService : Service() { ) } - val builder = Request.Builder().url(task.url) - if (existing > 0) builder.header("Range", "bytes=$existing-") - val call = AppGraph.http.newCall(builder.build()) - call.execute().use { response -> - if (!response.isSuccessful) { - throw IOException("The Store server answered ${response.code} for this APK.") - } - val body = response.body ?: throw IOException("The Store returned an empty APK response.") - val offset = if (response.code == 206) existing else 0L - val contentLength = body.contentLength() - val total = when { - task.totalBytes > 0 -> task.totalBytes - contentLength > 0 -> offset + contentLength - else -> 0L - } - var written = offset - var lastTick = 0L - var lastBytes = written - var lastTimestamp = System.currentTimeMillis() + val candidateUrls = FdroidCatalog.candidateUrls(task.url) + var lastError: Throwable? = null + var succeeded = false - FileOutputStream(target, offset > 0).use { out -> - body.byteStream().use { input -> - val buffer = ByteArray(128 * 1024) - while (true) { - coroutineContext.ensureActive() - val read = input.read(buffer) - if (read <= 0) break - out.write(buffer, 0, read) - written += read - val now = System.currentTimeMillis() - if (now - lastTick >= 400) { - lastTick = now - val elapsed = (now - lastTimestamp).coerceAtLeast(1L) - val speed = ((written - lastBytes) * 1000L) / elapsed - lastBytes = written - lastTimestamp = now - store.update(task.id, persist = false) { - it.copy(bytesDownloaded = written, totalBytes = total, status = DownloadStatus.RUNNING) - } - if (AppGraph.settings.current.notifyDownloads) { - getSystemService(NotificationManager::class.java)?.notify( - task.id.hashCode(), - DownloadNotifications.progress( - this@ApkDownloadService, - task.copy(bytesDownloaded = written, totalBytes = total), - speed, - ), - ) + for (candidateUrl in candidateUrls) { + coroutineContext.ensureActive() + val currentExisting = if (target.exists()) target.length() else 0L + val builder = Request.Builder().url(candidateUrl) + if (currentExisting > 0) builder.header("Range", "bytes=$currentExisting-") + try { + val call = AppGraph.http.newCall(builder.build()) + call.execute().use { response -> + if (!response.isSuccessful) { + throw IOException("The Store server answered ${response.code} for this APK.") + } + val body = response.body ?: throw IOException("The Store returned an empty APK response.") + val offset = if (response.code == 206) currentExisting else 0L + val contentLength = body.contentLength() + val total = when { + task.totalBytes > 0 -> task.totalBytes + contentLength > 0 -> offset + contentLength + else -> 0L + } + var written = offset + var lastTick = 0L + var lastBytes = written + var lastTimestamp = System.currentTimeMillis() + + FileOutputStream(target, offset > 0).use { out -> + body.byteStream().use { input -> + val buffer = ByteArray(128 * 1024) + while (true) { + coroutineContext.ensureActive() + val read = input.read(buffer) + if (read <= 0) break + out.write(buffer, 0, read) + written += read + val now = System.currentTimeMillis() + if (now - lastTick >= 400) { + lastTick = now + val elapsed = (now - lastTimestamp).coerceAtLeast(1L) + val speed = ((written - lastBytes) * 1000L) / elapsed + lastBytes = written + lastTimestamp = now + store.update(task.id, persist = false) { + it.copy(bytesDownloaded = written, totalBytes = total, status = DownloadStatus.RUNNING) + } + if (AppGraph.settings.current.notifyDownloads) { + getSystemService(NotificationManager::class.java)?.notify( + task.id.hashCode(), + DownloadNotifications.progress( + this@ApkDownloadService, + task.copy(bytesDownloaded = written, totalBytes = total), + speed, + ), + ) + } + } } + out.flush() } } - out.flush() + + val finalBytes = if (total > 0) total else written + val completed = task.copy( + bytesDownloaded = finalBytes, + totalBytes = finalBytes, + status = DownloadStatus.COMPLETED, + filePath = target.absolutePath, + downloadedAt = now(), + updatedAt = now(), + error = null, + ) + store.upsert(completed) + finish(completed, target) + succeeded = true } + if (succeeded) break + } catch (cancellation: CancellationException) { + throw cancellation + } catch (error: Throwable) { + lastError = error } + } - val finalBytes = if (total > 0) total else written - val completed = task.copy( - bytesDownloaded = finalBytes, - totalBytes = finalBytes, - status = DownloadStatus.COMPLETED, - filePath = target.absolutePath, - downloadedAt = now(), - updatedAt = now(), - error = null, - ) - store.upsert(completed) - finish(completed, target) + if (!succeeded) { + throw lastError ?: IOException("The download could not be completed.") } } diff --git a/app/src/main/java/com/gothwad/store/ui/StoreViewModel.kt b/app/src/main/java/com/gothwad/store/ui/StoreViewModel.kt index 3ac0bfd..bdfbe17 100644 --- a/app/src/main/java/com/gothwad/store/ui/StoreViewModel.kt +++ b/app/src/main/java/com/gothwad/store/ui/StoreViewModel.kt @@ -83,7 +83,7 @@ class StoreViewModel : ViewModel() { private val _events = MutableStateFlow(null) val events: StateFlow = _events.asStateFlow() - /** appId → newest release the Worker published for this device. */ + /** appId → newest release the Worker or F-Droid published for this device. */ private val _releases = MutableStateFlow>(emptyMap()) val releases: StateFlow> = _releases.asStateFlow() @@ -92,8 +92,9 @@ class StoreViewModel : ViewModel() { val knownApps: StateFlow> = _knownApps.asStateFlow() /** - * The optional F-Droid catalogue, kept apart from the first-party - * listings so Home can show it as its own shelf. + * The F-Droid catalogue, kept apart from the first-party listings so Home + * can show it as its own dedicated shelf while also merging it into Apps, + * Games, Categories, Top Charts, and Search. */ private val _fdroid = MutableStateFlow(CatalogBundle()) val fdroid: StateFlow = _fdroid.asStateFlow() @@ -104,6 +105,19 @@ class StoreViewModel : ViewModel() { val downloads: StateFlow> = downloader.tasks val installPhases: StateFlow> = installer.phases + private var firstPartyHomeCache: CatalogResponse = CatalogResponse() + private var firstPartyAppsCache: CatalogResponse = CatalogResponse() + private var currentAppsKind: String? = "app" + private var currentAppsCategory: String? = null + private var firstPartyGamesCache: CatalogResponse = CatalogResponse() + private var firstPartyChartsCache: CatalogResponse = CatalogResponse() + private var currentChartsKind: String? = null + private var firstPartyCategoryCache: CatalogResponse = CatalogResponse() + private var currentCategorySlug: String? = null + private var currentCategoryKind: String? = null + private var firstPartySearchCache: CatalogResponse = CatalogResponse() + private var currentSearchQuery: String = "" + private var homeJob: Job? = null private var appsJob: Job? = null private var gamesJob: Job? = null @@ -114,25 +128,108 @@ class StoreViewModel : ViewModel() { private var fdroidJob: Job? = null init { + observeFdroidUpdates() refreshHome() refreshCategories() + loadFdroid() + } + + /** + * Reactively merges F-Droid + IzzyOnDroid catalogue emissions into every + * screen state so that whether F-Droid finishes before or after the + * first-party Worker (or emits progressively as each repo completes), + * every tab automatically shows the combined apps. + */ + private fun observeFdroidUpdates() { + viewModelScope.launch { + repository.fdroidUpdates.collect { external -> + if (external == null || external.apps.isEmpty()) return@collect + val index = repository.releaseIndex(external) + if (index.isNotEmpty()) _releases.value = _releases.value + index + _knownApps.value = _knownApps.value + external.apps.associateBy { it.id } + _fdroid.value = CatalogBundle(external.apps, index) + + val mergedHome = repository.mergeWithExternal( + gothwad = firstPartyHomeCache, + external = external, + kind = null, + category = null, + ) + _home.value = registerAndBuildState(mergedHome) + + val mergedApps = repository.mergeWithExternal( + gothwad = firstPartyAppsCache, + external = external, + kind = currentAppsKind, + category = currentAppsCategory, + ) + _apps.value = registerAndBuildState(mergedApps) + + val mergedGames = repository.mergeWithExternal( + gothwad = firstPartyGamesCache, + external = external, + kind = "game", + category = null, + ) + _games.value = registerAndBuildState(mergedGames) + + if (_charts.value is UiState.Ready || chartsJob != null) { + val mergedCharts = repository.mergeWithExternal( + gothwad = firstPartyChartsCache, + external = external, + kind = currentChartsKind, + category = null, + ) + _charts.value = registerAndBuildState(mergedCharts) + } + + val slug = currentCategorySlug + if (!slug.isNullOrBlank()) { + val mergedCat = repository.mergeWithExternal( + gothwad = firstPartyCategoryCache, + external = external, + kind = currentCategoryKind, + category = slug, + ) + _categoryApps.value = registerAndBuildState(mergedCat) + } + + val q = currentSearchQuery + if (q.isNotEmpty()) { + val mergedSearch = repository.mergeWithExternal( + gothwad = firstPartySearchCache, + external = external, + kind = null, + category = null, + query = q, + ) + _searchResults.value = registerAndBuildState(mergedSearch) + } + } + } } // ── Catalogue ───────────────────────────────────────────────────────── fun refreshHome(showLoading: Boolean = true) { homeJob?.cancel() homeJob = viewModelScope.launch { - if (showLoading) _home.value = UiState.Loading - _home.value = load { repository.firstPartyHome(null) } + if (showLoading && _home.value !is UiState.Ready) _home.value = UiState.Loading + _home.value = loadWithUpstream( + firstParty = { repository.firstPartyHome(null).also { firstPartyHomeCache = it } }, + merged = { repository.withFdroid(it, null, null) }, + publish = { _home.value = it }, + ) } } fun refreshApps(kind: String? = null, category: String? = null) { + currentAppsKind = kind + currentAppsCategory = category appsJob?.cancel() appsJob = viewModelScope.launch { _apps.value = UiState.Loading _apps.value = loadWithUpstream( - firstParty = { repository.firstPartyCatalog(kind, category, "") }, + firstParty = { repository.firstPartyCatalog(kind, category, "").also { firstPartyAppsCache = it } }, merged = { repository.withFdroid(it, kind, category) }, publish = { _apps.value = it }, ) @@ -142,9 +239,9 @@ class StoreViewModel : ViewModel() { fun refreshGames() { gamesJob?.cancel() gamesJob = viewModelScope.launch { - _games.value = UiState.Loading + if (_games.value !is UiState.Ready) _games.value = UiState.Loading _games.value = loadWithUpstream( - firstParty = { repository.firstPartyCatalog("game", null, "") }, + firstParty = { repository.firstPartyCatalog("game", null, "").also { firstPartyGamesCache = it } }, merged = { repository.withFdroid(it, "game", null) }, publish = { _games.value = it }, ) @@ -152,14 +249,17 @@ class StoreViewModel : ViewModel() { } /** - * Loads the optional F-Droid catalogue once. It is a separate shelf on - * Home and an extra source in Apps, Search and the category pages. + * Loads the F-Droid catalogue. It is shown as its own shelf on Home and + * merged into Apps, Games, Search, Top Charts, and Category screens. */ - fun loadFdroid() { - if (fdroidJob?.isActive == true) return + fun loadFdroid() = loadFdroid(forceRefresh = false) + + fun loadFdroid(forceRefresh: Boolean) { + if (!forceRefresh && fdroidJob?.isActive == true) return + fdroidJob?.cancel() fdroidJob = viewModelScope.launch { val response = try { - repository.fdroidCatalog() + repository.fdroidCatalog(forceRefresh) } catch (cancellation: CancellationException) { throw cancellation } catch (_: Exception) { @@ -167,16 +267,27 @@ class StoreViewModel : ViewModel() { } val index = repository.releaseIndex(response) if (index.isNotEmpty()) _releases.value = _releases.value + index + if (response.apps.isNotEmpty()) { + _knownApps.value = _knownApps.value + response.apps.associateBy { it.id } + } _fdroid.value = CatalogBundle(response.apps, index) } } + fun retryFdroid() { + loadFdroid(forceRefresh = true) + refreshHome(showLoading = false) + refreshApps(kind = currentAppsKind, category = currentAppsCategory) + refreshGames() + } + fun refreshCharts(kind: String?) { + currentChartsKind = kind chartsJob?.cancel() chartsJob = viewModelScope.launch { _charts.value = UiState.Loading _charts.value = loadWithUpstream( - firstParty = { repository.charts(kind) }, + firstParty = { repository.charts(kind).also { firstPartyChartsCache = it } }, merged = { repository.withFdroid(it, kind, null) }, publish = { _charts.value = it }, ) @@ -197,12 +308,14 @@ class StoreViewModel : ViewModel() { } fun loadCategory(slug: String, kind: String?) { - categoryJob?.cancel() val filter = kind?.takeIf { it != "both" } + currentCategorySlug = slug + currentCategoryKind = filter + categoryJob?.cancel() categoryJob = viewModelScope.launch { _categoryApps.value = UiState.Loading _categoryApps.value = loadWithUpstream( - firstParty = { repository.firstPartyCatalog(filter, slug, "") }, + firstParty = { repository.firstPartyCatalog(filter, slug, "").also { firstPartyCategoryCache = it } }, merged = { repository.withFdroid(it, filter, slug) }, publish = { _categoryApps.value = it }, ) @@ -212,7 +325,9 @@ class StoreViewModel : ViewModel() { fun search(query: String, debounceMs: Long = 300) { searchJob?.cancel() val trimmed = query.trim() + currentSearchQuery = trimmed if (trimmed.isEmpty()) { + firstPartySearchCache = CatalogResponse() _searchResults.value = UiState.Ready(CatalogBundle()) return } @@ -220,7 +335,7 @@ class StoreViewModel : ViewModel() { delay(debounceMs) _searchResults.value = UiState.Loading _searchResults.value = loadWithUpstream( - firstParty = { repository.firstPartyCatalog(null, null, trimmed) }, + firstParty = { repository.firstPartyCatalog(null, null, trimmed).also { firstPartySearchCache = it } }, merged = { repository.withFdroid(it, null, null, trimmed) }, publish = { _searchResults.value = it }, ) @@ -429,16 +544,22 @@ class StoreViewModel : ViewModel() { } // ── Internals ───────────────────────────────────────────────────────── - private suspend fun load(block: suspend () -> CatalogResponse): UiState = try { - val response = block() + private fun registerAndBuildState(response: CatalogResponse): UiState.Ready { val index = repository.releaseIndex(response) if (index.isNotEmpty()) _releases.value = _releases.value + index if (response.apps.isNotEmpty()) { _knownApps.value = _knownApps.value + response.apps.associateBy { it.id } } - if (response.apps.isEmpty()) UiState.Ready(CatalogBundle()) else UiState.Ready( - CatalogBundle(response.apps, index), - ) + return if (response.apps.isEmpty()) { + UiState.Ready(CatalogBundle()) + } else { + UiState.Ready(CatalogBundle(response.apps, index)) + } + } + + private suspend fun load(block: suspend () -> CatalogResponse): UiState = try { + val response = block() + registerAndBuildState(response) } catch (cancellation: CancellationException) { throw cancellation } catch (error: Exception) { @@ -446,13 +567,12 @@ class StoreViewModel : ViewModel() { } /** - * Publishes the first-party catalogue right away and then adds the - * optional F-Droid catalogue once its index has been downloaded. + * Publishes the first-party catalogue right away when non-empty and then + * adds the F-Droid catalogue once its index is available. * - * F-Droid's index is several megabytes, so waiting for the merge before - * rendering anything would leave the user staring at a spinner on a slow - * connection. If the upstream catalogue cannot be reached, the - * first-party list simply stays. + * Avoids flashing a premature empty state on Games, Category, or Search + * screens when first-party returns 0 items while F-Droid is still loading, + * and falls back to F-Droid if the first-party Worker is unreachable. */ private suspend fun loadWithUpstream( firstParty: suspend () -> CatalogResponse, @@ -464,12 +584,21 @@ class StoreViewModel : ViewModel() { } catch (cancellation: CancellationException) { throw cancellation } catch (error: Exception) { + val fallback = runCatching { merged(CatalogResponse()) }.getOrNull() + if (fallback != null && fallback.apps.isNotEmpty()) { + return registerAndBuildState(fallback).also(publish) + } return error.toUiState() } - val first = load { base } - publish(first) + if (base.apps.isNotEmpty()) { + publish(registerAndBuildState(base)) + } val second = load { merged(base) } - return if (second is UiState.Ready) second else first + return if (second is UiState.Ready) { + second.also(publish) + } else { + registerAndBuildState(base).also(publish) + } } private fun Throwable.toUiState(): UiState.Failed { diff --git a/app/src/main/java/com/gothwad/store/ui/screens/HomeScreen.kt b/app/src/main/java/com/gothwad/store/ui/screens/HomeScreen.kt index 557b9f7..e263df9 100644 --- a/app/src/main/java/com/gothwad/store/ui/screens/HomeScreen.kt +++ b/app/src/main/java/com/gothwad/store/ui/screens/HomeScreen.kt @@ -135,7 +135,7 @@ private fun HomeContent( message = error, tone = BannerTone.Warning, action = { - TextButton(onClick = vm::loadFdroid) { Text("Retry") } + TextButton(onClick = vm::retryFdroid) { Text("Retry") } }, ) } @@ -156,21 +156,26 @@ private fun HomeContent( val suggested = (nonGames + games).take(12) val trending = apps.sortedByDescending { it.downloadsCount }.take(5) val topFree = nonGames.ifEmpty { apps }.sortedByDescending { it.downloadsCount }.take(9) - val firstPartyPackages = apps.mapTo(mutableSetOf()) { it.packageName } - val fdroidPicks = fdroidApps.filterNot { it.packageName in firstPartyPackages }.take(12) + val firstPartyPackages = apps + .filter { !it.source.equals("F-Droid", ignoreCase = true) } + .mapTo(mutableSetOf()) { it.packageName } + val fdroidPicks = fdroidApps + .ifEmpty { apps.filter { it.source.equals("F-Droid", ignoreCase = true) } } + .filterNot { it.packageName in firstPartyPackages } + .take(16) LazyColumn( modifier = Modifier.fillMaxWidth(), contentPadding = PaddingValues(top = 8.dp, bottom = bottomPadding + Spacing.bottomContentPadding), ) { - if (fdroidError != null) { + if (fdroidError != null && fdroidPicks.isEmpty()) { item { InfoBanner( title = "F-Droid catalogue unavailable", message = fdroidError, tone = BannerTone.Warning, action = { - TextButton(onClick = vm::loadFdroid) { Text("Retry") } + TextButton(onClick = vm::retryFdroid) { Text("Retry") } }, ) } @@ -352,13 +357,16 @@ private fun HomeContent( private val CATEGORY_ICONS: Map = mapOf( "social" to Icons.Filled.Groups, "communication" to Icons.Filled.ChatBubbleOutline, + "browser" to Icons.Filled.Groups, "tools" to Icons.Filled.Build, "games" to Icons.Filled.SportsEsports, "game" to Icons.Filled.SportsEsports, "photography" to Icons.Filled.PhotoCamera, + "photo" to Icons.Filled.PhotoCamera, "education" to Icons.Filled.School, "finance" to Icons.Filled.AccountBalance, "health" to Icons.Filled.FavoriteBorder, + "lifestyle" to Icons.Filled.FavoriteBorder, "shopping" to Icons.Filled.ShoppingBag, "music" to Icons.Filled.MusicNote, "video" to Icons.Filled.PlayCircle, diff --git a/app/src/main/java/com/gothwad/store/ui/screens/ListingScreens.kt b/app/src/main/java/com/gothwad/store/ui/screens/ListingScreens.kt index 6b4178b..780e7b3 100644 --- a/app/src/main/java/com/gothwad/store/ui/screens/ListingScreens.kt +++ b/app/src/main/java/com/gothwad/store/ui/screens/ListingScreens.kt @@ -66,7 +66,10 @@ private fun FdroidStatusNotice(vm: StoreViewModel, onRetry: () -> Unit) { message = error.orEmpty(), tone = BannerTone.Warning, action = { - TextButton(onClick = onRetry) { Text("Retry") } + TextButton(onClick = { + vm.retryFdroid() + onRetry() + }) { Text("Retry") } }, ) } diff --git a/app/src/test/java/com/gothwad/store/FdroidIndexTest.kt b/app/src/test/java/com/gothwad/store/FdroidIndexTest.kt index 46d0416..5a2f744 100644 --- a/app/src/test/java/com/gothwad/store/FdroidIndexTest.kt +++ b/app/src/test/java/com/gothwad/store/FdroidIndexTest.kt @@ -3,6 +3,7 @@ package com.gothwad.store import com.gothwad.store.data.remote.FdroidCategories import com.gothwad.store.data.remote.FdroidIndex import com.gothwad.store.data.remote.FdroidIndexParser +import com.gothwad.store.data.remote.StoreRepository import com.gothwad.store.data.remote.toCatalog import com.squareup.moshi.JsonReader import okio.Buffer @@ -10,17 +11,26 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Test +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream /** - * F-Droid's `index-v1.json` is the only upstream catalogue the app reads, so - * its exact shape is pinned here. + * F-Droid's `index-v1.jar` / `index-v1.json` and `index-v2.json` schemas are + * pinned here. * - * The important detail: `apps` is an **array** of app objects while `packages` - * is a map of package name -> APK list, and the human readable name usually - * lives inside `localized["en-US"]`. Reading `apps` as a JSON object silently - * produced an empty catalogue, which is why no F-Droid app ever showed up. - * - * The payload below is trimmed from the real `https://f-droid.org/repo/index-v1.json`. + * Key details tested: + * 1. `index-v1.jar` is a ZIP/JAR archive containing `index-v1.json` (which is + * what Droid-ify and the official F-Droid client download). + * 2. `apps` in v1 is an **array** of app objects while `packages` is a map of + * package name -> APK list, and the human-readable name/summary/icon often + * live inside `localized["en-US"]` (or another locale). + * 3. Multi-ABI packages (`nativecode`) select the highest versionCode + * compatible with the device's `Build.SUPPORTED_ABIS` and `SDK_INT`. + * 4. `toCatalog` sorts by `lastUpdated` descending, cleans HTML from + * descriptions, and maps all F-Droid categories + aliases so every category + * screen and shelf is populated. */ class FdroidIndexTest { @@ -45,7 +55,8 @@ class FdroidIndexTest { "name": "0x54ask", "summary": "Todo.txt manager. Offline, works with Syncthing.", "description": "0x54ask is a todo.txt manager.", - "icon": "icon_FOzSYq6etfsaWRiMc7bx-8vLVKtsug1dmhT9NvxRj9w=.png" + "icon": "icon_FOzSYq6etfsaWRiMc7bx-8vLVKtsug1dmhT9NvxRj9w=.png", + "phoneScreenshots": ["shot1.png", "shot2.png"] } } }, @@ -53,7 +64,7 @@ class FdroidIndexTest { "packageName": "org.fdroid.fdroid", "name": "F-Droid", "summary": "The official F-Droid Free Software repository.", - "description": "F-Droid is an installable app catalogue.", + "description": "

F-Droid is an installable app catalogue.

", "license": "GPL-2.0-or-later", "categories": ["System"], "icon": "org.fdroid.fdroid.1017010.png", @@ -94,7 +105,8 @@ class FdroidIndexTest { "size": 12345, "targetSdkVersion": 36, "versionCode": 1010200, - "versionName": "1.1.2" + "versionName": "1.1.2", + "uses-permission": [["android.permission.INTERNET", null]] }, { "apkName": "com.terokarvinen.x54ask_1000100.apk", @@ -168,10 +180,14 @@ class FdroidIndexTest { // "com.example.toonew" needs API 40 and "com.example.party" has no APK. assertEquals(3, catalog.apps.size) assertEquals("F-Droid", catalog.source) + // Sorted by lastUpdated descending: 0x54ask (newest) -> F-Droid -> Retro Racer (oldest) + assertEquals("com.terokarvinen.x54ask", catalog.apps[0].packageName) + assertEquals("org.fdroid.fdroid", catalog.apps[1].packageName) + assertEquals("com.example.oldgame", catalog.apps[2].packageName) } @Test - fun `localized name wins over the package name`() { + fun `localized name wins over the package name and html is cleaned`() { val catalog = parse().toCatalog(repoUrl = "https://f-droid.org/repo") val app = catalog.apps.first { it.packageName == "com.terokarvinen.x54ask" } assertEquals("0x54ask", app.name) @@ -181,10 +197,19 @@ class FdroidIndexTest { assertEquals("Tero Karvinen", app.developerName) assertEquals("published", app.status) assertEquals("F-Droid", app.source) + assertEquals(2, app.screenshots.size) + assertEquals( + "https://f-droid.org/repo/com.terokarvinen.x54ask/en-US/phoneScreenshots/shot1.png", + app.screenshots[0], + ) + + val fdroidApp = catalog.apps.first { it.packageName == "org.fdroid.fdroid" } + assertEquals("F-Droid is an installable app catalogue.", fdroidApp.fullDesc) + assertTrue(fdroidApp.updatedAt.isNotBlank()) } @Test - fun `f-droid categories map onto kiwi store slugs`() { + fun `f-droid categories map onto kiwi store slugs and aliases`() { val catalog = parse().toCatalog(repoUrl = "https://f-droid.org/repo") assertEquals( "productivity", @@ -195,6 +220,10 @@ class FdroidIndexTest { val game = catalog.apps.first { it.packageName == "com.example.oldgame" } assertEquals("game", game.kind) assertEquals("entertainment", game.category) + + val repo = StoreRepository() + assertTrue(repo.categoryMatches(game, "games")) + assertTrue(repo.categoryMatches(game, "entertainment")) } @Test @@ -223,6 +252,7 @@ class FdroidIndexTest { assertEquals("aaaa", release.apkSha256) assertEquals(29, release.minSdk!!) assertEquals(36, release.targetSdk!!) + assertEquals(listOf("android.permission.INTERNET"), release.permissions) assertTrue(release.isLive) assertEquals("approved", release.reviewStatus) @@ -254,4 +284,132 @@ class FdroidIndexTest { assertNotNull(catalog) assertTrue(catalog.apps.none { it.packageName == "com.example.oldgame" }) } + + @Test + fun `parses index-v1 jar archive via parseSource and parseFile`() { + val jarBytes = ByteArrayOutputStream().also { baos -> + ZipOutputStream(baos).use { zos -> + zos.putNextEntry(ZipEntry("META-INF/MANIFEST.MF")) + zos.write("Manifest-Version: 1.0\n".toByteArray()) + zos.closeEntry() + zos.putNextEntry(ZipEntry("index-v1.json")) + zos.write(indexPayload.toByteArray(Charsets.UTF_8)) + zos.closeEntry() + } + }.toByteArray() + + val fromStream = FdroidIndexParser.parseSource( + source = Buffer().write(jarBytes), + sdkInt = 34, + ).toCatalog("https://f-droid.org/repo") + assertEquals(3, fromStream.apps.size) + + val tempJar = File.createTempFile("fdroid_test_", ".jar") + try { + tempJar.writeBytes(jarBytes) + val fromFile = FdroidIndexParser.parseFile( + file = tempJar, + sdkInt = 34, + ).toCatalog("https://f-droid.org/repo") + assertEquals(3, fromFile.apps.size) + } finally { + tempJar.delete() + } + } + + @Test + fun `nativecode abi filtering skips incompatible x86_64 builds on arm64 devices`() { + val multiAbiPayload = """ + { + "apps": [ + { + "packageName": "com.junkfood.seal", + "name": "Seal", + "summary": "Video/Audio Downloader", + "categories": ["Multimedia"] + } + ], + "packages": { + "com.junkfood.seal": [ + { + "apkName": "com.junkfood.seal_12014.apk", + "versionCode": 12014, + "versionName": "1.12.1", + "minSdkVersion": 24, + "nativecode": ["x86_64"] + }, + { + "apkName": "com.junkfood.seal_12012.apk", + "versionCode": 12012, + "versionName": "1.12.1", + "minSdkVersion": 24, + "nativecode": ["arm64-v8a"] + } + ] + } + } + """.trimIndent() + + val index = FdroidIndexParser.parse( + reader = JsonReader.of(Buffer().writeUtf8(multiAbiPayload)), + sdkInt = 34, + supportedAbis = setOf("arm64-v8a", "armeabi-v7a"), + ) + val apk = index.packages["com.junkfood.seal"] + assertNotNull(apk) + assertEquals(12012L, apk!!.versionCode) + assertEquals("com.junkfood.seal_12012.apk", apk.apkName) + } + + @Test + fun `parses index-v2 schema with localized metadata and versions`() { + val v2Payload = """ + { + "repo": { "name": { "en-US": "F-Droid" }, "timestamp": 1792190113856 }, + "packages": { + "org.videolan.vlc": { + "metadata": { + "name": { "en-US": "VLC" }, + "summary": { "en-US": "The best open-source media player" }, + "description": { "en-US": "Plays everything." }, + "categories": ["Multimedia"], + "icon": { "en-US": { "name": "/org.videolan.vlc/en-US/icon.png" } }, + "added": 1600000000000, + "lastUpdated": 1700000000000 + }, + "versions": { + "sha256_1": { + "added": 1700000000000, + "file": { + "name": "/org.videolan.vlc_13050304.apk", + "sha256": "abcd1234", + "size": 42000000 + }, + "manifest": { + "versionName": "3.5.3", + "versionCode": 13050304, + "usesSdk": { "minSdkVersion": 21, "targetSdkVersion": 33 }, + "nativecode": ["arm64-v8a"] + } + } + } + } + } + } + """.trimIndent() + + val catalog = FdroidIndexParser.parse( + reader = JsonReader.of(Buffer().writeUtf8(v2Payload)), + sdkInt = 34, + supportedAbis = setOf("arm64-v8a"), + ).toCatalog("https://f-droid.org/repo") + + assertEquals(1, catalog.apps.size) + val vlc = catalog.apps.first() + assertEquals("VLC", vlc.name) + assertEquals("https://f-droid.org/repo/org.videolan.vlc/en-US/icon.png", vlc.iconUrl) + val rel = catalog.releases.first() + assertEquals("https://f-droid.org/repo/org.videolan.vlc_13050304.apk", rel.apkUrl) + assertEquals(13050304L, rel.versionCode) + } }