Keep blocklists out of the heap to fix out-of-memory crash on large lists

The merged blocklist was built as one LongArray next to every per-list
array and the merge copies; with ~7M domains this ran out of memory on
256 MB heaps (Galaxy A07) and the uncaught error took the VPN down.

Lists and the merged set are now memory-mapped files: the merge streams
from the list files into merged.bin, which is reused at start-up while
the switched-on lists are unchanged. Downloads sort in their own buffer
instead of copying, and the update job no longer crashes the app if it
still runs out of memory.
This commit is contained in:
2026-11-12 10:46:27 +02:00
parent fb0c13bbcc
commit 6d015e9527
3 changed files with 154 additions and 56 deletions
@@ -49,14 +49,18 @@ object AutoUpdate {
class BlocklistUpdateJob : JobService() { class BlocklistUpdateJob : JobService() {
override fun onStartJob(params: JobParameters): Boolean { override fun onStartJob(params: JobParameters): Boolean {
Thread({ Thread({
val ok = try { // A failed download is retried later with the system's back-off
Blocklist.updateAllNow(applicationContext) val retry = try {
!Blocklist.updateAllNow(applicationContext)
} catch (e: Exception) { } catch (e: Exception) {
Log.w("SparkleDNS", "Blocklist update failed", e) Log.w("SparkleDNS", "Blocklist update failed", e)
true
} catch (e: OutOfMemoryError) {
// Must not take the VPN down with it; retrying would only run out of memory again
Log.e("SparkleDNS", "Blocklist update ran out of memory", e)
false false
} }
// A failed run is retried later with the system's back-off jobFinished(params, retry)
jobFinished(params, !ok)
}, "blocklist-update").start() }, "blocklist-update").start()
return true return true
} }
@@ -20,6 +20,7 @@ import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.io.FileOutputStream import java.io.FileOutputStream
import java.io.IOException import java.io.IOException
import java.nio.LongBuffer
import java.nio.channels.FileChannel import java.nio.channels.FileChannel
import java.security.MessageDigest import java.security.MessageDigest
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@@ -28,7 +29,8 @@ import java.util.concurrent.TimeUnit
data class RemoteList(val url: String, val count: Int, val updated: Long, val enabled: Boolean = true) data class RemoteList(val url: String, val count: Int, val updated: Long, val enabled: Boolean = true)
// Blocks a domain together with all its subdomains. Changes apply instantly, no reconnect needed. // Blocks a domain together with all its subdomains. Changes apply instantly, no reconnect needed.
// Domains are kept as sorted 64-bit hashes: about 8 bytes per entry, so million-entry lists fit easily. // Domains are kept as sorted 64-bit hashes in files that are memory-mapped, not read into the heap:
// lists with millions of entries cost page cache, not app memory, so low-end phones cope too.
object Blocklist { object Blocklist {
const val ALL_LISTS = "*" const val ALL_LISTS = "*"
@@ -60,7 +62,7 @@ object Blocklist {
// Downloaded lists and own domains are kept apart, so editing a few own domains // Downloaded lists and own domains are kept apart, so editing a few own domains
// does not rebuild a set that can hold millions of entries // does not rebuild a set that can hold millions of entries
@Volatile @Volatile
private var listHashes = LongArray(0) private var listHashes: LongBuffer = EMPTY_SET
@Volatile @Volatile
private var customHashes = LongArray(0) private var customHashes = LongArray(0)
@@ -223,7 +225,12 @@ object Blocklist {
} }
private fun reloadListHashes(store: BlocklistStore) { private fun reloadListHashes(store: BlocklistStore) {
listHashes = mergeSorted(_lists.value.filter { it.enabled }.map { store.loadListHashes(it.url) }) try {
listHashes = store.mergedHashes(_lists.value.filter { it.enabled })
} catch (e: IOException) {
// Out of storage: the lists in use stay active until the next change
Log.w("SparkleDNS", "Cannot merge blocklists", e)
}
_total.value = unionSize(listHashes, customHashes) _total.value = unionSize(listHashes, customHashes)
} }
@@ -236,13 +243,13 @@ object Blocklist {
// True when the name or one of its parent domains is in either sorted hash set. Runs for every // True when the name or one of its parent domains is in either sorted hash set. Runs for every
// query, so trailing dots are skipped by index rather than trimmed off a copy of the name. // query, so trailing dots are skipped by index rather than trimmed off a copy of the name.
internal fun isBlockedIn(name: String, lists: LongArray, custom: LongArray): Boolean { internal fun isBlockedIn(name: String, lists: LongBuffer, custom: LongArray): Boolean {
if (lists.isEmpty() && custom.isEmpty()) return false if (lists.limit() == 0 && custom.isEmpty()) return false
val end = nameEnd(name) val end = nameEnd(name)
var start = 0 var start = 0
while (true) { while (true) {
val hash = DomainHash.of(name, start, end) val hash = DomainHash.of(name, start, end)
if (lists.binarySearch(hash) >= 0 || custom.binarySearch(hash) >= 0) return true if (lists.containsSorted(hash) || custom.binarySearch(hash) >= 0) return true
val dot = name.indexOf('.', start) val dot = name.indexOf('.', start)
if (dot < 0 || dot >= end) return false if (dot < 0 || dot >= end) return false
start = dot + 1 start = dot + 1
@@ -256,37 +263,59 @@ internal fun nameEnd(name: String): Int {
return end return end
} }
// Number of distinct entries in both sorted sets; the custom set is small internal val EMPTY_SET: LongBuffer = LongBuffer.wrap(LongArray(0))
internal fun unionSize(lists: LongArray, custom: LongArray): Int =
lists.size + custom.count { lists.binarySearch(it) < 0 }
// Merges sorted, duplicate-free arrays in linear time instead of sorting their concatenation // Binary search by absolute index, so several threads can read the same buffer at once
internal fun mergeSorted(parts: List<LongArray>): LongArray { internal fun LongBuffer.containsSorted(value: Long): Boolean {
var arrays = parts.filter { it.isNotEmpty() }.map { if (it.isSortedUnique()) it else it.copyOf().sortUnique() } var low = 0
if (arrays.isEmpty()) return LongArray(0) var high = limit() - 1
while (arrays.size > 1) { while (low <= high) {
arrays = arrays.chunked(2).map { if (it.size == 2) mergeTwo(it[0], it[1]) else it[0] } val mid = (low + high) ushr 1
val v = get(mid)
when {
v < value -> low = mid + 1
v > value -> high = mid - 1
else -> return true
} }
return arrays[0] }
return false
} }
private fun mergeTwo(a: LongArray, b: LongArray): LongArray { // Number of distinct entries in both sorted sets; the custom set is small
val out = LongArray(a.size + b.size) internal fun unionSize(lists: LongBuffer, custom: LongArray): Int =
var i = 0 lists.limit() + custom.count { !lists.containsSorted(it) }
var j = 0
var n = 0 // Merges sorted, duplicate-free sets in one pass and hands each value to [emit] once, in order.
while (i < a.size && j < b.size) { // Only the read positions are kept in memory, so inputs and output can be files of any size.
val x = a[i] internal fun mergeSortedInto(parts: List<LongBuffer>, emit: (Long) -> Unit): Int {
val y = b[j] val sets = parts.filter { it.limit() > 0 }
when { val pos = IntArray(sets.size)
x < y -> { out[n++] = x; i++ } var count = 0
x > y -> { out[n++] = y; j++ } while (true) {
else -> { out[n++] = x; i++; j++ } var min = 0L
var found = false
for (i in sets.indices) {
if (pos[i] < sets[i].limit()) {
val v = sets[i].get(pos[i])
if (!found || v < min) {
min = v
found = true
} }
} }
while (i < a.size) out[n++] = a[i++] }
while (j < b.size) out[n++] = b[j++] if (!found) return count
return if (n == out.size) out else out.copyOf(n) for (i in sets.indices) if (pos[i] < sets[i].limit() && sets[i].get(pos[i]) == min) pos[i]++
emit(min)
count++
}
}
// In-memory variant for small sets; inputs that are not sorted yet are sorted first
internal fun mergeSorted(parts: List<LongArray>): LongArray {
val sorted = parts.map { LongBuffer.wrap(if (it.isSortedUnique()) it else it.copyOf().sortUnique()) }
val out = LongList()
mergeSortedInto(sorted) { out.add(it) }
return out.toArray()
} }
private fun LongArray.isSortedUnique(): Boolean { private fun LongArray.isSortedUnique(): Boolean {
@@ -320,6 +349,20 @@ private class LongList {
} }
fun toArray(): LongArray = data.copyOf(size) fun toArray(): LongArray = data.copyOf(size)
// Sorts the filled part in place and drops duplicates, without copying; returns the new size
fun sortUnique(): Int {
if (size < 2) return size
java.util.Arrays.sort(data, 0, size)
var n = 1
for (i in 1 until size) if (data[i] != data[n - 1]) data[n++] = data[i]
size = n
return n
}
fun writeTo(out: DataOutputStream) {
for (i in 0 until size) out.writeLong(data[i])
}
} }
// Sorts in place and drops duplicates // Sorts in place and drops duplicates
@@ -375,17 +418,42 @@ private class BlocklistStore(context: Context) {
sp.edit().putString("lists", array.toString()).commit() sp.edit().putString("lists", array.toString()).commit()
} }
fun loadListHashes(url: String): LongArray { fun loadListHashes(url: String): LongBuffer = map(binFile(url)) ?: EMPTY_SET
val file = binFile(url)
if (!file.exists()) return LongArray(0) // One list is used as it is; several are merged into one file, which is reused at the next
// start as long as the switched-on lists have not changed
fun mergedHashes(lists: List<RemoteList>): LongBuffer {
if (lists.size < 2) {
mergedFile.delete()
return lists.firstOrNull()?.let { loadListHashes(it.url) } ?: EMPTY_SET
}
val signature = hash(lists.sortedBy { it.url }.joinToString("\n") { "${it.url} ${it.count} ${it.updated}" })
if (sp.getString(MERGED_KEY, null) == signature) map(mergedFile)?.let { return it }
val parts = lists.map { loadListHashes(it.url) }
val tmp = File.createTempFile("merged_", ".tmp", dir)
try {
DataOutputStream(BufferedOutputStream(FileOutputStream(tmp), 64 * 1024)).use { out ->
mergeSortedInto(parts) { out.writeLong(it) }
}
if (!tmp.renameTo(mergedFile)) throw IOException("Cannot save lists")
} finally {
tmp.delete()
}
sp.edit().putString(MERGED_KEY, signature).commit()
return map(mergedFile) ?: EMPTY_SET
}
// The mapping stays valid after the file is replaced or deleted, so readers are never cut off
private fun map(file: File): LongBuffer? {
if (!file.exists()) return null
return runCatching { return runCatching {
FileInputStream(file).use { stream -> FileInputStream(file).use { stream ->
val channel = stream.channel val channel = stream.channel
val result = LongArray((channel.size() / 8).toInt()) val bytes = channel.size() / 8 * 8
channel.map(FileChannel.MapMode.READ_ONLY, 0, result.size * 8L).asLongBuffer().get(result) if (bytes == 0L) EMPTY_SET else channel.map(FileChannel.MapMode.READ_ONLY, 0, bytes).asLongBuffer()
result
} }
}.getOrDefault(LongArray(0)) }.getOrNull()
} }
fun deleteListFile(url: String) { fun deleteListFile(url: String) {
@@ -415,22 +483,23 @@ private class BlocklistStore(context: Context) {
parsed.add(DomainHash.of(domain)) parsed.add(DomainHash.of(domain))
} }
} }
val result = parsed.toArray().sortUnique() // Sorted inside the download buffer: a list with millions of entries is never copied
if (result.isEmpty()) throw IOException("No domains found") val count = parsed.sortUnique()
write(url, result) if (count == 0) throw IOException("No domains found")
write(url, parsed)
sp.edit() sp.edit()
.putOrRemove(etagKey(url), response.header("ETag")) .putOrRemove(etagKey(url), response.header("ETag"))
.putOrRemove(modifiedKey(url), response.header("Last-Modified")) .putOrRemove(modifiedKey(url), response.header("Last-Modified"))
.apply() .apply()
return RemoteList(url, result.size, System.currentTimeMillis()) return RemoteList(url, count, System.currentTimeMillis())
} }
} }
private fun write(url: String, data: LongArray) { private fun write(url: String, data: LongList) {
val tmp = File.createTempFile("list_", ".tmp", dir) val tmp = File.createTempFile("list_", ".tmp", dir)
try { try {
DataOutputStream(BufferedOutputStream(FileOutputStream(tmp), 64 * 1024)).use { out -> DataOutputStream(BufferedOutputStream(FileOutputStream(tmp), 64 * 1024)).use { out ->
data.forEach { out.writeLong(it) } data.writeTo(out)
} }
if (!tmp.renameTo(binFile(url))) throw IOException("Cannot save list") if (!tmp.renameTo(binFile(url))) throw IOException("Cannot save list")
} finally { } finally {
@@ -445,12 +514,14 @@ private class BlocklistStore(context: Context) {
if (value == null) remove(key) else putString(key, value) if (value == null) remove(key) else putString(key, value)
private fun binFile(url: String) = File(dir, "list_${hash(url)}.bin") private fun binFile(url: String) = File(dir, "list_${hash(url)}.bin")
private val mergedFile get() = File(dir, "merged.bin")
private fun hash(url: String) = MessageDigest.getInstance("SHA-256").digest(url.toByteArray()) private fun hash(url: String) = MessageDigest.getInstance("SHA-256").digest(url.toByteArray())
.take(8).joinToString("") { "%02x".format(it) } .take(8).joinToString("") { "%02x".format(it) }
companion object { companion object {
const val MAX_LIST_DOMAINS = 4_000_000 const val MAX_LIST_DOMAINS = 4_000_000
private const val MERGED_KEY = "merged"
val client: OkHttpClient by lazy { val client: OkHttpClient by lazy {
Http.client.newBuilder() Http.client.newBuilder()
@@ -5,14 +5,17 @@ import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue import org.junit.Assert.assertTrue
import org.junit.Test import org.junit.Test
import java.nio.LongBuffer
class BlocklistMatchTest { class BlocklistMatchTest {
private fun set(vararg domains: String) = domains.map { DomainHash.of(it) }.sorted().toLongArray() private fun set(vararg domains: String) = domains.map { DomainHash.of(it) }.sorted().toLongArray()
private fun buf(vararg domains: String) = LongBuffer.wrap(set(*domains))
private fun buf(values: LongArray) = LongBuffer.wrap(values)
@Test @Test
fun blocksDomainsAndTheirSubdomains() { fun blocksDomainsAndTheirSubdomains() {
val lists = set("ads.example.com") val lists = buf("ads.example.com")
val custom = set("tracker.net") val custom = set("tracker.net")
assertTrue(isBlockedIn("ads.example.com", lists, custom)) assertTrue(isBlockedIn("ads.example.com", lists, custom))
@@ -21,12 +24,12 @@ class BlocklistMatchTest {
assertFalse(isBlockedIn("example.com", lists, custom)) assertFalse(isBlockedIn("example.com", lists, custom))
assertFalse(isBlockedIn("badads.example.com", lists, custom)) assertFalse(isBlockedIn("badads.example.com", lists, custom))
assertFalse(isBlockedIn("tracker.network", lists, custom)) assertFalse(isBlockedIn("tracker.network", lists, custom))
assertFalse(isBlockedIn("ads.example.com", LongArray(0), LongArray(0))) assertFalse(isBlockedIn("ads.example.com", EMPTY_SET, LongArray(0)))
} }
@Test @Test
fun skipsTrailingDots() { fun skipsTrailingDots() {
val lists = set("example.com") val lists = buf("example.com")
assertTrue(isBlockedIn("www.example.com..", lists, LongArray(0))) assertTrue(isBlockedIn("www.example.com..", lists, LongArray(0)))
assertFalse(isBlockedIn("example.org.", lists, LongArray(0))) assertFalse(isBlockedIn("example.org.", lists, LongArray(0)))
@@ -51,8 +54,28 @@ class BlocklistMatchTest {
@Test @Test
fun countsEntriesInBothSetsOnce() { fun countsEntriesInBothSetsOnce() {
assertEquals(5, unionSize(longArrayOf(1, 2, 3, 4), longArrayOf(2, 5))) assertEquals(5, unionSize(buf(longArrayOf(1, 2, 3, 4)), longArrayOf(2, 5)))
assertEquals(2, unionSize(LongArray(0), longArrayOf(2, 5))) assertEquals(2, unionSize(EMPTY_SET, longArrayOf(2, 5)))
assertEquals(4, unionSize(longArrayOf(1, 2, 3, 4), LongArray(0))) assertEquals(4, unionSize(buf(longArrayOf(1, 2, 3, 4)), LongArray(0)))
}
@Test
fun streamsMergedValuesInOrderOnce() {
val out = ArrayList<Long>()
val count = mergeSortedInto(
listOf(buf(longArrayOf(-5, 1, 4, 9)), buf(longArrayOf(1, 2, 9, 12)), EMPTY_SET, buf(longArrayOf(12, 40)))
) { out.add(it) }
assertEquals(listOf(-5L, 1, 2, 4, 9, 12, 40), out)
assertEquals(7, count)
assertEquals(0, mergeSortedInto(emptyList()) { out.add(it) })
}
@Test
fun findsValuesInSortedBuffer() {
val values = buf(longArrayOf(Long.MIN_VALUE, -3, 0, 7, Long.MAX_VALUE))
listOf(Long.MIN_VALUE, -3L, 0L, 7L, Long.MAX_VALUE).forEach { assertTrue(values.containsSorted(it)) }
listOf(-4L, 1L, 8L).forEach { assertFalse(values.containsSorted(it)) }
assertFalse(EMPTY_SET.containsSorted(0))
} }
} }