Keep blocklists out of the heap to fix out-of-memory crash on large lists

The merged blocklist was built as one LongArray next to every per-list
array and the merge copies; with ~7M domains this ran out of memory on
256 MB heaps (Galaxy A07) and the uncaught error took the VPN down.

Lists and the merged set are now memory-mapped files: the merge streams
from the list files into merged.bin, which is reused at start-up while
the switched-on lists are unchanged. Downloads sort in their own buffer
instead of copying, and the update job no longer crashes the app if it
still runs out of memory.
This commit is contained in:
2026-11-12 10:46:27 +02:00
parent fb0c13bbcc
commit 6d015e9527
3 changed files with 154 additions and 56 deletions
@@ -5,14 +5,17 @@ import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.nio.LongBuffer
class BlocklistMatchTest {
private fun set(vararg domains: String) = domains.map { DomainHash.of(it) }.sorted().toLongArray()
private fun buf(vararg domains: String) = LongBuffer.wrap(set(*domains))
private fun buf(values: LongArray) = LongBuffer.wrap(values)
@Test
fun blocksDomainsAndTheirSubdomains() {
val lists = set("ads.example.com")
val lists = buf("ads.example.com")
val custom = set("tracker.net")
assertTrue(isBlockedIn("ads.example.com", lists, custom))
@@ -21,12 +24,12 @@ class BlocklistMatchTest {
assertFalse(isBlockedIn("example.com", lists, custom))
assertFalse(isBlockedIn("badads.example.com", lists, custom))
assertFalse(isBlockedIn("tracker.network", lists, custom))
assertFalse(isBlockedIn("ads.example.com", LongArray(0), LongArray(0)))
assertFalse(isBlockedIn("ads.example.com", EMPTY_SET, LongArray(0)))
}
@Test
fun skipsTrailingDots() {
val lists = set("example.com")
val lists = buf("example.com")
assertTrue(isBlockedIn("www.example.com..", lists, LongArray(0)))
assertFalse(isBlockedIn("example.org.", lists, LongArray(0)))
@@ -51,8 +54,28 @@ class BlocklistMatchTest {
@Test
fun countsEntriesInBothSetsOnce() {
assertEquals(5, unionSize(longArrayOf(1, 2, 3, 4), longArrayOf(2, 5)))
assertEquals(2, unionSize(LongArray(0), longArrayOf(2, 5)))
assertEquals(4, unionSize(longArrayOf(1, 2, 3, 4), LongArray(0)))
assertEquals(5, unionSize(buf(longArrayOf(1, 2, 3, 4)), longArrayOf(2, 5)))
assertEquals(2, unionSize(EMPTY_SET, longArrayOf(2, 5)))
assertEquals(4, unionSize(buf(longArrayOf(1, 2, 3, 4)), LongArray(0)))
}
@Test
fun streamsMergedValuesInOrderOnce() {
val out = ArrayList<Long>()
val count = mergeSortedInto(
listOf(buf(longArrayOf(-5, 1, 4, 9)), buf(longArrayOf(1, 2, 9, 12)), EMPTY_SET, buf(longArrayOf(12, 40)))
) { out.add(it) }
assertEquals(listOf(-5L, 1, 2, 4, 9, 12, 40), out)
assertEquals(7, count)
assertEquals(0, mergeSortedInto(emptyList()) { out.add(it) })
}
@Test
fun findsValuesInSortedBuffer() {
val values = buf(longArrayOf(Long.MIN_VALUE, -3, 0, 7, Long.MAX_VALUE))
listOf(Long.MIN_VALUE, -3L, 0L, 7L, Long.MAX_VALUE).forEach { assertTrue(values.containsSorted(it)) }
listOf(-4L, 1L, 8L).forEach { assertFalse(values.containsSorted(it)) }
assertFalse(EMPTY_SET.containsSorted(0))
}
}