6 Commits
Author SHA1 Message Date
kalarina 6d015e9527 Keep blocklists out of the heap to fix out-of-memory crash on large lists
The merged blocklist was built as one LongArray next to every per-list
array and the merge copies; with ~7M domains this ran out of memory on
256 MB heaps (Galaxy A07) and the uncaught error took the VPN down.

Lists and the merged set are now memory-mapped files: the merge streams
from the list files into merged.bin, which is reused at start-up while
the switched-on lists are unchanged. Downloads sort in their own buffer
instead of copying, and the update job no longer crashes the app if it
still runs out of memory.
2026-11-12 10:46:27 +02:00
kalarina fb0c13bbcc Add dnsforge.de (unfiltered) to built-in DoH servers 2026-11-11 21:15:21 +02:00
kalarina 4ef8cef4e8 Expand call for volunteer designers 2026-11-10 15:33:26 +02:00
kalarina 64a122fe25 Add call for volunteer designers to README 2026-11-10 15:30:49 +02:00
kalarina 180a9a65fd Move Google Play notice to the top of README 2026-11-08 17:26:06 +02:00
kalarina 845e975c05 Add Google Play closed testing notice to README 2026-11-08 17:23:02 +02:00
5 changed files with 166 additions and 56 deletions
+6
View File
@@ -9,6 +9,12 @@
A local VPN that captures only DNS traffic and sends every query to the DoH server you choose (RFC 8484). Everything else goes through your normal connection.
</p>
> [!IMPORTANT]
> **SparkleDNS is coming to Google Play!** The app is now in closed testing there ahead of the public release. Ideas, wishes and suggestions for improvement are very welcome — send them wherever is easiest for you: an issue on [Gitea](https://git.skysparkle.cc/kalarina/Sparkledns/issues) or [GitHub](https://github.com/kalarina-bit/Sparkledns/issues), or an email to [cuken107@proton.me](mailto:cuken107@proton.me).
> [!NOTE]
> **Looking for designers!** SparkleDNS is looking for volunteer designers to help with icons, logos, animations, cover art (store graphics) and UI design. This is unpaid help for an open-source project. If you're interested, write to [cuken107@proton.me](mailto:cuken107@proton.me).
<p align="center">
<a href="https://github.com/kalarina-bit/Sparkledns/releases/latest"><img src="https://img.shields.io/github/v/release/kalarina-bit/Sparkledns?style=for-the-badge&label=release&labelColor=1b1b1b&color=2e7d32" alt="Latest release"></a>
</p>
@@ -49,14 +49,18 @@ object AutoUpdate {
class BlocklistUpdateJob : JobService() {
override fun onStartJob(params: JobParameters): Boolean {
Thread({
val ok = try {
Blocklist.updateAllNow(applicationContext)
// A failed download is retried later with the system's back-off
val retry = try {
!Blocklist.updateAllNow(applicationContext)
} catch (e: Exception) {
Log.w("SparkleDNS", "Blocklist update failed", e)
true
} catch (e: OutOfMemoryError) {
// Must not take the VPN down with it; retrying would only run out of memory again
Log.e("SparkleDNS", "Blocklist update ran out of memory", e)
false
}
// A failed run is retried later with the system's back-off
jobFinished(params, !ok)
jobFinished(params, retry)
}, "blocklist-update").start()
return true
}
@@ -20,6 +20,7 @@ import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.nio.LongBuffer
import java.nio.channels.FileChannel
import java.security.MessageDigest
import java.util.concurrent.TimeUnit
@@ -28,7 +29,8 @@ import java.util.concurrent.TimeUnit
data class RemoteList(val url: String, val count: Int, val updated: Long, val enabled: Boolean = true)
// Blocks a domain together with all its subdomains. Changes apply instantly, no reconnect needed.
// Domains are kept as sorted 64-bit hashes: about 8 bytes per entry, so million-entry lists fit easily.
// Domains are kept as sorted 64-bit hashes in files that are memory-mapped, not read into the heap:
// lists with millions of entries cost page cache, not app memory, so low-end phones cope too.
object Blocklist {
const val ALL_LISTS = "*"
@@ -60,7 +62,7 @@ object Blocklist {
// Downloaded lists and own domains are kept apart, so editing a few own domains
// does not rebuild a set that can hold millions of entries
@Volatile
private var listHashes = LongArray(0)
private var listHashes: LongBuffer = EMPTY_SET
@Volatile
private var customHashes = LongArray(0)
@@ -223,7 +225,12 @@ object Blocklist {
}
private fun reloadListHashes(store: BlocklistStore) {
listHashes = mergeSorted(_lists.value.filter { it.enabled }.map { store.loadListHashes(it.url) })
try {
listHashes = store.mergedHashes(_lists.value.filter { it.enabled })
} catch (e: IOException) {
// Out of storage: the lists in use stay active until the next change
Log.w("SparkleDNS", "Cannot merge blocklists", e)
}
_total.value = unionSize(listHashes, customHashes)
}
@@ -236,13 +243,13 @@ object Blocklist {
// True when the name or one of its parent domains is in either sorted hash set. Runs for every
// query, so trailing dots are skipped by index rather than trimmed off a copy of the name.
internal fun isBlockedIn(name: String, lists: LongArray, custom: LongArray): Boolean {
if (lists.isEmpty() && custom.isEmpty()) return false
internal fun isBlockedIn(name: String, lists: LongBuffer, custom: LongArray): Boolean {
if (lists.limit() == 0 && custom.isEmpty()) return false
val end = nameEnd(name)
var start = 0
while (true) {
val hash = DomainHash.of(name, start, end)
if (lists.binarySearch(hash) >= 0 || custom.binarySearch(hash) >= 0) return true
if (lists.containsSorted(hash) || custom.binarySearch(hash) >= 0) return true
val dot = name.indexOf('.', start)
if (dot < 0 || dot >= end) return false
start = dot + 1
@@ -256,37 +263,59 @@ internal fun nameEnd(name: String): Int {
return end
}
// Number of distinct entries in both sorted sets; the custom set is small
internal fun unionSize(lists: LongArray, custom: LongArray): Int =
lists.size + custom.count { lists.binarySearch(it) < 0 }
internal val EMPTY_SET: LongBuffer = LongBuffer.wrap(LongArray(0))
// Merges sorted, duplicate-free arrays in linear time instead of sorting their concatenation
internal fun mergeSorted(parts: List<LongArray>): LongArray {
var arrays = parts.filter { it.isNotEmpty() }.map { if (it.isSortedUnique()) it else it.copyOf().sortUnique() }
if (arrays.isEmpty()) return LongArray(0)
while (arrays.size > 1) {
arrays = arrays.chunked(2).map { if (it.size == 2) mergeTwo(it[0], it[1]) else it[0] }
}
return arrays[0]
}
private fun mergeTwo(a: LongArray, b: LongArray): LongArray {
val out = LongArray(a.size + b.size)
var i = 0
var j = 0
var n = 0
while (i < a.size && j < b.size) {
val x = a[i]
val y = b[j]
// Binary search by absolute index, so several threads can read the same buffer at once
internal fun LongBuffer.containsSorted(value: Long): Boolean {
var low = 0
var high = limit() - 1
while (low <= high) {
val mid = (low + high) ushr 1
val v = get(mid)
when {
x < y -> { out[n++] = x; i++ }
x > y -> { out[n++] = y; j++ }
else -> { out[n++] = x; i++; j++ }
v < value -> low = mid + 1
v > value -> high = mid - 1
else -> return true
}
}
while (i < a.size) out[n++] = a[i++]
while (j < b.size) out[n++] = b[j++]
return if (n == out.size) out else out.copyOf(n)
return false
}
// Number of distinct entries in both sorted sets; the custom set is small
internal fun unionSize(lists: LongBuffer, custom: LongArray): Int =
lists.limit() + custom.count { !lists.containsSorted(it) }
// Merges sorted, duplicate-free sets in one pass and hands each value to [emit] once, in order.
// Only the read positions are kept in memory, so inputs and output can be files of any size.
internal fun mergeSortedInto(parts: List<LongBuffer>, emit: (Long) -> Unit): Int {
val sets = parts.filter { it.limit() > 0 }
val pos = IntArray(sets.size)
var count = 0
while (true) {
var min = 0L
var found = false
for (i in sets.indices) {
if (pos[i] < sets[i].limit()) {
val v = sets[i].get(pos[i])
if (!found || v < min) {
min = v
found = true
}
}
}
if (!found) return count
for (i in sets.indices) if (pos[i] < sets[i].limit() && sets[i].get(pos[i]) == min) pos[i]++
emit(min)
count++
}
}
// In-memory variant for small sets; inputs that are not sorted yet are sorted first
internal fun mergeSorted(parts: List<LongArray>): LongArray {
val sorted = parts.map { LongBuffer.wrap(if (it.isSortedUnique()) it else it.copyOf().sortUnique()) }
val out = LongList()
mergeSortedInto(sorted) { out.add(it) }
return out.toArray()
}
private fun LongArray.isSortedUnique(): Boolean {
@@ -320,6 +349,20 @@ private class LongList {
}
fun toArray(): LongArray = data.copyOf(size)
// Sorts the filled part in place and drops duplicates, without copying; returns the new size
fun sortUnique(): Int {
if (size < 2) return size
java.util.Arrays.sort(data, 0, size)
var n = 1
for (i in 1 until size) if (data[i] != data[n - 1]) data[n++] = data[i]
size = n
return n
}
fun writeTo(out: DataOutputStream) {
for (i in 0 until size) out.writeLong(data[i])
}
}
// Sorts in place and drops duplicates
@@ -375,17 +418,42 @@ private class BlocklistStore(context: Context) {
sp.edit().putString("lists", array.toString()).commit()
}
fun loadListHashes(url: String): LongArray {
val file = binFile(url)
if (!file.exists()) return LongArray(0)
fun loadListHashes(url: String): LongBuffer = map(binFile(url)) ?: EMPTY_SET
// One list is used as it is; several are merged into one file, which is reused at the next
// start as long as the switched-on lists have not changed
fun mergedHashes(lists: List<RemoteList>): LongBuffer {
if (lists.size < 2) {
mergedFile.delete()
return lists.firstOrNull()?.let { loadListHashes(it.url) } ?: EMPTY_SET
}
val signature = hash(lists.sortedBy { it.url }.joinToString("\n") { "${it.url} ${it.count} ${it.updated}" })
if (sp.getString(MERGED_KEY, null) == signature) map(mergedFile)?.let { return it }
val parts = lists.map { loadListHashes(it.url) }
val tmp = File.createTempFile("merged_", ".tmp", dir)
try {
DataOutputStream(BufferedOutputStream(FileOutputStream(tmp), 64 * 1024)).use { out ->
mergeSortedInto(parts) { out.writeLong(it) }
}
if (!tmp.renameTo(mergedFile)) throw IOException("Cannot save lists")
} finally {
tmp.delete()
}
sp.edit().putString(MERGED_KEY, signature).commit()
return map(mergedFile) ?: EMPTY_SET
}
// The mapping stays valid after the file is replaced or deleted, so readers are never cut off
private fun map(file: File): LongBuffer? {
if (!file.exists()) return null
return runCatching {
FileInputStream(file).use { stream ->
val channel = stream.channel
val result = LongArray((channel.size() / 8).toInt())
channel.map(FileChannel.MapMode.READ_ONLY, 0, result.size * 8L).asLongBuffer().get(result)
result
val bytes = channel.size() / 8 * 8
if (bytes == 0L) EMPTY_SET else channel.map(FileChannel.MapMode.READ_ONLY, 0, bytes).asLongBuffer()
}
}.getOrDefault(LongArray(0))
}.getOrNull()
}
fun deleteListFile(url: String) {
@@ -415,22 +483,23 @@ private class BlocklistStore(context: Context) {
parsed.add(DomainHash.of(domain))
}
}
val result = parsed.toArray().sortUnique()
if (result.isEmpty()) throw IOException("No domains found")
write(url, result)
// Sorted inside the download buffer: a list with millions of entries is never copied
val count = parsed.sortUnique()
if (count == 0) throw IOException("No domains found")
write(url, parsed)
sp.edit()
.putOrRemove(etagKey(url), response.header("ETag"))
.putOrRemove(modifiedKey(url), response.header("Last-Modified"))
.apply()
return RemoteList(url, result.size, System.currentTimeMillis())
return RemoteList(url, count, System.currentTimeMillis())
}
}
private fun write(url: String, data: LongArray) {
private fun write(url: String, data: LongList) {
val tmp = File.createTempFile("list_", ".tmp", dir)
try {
DataOutputStream(BufferedOutputStream(FileOutputStream(tmp), 64 * 1024)).use { out ->
data.forEach { out.writeLong(it) }
data.writeTo(out)
}
if (!tmp.renameTo(binFile(url))) throw IOException("Cannot save list")
} finally {
@@ -445,12 +514,14 @@ private class BlocklistStore(context: Context) {
if (value == null) remove(key) else putString(key, value)
private fun binFile(url: String) = File(dir, "list_${hash(url)}.bin")
private val mergedFile get() = File(dir, "merged.bin")
private fun hash(url: String) = MessageDigest.getInstance("SHA-256").digest(url.toByteArray())
.take(8).joinToString("") { "%02x".format(it) }
companion object {
const val MAX_LIST_DOMAINS = 4_000_000
private const val MERGED_KEY = "merged"
val client: OkHttpClient by lazy {
Http.client.newBuilder()
@@ -44,6 +44,12 @@ object Providers {
policy = "https://ffmuc.net/dns-privacy/",
donate = "https://spende.ffmuc.net/",
),
DohProvider(
"dnsforge", "dnsforge.de", "https://blank.dnsforge.de/dns-query",
listOf("78.47.71.194", "138.199.149.249"),
policy = "https://adminforge.de/datenschutz/",
donate = "https://adminforge.de/unterstuetzen/",
),
)
// The project's own server stays the default whatever order the list uses
@@ -5,14 +5,17 @@ import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import java.nio.LongBuffer
class BlocklistMatchTest {
private fun set(vararg domains: String) = domains.map { DomainHash.of(it) }.sorted().toLongArray()
private fun buf(vararg domains: String) = LongBuffer.wrap(set(*domains))
private fun buf(values: LongArray) = LongBuffer.wrap(values)
@Test
fun blocksDomainsAndTheirSubdomains() {
val lists = set("ads.example.com")
val lists = buf("ads.example.com")
val custom = set("tracker.net")
assertTrue(isBlockedIn("ads.example.com", lists, custom))
@@ -21,12 +24,12 @@ class BlocklistMatchTest {
assertFalse(isBlockedIn("example.com", lists, custom))
assertFalse(isBlockedIn("badads.example.com", lists, custom))
assertFalse(isBlockedIn("tracker.network", lists, custom))
assertFalse(isBlockedIn("ads.example.com", LongArray(0), LongArray(0)))
assertFalse(isBlockedIn("ads.example.com", EMPTY_SET, LongArray(0)))
}
@Test
fun skipsTrailingDots() {
val lists = set("example.com")
val lists = buf("example.com")
assertTrue(isBlockedIn("www.example.com..", lists, LongArray(0)))
assertFalse(isBlockedIn("example.org.", lists, LongArray(0)))
@@ -51,8 +54,28 @@ class BlocklistMatchTest {
@Test
fun countsEntriesInBothSetsOnce() {
assertEquals(5, unionSize(longArrayOf(1, 2, 3, 4), longArrayOf(2, 5)))
assertEquals(2, unionSize(LongArray(0), longArrayOf(2, 5)))
assertEquals(4, unionSize(longArrayOf(1, 2, 3, 4), LongArray(0)))
assertEquals(5, unionSize(buf(longArrayOf(1, 2, 3, 4)), longArrayOf(2, 5)))
assertEquals(2, unionSize(EMPTY_SET, longArrayOf(2, 5)))
assertEquals(4, unionSize(buf(longArrayOf(1, 2, 3, 4)), LongArray(0)))
}
@Test
fun streamsMergedValuesInOrderOnce() {
val out = ArrayList<Long>()
val count = mergeSortedInto(
listOf(buf(longArrayOf(-5, 1, 4, 9)), buf(longArrayOf(1, 2, 9, 12)), EMPTY_SET, buf(longArrayOf(12, 40)))
) { out.add(it) }
assertEquals(listOf(-5L, 1, 2, 4, 9, 12, 40), out)
assertEquals(7, count)
assertEquals(0, mergeSortedInto(emptyList()) { out.add(it) })
}
@Test
fun findsValuesInSortedBuffer() {
val values = buf(longArrayOf(Long.MIN_VALUE, -3, 0, 7, Long.MAX_VALUE))
listOf(Long.MIN_VALUE, -3L, 0L, 7L, Long.MAX_VALUE).forEach { assertTrue(values.containsSorted(it)) }
listOf(-4L, 1L, 8L).forEach { assertFalse(values.containsSorted(it)) }
assertFalse(EMPTY_SET.containsSorted(0))
}
}