mirror of
https://github.com/kalarina-bit/Sparkledns.git
synced 2026-11-22 15:46:09 +00:00
Keep own blocked domains apart from downloaded lists
Adding or removing one own domain re-sorted the whole combined set, allocating 8 bytes per entry of every list each time. Lists are now merged in linear time and own domains are looked up in their own array.
This commit is contained in:
@@ -55,24 +55,16 @@ object Blocklist {
|
|||||||
private val _error = MutableStateFlow<String?>(null)
|
private val _error = MutableStateFlow<String?>(null)
|
||||||
val error: StateFlow<String?> = _error
|
val error: StateFlow<String?> = _error
|
||||||
|
|
||||||
|
// Downloaded lists and own domains are kept apart, so editing a few own domains
|
||||||
|
// does not rebuild a set that can hold millions of entries
|
||||||
@Volatile
|
@Volatile
|
||||||
private var hashes = LongArray(0)
|
|
||||||
private var listHashes = LongArray(0)
|
private var listHashes = LongArray(0)
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var customHashes = LongArray(0)
|
||||||
private var loaded = false
|
private var loaded = false
|
||||||
|
|
||||||
fun isBlocked(name: String): Boolean {
|
fun isBlocked(name: String): Boolean = _enabled.value && isBlockedIn(name, listHashes, customHashes)
|
||||||
if (!_enabled.value) return false
|
|
||||||
val set = hashes
|
|
||||||
if (set.isEmpty()) return false
|
|
||||||
val domain = name.trimEnd('.')
|
|
||||||
var start = 0
|
|
||||||
while (true) {
|
|
||||||
if (set.binarySearch(DomainHash.of(domain, start)) >= 0) return true
|
|
||||||
val dot = domain.indexOf('.', start)
|
|
||||||
if (dot < 0) return false
|
|
||||||
start = dot + 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fun reload(context: Context) = synchronized(lock) { load(BlocklistStore(context)) }
|
fun reload(context: Context) = synchronized(lock) { load(BlocklistStore(context)) }
|
||||||
|
|
||||||
@@ -88,7 +80,7 @@ object Blocklist {
|
|||||||
val updated = (_custom.value + domain).distinct().sorted()
|
val updated = (_custom.value + domain).distinct().sorted()
|
||||||
store.saveCustom(updated)
|
store.saveCustom(updated)
|
||||||
_custom.value = updated
|
_custom.value = updated
|
||||||
combine()
|
updateCustomHashes()
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,7 +90,7 @@ object Blocklist {
|
|||||||
val updated = _custom.value - domain
|
val updated = _custom.value - domain
|
||||||
store.saveCustom(updated)
|
store.saveCustom(updated)
|
||||||
_custom.value = updated
|
_custom.value = updated
|
||||||
combine()
|
updateCustomHashes()
|
||||||
}
|
}
|
||||||
|
|
||||||
fun removeList(context: Context, url: String) = synchronized(lock) {
|
fun removeList(context: Context, url: String) = synchronized(lock) {
|
||||||
@@ -175,35 +167,76 @@ object Blocklist {
|
|||||||
_enabled.value = store.enabled
|
_enabled.value = store.enabled
|
||||||
_custom.value = store.loadCustom()
|
_custom.value = store.loadCustom()
|
||||||
_lists.value = store.loadLists()
|
_lists.value = store.loadLists()
|
||||||
|
val custom = _custom.value
|
||||||
|
customHashes = LongArray(custom.size) { DomainHash.of(custom[it]) }.sortUnique()
|
||||||
reloadListHashes(store)
|
reloadListHashes(store)
|
||||||
loaded = true
|
loaded = true
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun reloadListHashes(store: BlocklistStore) {
|
private fun reloadListHashes(store: BlocklistStore) {
|
||||||
listHashes = merge(_lists.value.map { store.loadListHashes(it.url) })
|
listHashes = mergeSorted(_lists.value.map { store.loadListHashes(it.url) })
|
||||||
combine()
|
_total.value = unionSize(listHashes, customHashes)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun combine() {
|
private fun updateCustomHashes() {
|
||||||
val custom = _custom.value
|
val custom = _custom.value
|
||||||
hashes = if (custom.isEmpty()) {
|
customHashes = LongArray(custom.size) { DomainHash.of(custom[it]) }.sortUnique()
|
||||||
listHashes
|
_total.value = unionSize(listHashes, customHashes)
|
||||||
} else {
|
|
||||||
merge(listOf(listHashes, LongArray(custom.size) { DomainHash.of(custom[it]) }))
|
|
||||||
}
|
|
||||||
_total.value = hashes.size
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun merge(parts: List<LongArray>): LongArray {
|
// True when the name or one of its parent domains is in either sorted hash set
|
||||||
val nonEmpty = parts.filter { it.isNotEmpty() }
|
internal fun isBlockedIn(name: String, lists: LongArray, custom: LongArray): Boolean {
|
||||||
if (nonEmpty.size <= 1) return nonEmpty.firstOrNull() ?: LongArray(0) // stored lists are already sorted
|
if (lists.isEmpty() && custom.isEmpty()) return false
|
||||||
val all = LongArray(nonEmpty.sumOf { it.size })
|
val domain = name.trimEnd('.')
|
||||||
var pos = 0
|
var start = 0
|
||||||
nonEmpty.forEach { it.copyInto(all, pos); pos += it.size }
|
while (true) {
|
||||||
return all.sortUnique()
|
val hash = DomainHash.of(domain, start)
|
||||||
|
if (lists.binarySearch(hash) >= 0 || custom.binarySearch(hash) >= 0) return true
|
||||||
|
val dot = domain.indexOf('.', start)
|
||||||
|
if (dot < 0) return false
|
||||||
|
start = dot + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Number of distinct entries in both sorted sets; the custom set is small
|
||||||
|
internal fun unionSize(lists: LongArray, custom: LongArray): Int =
|
||||||
|
lists.size + custom.count { lists.binarySearch(it) < 0 }
|
||||||
|
|
||||||
|
// Merges sorted, duplicate-free arrays in linear time instead of sorting their concatenation
|
||||||
|
internal fun mergeSorted(parts: List<LongArray>): LongArray {
|
||||||
|
var arrays = parts.filter { it.isNotEmpty() }.map { if (it.isSortedUnique()) it else it.copyOf().sortUnique() }
|
||||||
|
if (arrays.isEmpty()) return LongArray(0)
|
||||||
|
while (arrays.size > 1) {
|
||||||
|
arrays = arrays.chunked(2).map { if (it.size == 2) mergeTwo(it[0], it[1]) else it[0] }
|
||||||
|
}
|
||||||
|
return arrays[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun mergeTwo(a: LongArray, b: LongArray): LongArray {
|
||||||
|
val out = LongArray(a.size + b.size)
|
||||||
|
var i = 0
|
||||||
|
var j = 0
|
||||||
|
var n = 0
|
||||||
|
while (i < a.size && j < b.size) {
|
||||||
|
val x = a[i]
|
||||||
|
val y = b[j]
|
||||||
|
when {
|
||||||
|
x < y -> { out[n++] = x; i++ }
|
||||||
|
x > y -> { out[n++] = y; j++ }
|
||||||
|
else -> { out[n++] = x; i++; j++ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
while (i < a.size) out[n++] = a[i++]
|
||||||
|
while (j < b.size) out[n++] = b[j++]
|
||||||
|
return if (n == out.size) out else out.copyOf(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun LongArray.isSortedUnique(): Boolean {
|
||||||
|
for (i in 1 until size) if (this[i - 1] >= this[i]) return false
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
internal object DomainHash {
|
internal object DomainHash {
|
||||||
private const val FNV_OFFSET = -3750763034362895579L // 0xcbf29ce484222325
|
private const val FNV_OFFSET = -3750763034362895579L // 0xcbf29ce484222325
|
||||||
private const val FNV_PRIME = 1099511628211L
|
private const val FNV_PRIME = 1099511628211L
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package cc.skysparkle.sparkledns
|
||||||
|
|
||||||
|
import org.junit.Assert.assertArrayEquals
|
||||||
|
import org.junit.Assert.assertEquals
|
||||||
|
import org.junit.Assert.assertFalse
|
||||||
|
import org.junit.Assert.assertTrue
|
||||||
|
import org.junit.Test
|
||||||
|
|
||||||
|
class BlocklistMatchTest {
|
||||||
|
|
||||||
|
private fun set(vararg domains: String) = domains.map { DomainHash.of(it) }.sorted().toLongArray()
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun blocksDomainsAndTheirSubdomains() {
|
||||||
|
val lists = set("ads.example.com")
|
||||||
|
val custom = set("tracker.net")
|
||||||
|
|
||||||
|
assertTrue(isBlockedIn("ads.example.com", lists, custom))
|
||||||
|
assertTrue(isBlockedIn("a.b.ads.example.com.", lists, custom))
|
||||||
|
assertTrue(isBlockedIn("cdn.tracker.net", lists, custom))
|
||||||
|
assertFalse(isBlockedIn("example.com", lists, custom))
|
||||||
|
assertFalse(isBlockedIn("badads.example.com", lists, custom))
|
||||||
|
assertFalse(isBlockedIn("tracker.network", lists, custom))
|
||||||
|
assertFalse(isBlockedIn("ads.example.com", LongArray(0), LongArray(0)))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun mergesSortedListsWithoutDuplicates() {
|
||||||
|
val a = longArrayOf(-5, 1, 4, 9)
|
||||||
|
val b = longArrayOf(1, 2, 9, 12)
|
||||||
|
val c = longArrayOf(3)
|
||||||
|
|
||||||
|
assertArrayEquals(longArrayOf(-5, 1, 2, 3, 4, 9, 12), mergeSorted(listOf(a, b, c, LongArray(0))))
|
||||||
|
assertArrayEquals(LongArray(0), mergeSorted(emptyList()))
|
||||||
|
assertArrayEquals(a, mergeSorted(listOf(a)))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun sortsInputsThatAreNotSorted() {
|
||||||
|
assertArrayEquals(longArrayOf(1, 2, 3, 7), mergeSorted(listOf(longArrayOf(3, 1, 3), longArrayOf(7, 2))))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun countsEntriesInBothSetsOnce() {
|
||||||
|
assertEquals(5, unionSize(longArrayOf(1, 2, 3, 4), longArrayOf(2, 5)))
|
||||||
|
assertEquals(2, unionSize(LongArray(0), longArrayOf(2, 5)))
|
||||||
|
assertEquals(4, unionSize(longArrayOf(1, 2, 3, 4), LongArray(0)))
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user