mirror of
https://github.com/kalarina-bit/Sparkledns.git
synced 2026-11-22 15:46:09 +00:00
Make bootstrap addresses work with HTTP/3 and bound response and list sizes
- HttpEngine resolves the server with system DNS and ignores bootstrap addresses. When it cannot resolve the name and the server has bootstrap addresses, the session switches to OkHttp, which connects to them. - AdBlock and URL rules with a path (||example.com/ads/, |http://example.com/x|) are skipped instead of blocking the whole domain. - A DoH response without Content-Length is read only up to the DNS size limit; blocklist downloads are limited to 256 MB, and lines over 4 KB are skipped without being read into memory.
This commit is contained in:
@@ -12,6 +12,7 @@ import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okio.BufferedSource
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import java.io.BufferedOutputStream
|
||||
@@ -475,13 +476,11 @@ private class BlocklistStore(context: Context) {
|
||||
if (!response.isSuccessful) throw IOException("HTTP ${response.code}")
|
||||
val body = response.body ?: throw IOException("Empty response")
|
||||
val parsed = LongList()
|
||||
body.charStream().buffered().useLines { lines ->
|
||||
lines.forEach { line ->
|
||||
val domain = BlocklistParser.parseLine(line) ?: return@forEach
|
||||
// Far above the largest common lists; keeps a wrong URL from exhausting memory
|
||||
if (parsed.size >= MAX_LIST_DOMAINS) throw IOException("List too large")
|
||||
parsed.add(DomainHash.of(domain))
|
||||
}
|
||||
BlocklistParser.readLimitedLines(body.source()) { line ->
|
||||
val domain = BlocklistParser.parseLine(line) ?: return@readLimitedLines
|
||||
// Far above the largest common lists; keeps a wrong URL from exhausting memory
|
||||
if (parsed.size >= MAX_LIST_DOMAINS) throw IOException("List too large")
|
||||
parsed.add(DomainHash.of(domain))
|
||||
}
|
||||
// Sorted inside the download buffer: a list with millions of entries is never copied
|
||||
val count = parsed.sortUnique()
|
||||
@@ -521,6 +520,7 @@ private class BlocklistStore(context: Context) {
|
||||
|
||||
companion object {
|
||||
const val MAX_LIST_DOMAINS = 4_000_000
|
||||
|
||||
private const val MERGED_KEY = "merged"
|
||||
|
||||
val client: OkHttpClient by lazy {
|
||||
@@ -533,6 +533,59 @@ private class BlocklistStore(context: Context) {
|
||||
|
||||
// Understands hosts files, plain domain lists, AdBlock rules (||domain^) and dnsmasq / unbound syntax
|
||||
object BlocklistParser {
|
||||
/** A list larger than this is refused; the biggest common lists are well below it. */
|
||||
const val MAX_LIST_BYTES = 256L * 1024 * 1024
|
||||
|
||||
/** Longer lines (no blocklist entry is that long) are skipped, not read into memory. */
|
||||
const val MAX_LINE_BYTES = 4096L
|
||||
|
||||
/**
|
||||
* Reads [source] line by line within [MAX_LIST_BYTES] in total and [MAX_LINE_BYTES] per
|
||||
* line, so a wrong URL (a binary file, a page with no line breaks) cannot fill the memory.
|
||||
*/
|
||||
fun readLimitedLines(source: BufferedSource, onLine: (String) -> Unit) {
|
||||
var total = 0L
|
||||
fun consumed(bytes: Long) {
|
||||
total += bytes
|
||||
if (total > MAX_LIST_BYTES) throw IOException("List too large")
|
||||
}
|
||||
while (true) {
|
||||
val newline = source.indexOf('\n'.code.toByte(), 0, MAX_LINE_BYTES + 1)
|
||||
when {
|
||||
newline >= 0 -> {
|
||||
onLine(source.readUtf8(newline))
|
||||
source.skip(1)
|
||||
consumed(newline + 1)
|
||||
}
|
||||
source.exhausted() -> return
|
||||
source.request(MAX_LINE_BYTES + 1) -> {
|
||||
// Overlong line: drop it piece by piece up to its end
|
||||
while (true) {
|
||||
// Only what is already buffered: the next line may start right after the break
|
||||
val end = source.buffer.indexOf('\n'.code.toByte())
|
||||
if (end >= 0) {
|
||||
source.skip(end + 1)
|
||||
consumed(end + 1)
|
||||
break
|
||||
}
|
||||
val chunk = source.buffer.size
|
||||
if (chunk == 0L) return
|
||||
source.skip(chunk)
|
||||
consumed(chunk)
|
||||
if (!source.request(1)) return
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
// Last line without a line break
|
||||
val rest = source.buffer.size
|
||||
onLine(source.readUtf8())
|
||||
consumed(rest)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val WHITESPACE = Regex("\\s+")
|
||||
private val IGNORED = setOf(
|
||||
"localhost", "localhost.localdomain", "local", "broadcasthost",
|
||||
@@ -548,6 +601,8 @@ object BlocklistParser {
|
||||
// Fast path for AdBlock-style lists such as HaGeZi: ||domain^
|
||||
if (line.startsWith("||")) {
|
||||
val end = line.indexOfAny(charArrayOf('^', '$', '/'), 2)
|
||||
// ||example.com/ads/ blocks one path in a browser; as DNS it would block the whole site
|
||||
if (end >= 0 && line[end] == '/' && hasPath(line.substring(end))) return null
|
||||
return normalize(if (end >= 0) line.substring(2, end) else line.substring(2))
|
||||
}
|
||||
if (DNSMASQ.any { line.startsWith(it) }) return normalize(line.substringAfter("=/").substringBefore('/'))
|
||||
@@ -557,9 +612,17 @@ object BlocklistParser {
|
||||
val parts = line.substringBefore('#').trim().split(WHITESPACE)
|
||||
val candidate = if (parts.size >= 2 && looksLikeIp(parts[0])) parts[1] else parts[0]
|
||||
if (candidate.lowercase() in IGNORED) return null
|
||||
// A URL with a path (|http://example.com/x|) stands for one page, not for the domain
|
||||
val afterScheme = candidate.substringAfter("://")
|
||||
val slash = afterScheme.indexOf('/')
|
||||
if (slash >= 0 && hasPath(afterScheme.substring(slash))) return null
|
||||
return normalize(candidate)
|
||||
}
|
||||
|
||||
// "/", "/^" or "/|" only end the host; anything longer is a path
|
||||
private fun hasPath(fromSlash: String): Boolean =
|
||||
fromSlash.trimEnd('^', '|').length > 1
|
||||
|
||||
fun normalize(input: String): String? {
|
||||
var s = input.trim().lowercase()
|
||||
s = s.substringAfter("://")
|
||||
|
||||
@@ -5,6 +5,7 @@ import android.net.ConnectivityManager
|
||||
import android.net.Network
|
||||
import android.net.http.HttpEngine
|
||||
import android.net.http.HttpException
|
||||
import android.net.http.NetworkException
|
||||
import android.net.http.UploadDataProvider
|
||||
import android.net.http.UploadDataSink
|
||||
import android.net.http.UrlRequest
|
||||
@@ -74,7 +75,10 @@ class DohClient(context: Context, provider: DohProvider) {
|
||||
private fun createTransport(context: Context, provider: DohProvider): Transport {
|
||||
if (provider.http3 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||
try {
|
||||
return Http3Transport(context, provider)
|
||||
val http3 = Http3Transport(context, provider)
|
||||
// HttpEngine resolves the host with system DNS and cannot use fixed addresses
|
||||
return if (provider.bootstrap.isEmpty()) http3
|
||||
else BootstrapFallbackTransport(http3) { OkHttpTransport(context, provider) }
|
||||
} catch (e: Throwable) {
|
||||
Log.w(TAG, "HttpEngine unavailable, falling back to HTTP/2", e)
|
||||
}
|
||||
@@ -89,6 +93,38 @@ private interface Transport {
|
||||
fun shutdown()
|
||||
}
|
||||
|
||||
private class UnresolvedHostException(cause: Throwable) : IOException("Host not resolved", cause)
|
||||
|
||||
// HTTP/3 first; once the network fails to resolve the server's name, the rest of the session
|
||||
// goes over OkHttp, which connects to the bootstrap addresses instead
|
||||
private class BootstrapFallbackTransport(
|
||||
private val http3: Transport,
|
||||
createFallback: () -> Transport,
|
||||
) : Transport {
|
||||
private val fallback = lazy(createFallback)
|
||||
@Volatile private var useFallback = false
|
||||
|
||||
override val lastProtocol: String?
|
||||
get() = if (useFallback) fallback.value.lastProtocol else http3.lastProtocol
|
||||
|
||||
override fun query(message: ByteArray): ByteArray {
|
||||
if (!useFallback) {
|
||||
try {
|
||||
return http3.query(message)
|
||||
} catch (e: UnresolvedHostException) {
|
||||
Log.w(TAG, "Server name not resolved over HTTP/3, using bootstrap addresses")
|
||||
useFallback = true
|
||||
}
|
||||
}
|
||||
return fallback.value.query(message)
|
||||
}
|
||||
|
||||
override fun shutdown() {
|
||||
http3.shutdown()
|
||||
if (fallback.isInitialized()) fallback.value.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
private class OkHttpTransport(context: Context, private val provider: DohProvider) : Transport {
|
||||
|
||||
private val url by lazy { provider.url.toHttpUrl() }
|
||||
@@ -140,7 +176,11 @@ private class OkHttpTransport(context: Context, private val provider: DohProvide
|
||||
if (!response.isSuccessful) throw IOException("HTTP ${response.code}")
|
||||
val body = response.body ?: throw IOException("Empty response")
|
||||
if (body.contentLength() > MAX_DNS_SIZE) throw IOException("Response too large")
|
||||
return body.bytes()
|
||||
// Without Content-Length the size is only known while reading: stop one byte past the limit
|
||||
val source = body.source()
|
||||
source.request(MAX_DNS_SIZE + 1L)
|
||||
if (source.buffer.size > MAX_DNS_SIZE) throw IOException("Response too large")
|
||||
return source.buffer.readByteArray()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +260,11 @@ private class Http3Transport(context: Context, private val provider: DohProvider
|
||||
}
|
||||
|
||||
override fun onFailed(request: UrlRequest, info: UrlResponseInfo?, error: HttpException) {
|
||||
result.completeExceptionally(IOException(error.message, error))
|
||||
val unresolved = error is NetworkException &&
|
||||
error.errorCode == NetworkException.ERROR_HOSTNAME_NOT_RESOLVED
|
||||
result.completeExceptionally(
|
||||
if (unresolved) UnresolvedHostException(error) else IOException(error.message, error)
|
||||
)
|
||||
}
|
||||
|
||||
override fun onCanceled(request: UrlRequest, info: UrlResponseInfo?) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package cc.skysparkle.sparkledns
|
||||
|
||||
import okio.Buffer
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Test
|
||||
@@ -51,4 +52,22 @@ class BlocklistParserTest {
|
||||
assertEquals(DomainHash.of(name), DomainHash.of(name, 0))
|
||||
assertEquals(DomainHash.of("b.example"), DomainHash.of(name, 2, name.indexOf(".com")))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rulesWithAPathAreSkippedNotTurnedIntoTheWholeDomain() {
|
||||
assertNull(BlocklistParser.parseLine("||example.com/ads/"))
|
||||
assertNull(BlocklistParser.parseLine("|http://example.com/x|"))
|
||||
assertNull(BlocklistParser.parseLine("https://example.com/banner.js"))
|
||||
assertEquals("example.com", BlocklistParser.parseLine("||example.com/^"))
|
||||
assertEquals("example.com", BlocklistParser.parseLine("https://example.com/"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun overlongLinesAreSkippedAndTheRestIsRead() {
|
||||
val long = "x".repeat(BlocklistParser.MAX_LINE_BYTES.toInt() * 3)
|
||||
val source = Buffer().writeUtf8("a.example\r\n$long\nb.example\nc.example")
|
||||
val lines = mutableListOf<String>()
|
||||
BlocklistParser.readLimitedLines(source) { lines += it.trim() }
|
||||
assertEquals(listOf("a.example", "b.example", "c.example"), lines)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user