Make bootstrap addresses work with HTTP/3 and bound response and list sizes

- HttpEngine resolves the server with system DNS and ignores bootstrap
  addresses. When it cannot resolve the name and the server has bootstrap
  addresses, the session switches to OkHttp, which connects to them.
- AdBlock and URL rules with a path (||example.com/ads/, |http://example.com/x|)
  are skipped instead of blocking the whole domain.
- A DoH response without Content-Length is read only up to the DNS size
  limit; blocklist downloads are limited to 256 MB, and lines over 4 KB are
  skipped without being read into memory.
This commit is contained in:
2026-11-12 17:24:21 +02:00
parent 6d015e9527
commit 2578af2bed
3 changed files with 136 additions and 10 deletions
@@ -12,6 +12,7 @@ import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import okhttp3.OkHttpClient import okhttp3.OkHttpClient
import okhttp3.Request import okhttp3.Request
import okio.BufferedSource
import org.json.JSONArray import org.json.JSONArray
import org.json.JSONObject import org.json.JSONObject
import java.io.BufferedOutputStream import java.io.BufferedOutputStream
@@ -475,13 +476,11 @@ private class BlocklistStore(context: Context) {
if (!response.isSuccessful) throw IOException("HTTP ${response.code}") if (!response.isSuccessful) throw IOException("HTTP ${response.code}")
val body = response.body ?: throw IOException("Empty response") val body = response.body ?: throw IOException("Empty response")
val parsed = LongList() val parsed = LongList()
body.charStream().buffered().useLines { lines -> BlocklistParser.readLimitedLines(body.source()) { line ->
lines.forEach { line -> val domain = BlocklistParser.parseLine(line) ?: return@readLimitedLines
val domain = BlocklistParser.parseLine(line) ?: return@forEach // Far above the largest common lists; keeps a wrong URL from exhausting memory
// Far above the largest common lists; keeps a wrong URL from exhausting memory if (parsed.size >= MAX_LIST_DOMAINS) throw IOException("List too large")
if (parsed.size >= MAX_LIST_DOMAINS) throw IOException("List too large") parsed.add(DomainHash.of(domain))
parsed.add(DomainHash.of(domain))
}
} }
// Sorted inside the download buffer: a list with millions of entries is never copied // Sorted inside the download buffer: a list with millions of entries is never copied
val count = parsed.sortUnique() val count = parsed.sortUnique()
@@ -521,6 +520,7 @@ private class BlocklistStore(context: Context) {
companion object { companion object {
const val MAX_LIST_DOMAINS = 4_000_000 const val MAX_LIST_DOMAINS = 4_000_000
private const val MERGED_KEY = "merged" private const val MERGED_KEY = "merged"
val client: OkHttpClient by lazy { val client: OkHttpClient by lazy {
@@ -533,6 +533,59 @@ private class BlocklistStore(context: Context) {
// Understands hosts files, plain domain lists, AdBlock rules (||domain^) and dnsmasq / unbound syntax // Understands hosts files, plain domain lists, AdBlock rules (||domain^) and dnsmasq / unbound syntax
object BlocklistParser { object BlocklistParser {
/** A list larger than this is refused; the biggest common lists are well below it. */
const val MAX_LIST_BYTES = 256L * 1024 * 1024
/** Longer lines (no blocklist entry is that long) are skipped, not read into memory. */
const val MAX_LINE_BYTES = 4096L
/**
* Reads [source] line by line within [MAX_LIST_BYTES] in total and [MAX_LINE_BYTES] per
* line, so a wrong URL (a binary file, a page with no line breaks) cannot fill the memory.
*/
fun readLimitedLines(source: BufferedSource, onLine: (String) -> Unit) {
var total = 0L
fun consumed(bytes: Long) {
total += bytes
if (total > MAX_LIST_BYTES) throw IOException("List too large")
}
while (true) {
val newline = source.indexOf('\n'.code.toByte(), 0, MAX_LINE_BYTES + 1)
when {
newline >= 0 -> {
onLine(source.readUtf8(newline))
source.skip(1)
consumed(newline + 1)
}
source.exhausted() -> return
source.request(MAX_LINE_BYTES + 1) -> {
// Overlong line: drop it piece by piece up to its end
while (true) {
// Only what is already buffered: the next line may start right after the break
val end = source.buffer.indexOf('\n'.code.toByte())
if (end >= 0) {
source.skip(end + 1)
consumed(end + 1)
break
}
val chunk = source.buffer.size
if (chunk == 0L) return
source.skip(chunk)
consumed(chunk)
if (!source.request(1)) return
}
}
else -> {
// Last line without a line break
val rest = source.buffer.size
onLine(source.readUtf8())
consumed(rest)
return
}
}
}
}
private val WHITESPACE = Regex("\\s+") private val WHITESPACE = Regex("\\s+")
private val IGNORED = setOf( private val IGNORED = setOf(
"localhost", "localhost.localdomain", "local", "broadcasthost", "localhost", "localhost.localdomain", "local", "broadcasthost",
@@ -548,6 +601,8 @@ object BlocklistParser {
// Fast path for AdBlock-style lists such as HaGeZi: ||domain^ // Fast path for AdBlock-style lists such as HaGeZi: ||domain^
if (line.startsWith("||")) { if (line.startsWith("||")) {
val end = line.indexOfAny(charArrayOf('^', '$', '/'), 2) val end = line.indexOfAny(charArrayOf('^', '$', '/'), 2)
// ||example.com/ads/ blocks one path in a browser; as DNS it would block the whole site
if (end >= 0 && line[end] == '/' && hasPath(line.substring(end))) return null
return normalize(if (end >= 0) line.substring(2, end) else line.substring(2)) return normalize(if (end >= 0) line.substring(2, end) else line.substring(2))
} }
if (DNSMASQ.any { line.startsWith(it) }) return normalize(line.substringAfter("=/").substringBefore('/')) if (DNSMASQ.any { line.startsWith(it) }) return normalize(line.substringAfter("=/").substringBefore('/'))
@@ -557,9 +612,17 @@ object BlocklistParser {
val parts = line.substringBefore('#').trim().split(WHITESPACE) val parts = line.substringBefore('#').trim().split(WHITESPACE)
val candidate = if (parts.size >= 2 && looksLikeIp(parts[0])) parts[1] else parts[0] val candidate = if (parts.size >= 2 && looksLikeIp(parts[0])) parts[1] else parts[0]
if (candidate.lowercase() in IGNORED) return null if (candidate.lowercase() in IGNORED) return null
// A URL with a path (|http://example.com/x|) stands for one page, not for the domain
val afterScheme = candidate.substringAfter("://")
val slash = afterScheme.indexOf('/')
if (slash >= 0 && hasPath(afterScheme.substring(slash))) return null
return normalize(candidate) return normalize(candidate)
} }
// "/", "/^" or "/|" only end the host; anything longer is a path
private fun hasPath(fromSlash: String): Boolean =
fromSlash.trimEnd('^', '|').length > 1
fun normalize(input: String): String? { fun normalize(input: String): String? {
var s = input.trim().lowercase() var s = input.trim().lowercase()
s = s.substringAfter("://") s = s.substringAfter("://")
@@ -5,6 +5,7 @@ import android.net.ConnectivityManager
import android.net.Network import android.net.Network
import android.net.http.HttpEngine import android.net.http.HttpEngine
import android.net.http.HttpException import android.net.http.HttpException
import android.net.http.NetworkException
import android.net.http.UploadDataProvider import android.net.http.UploadDataProvider
import android.net.http.UploadDataSink import android.net.http.UploadDataSink
import android.net.http.UrlRequest import android.net.http.UrlRequest
@@ -74,7 +75,10 @@ class DohClient(context: Context, provider: DohProvider) {
private fun createTransport(context: Context, provider: DohProvider): Transport { private fun createTransport(context: Context, provider: DohProvider): Transport {
if (provider.http3 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { if (provider.http3 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
try { try {
return Http3Transport(context, provider) val http3 = Http3Transport(context, provider)
// HttpEngine resolves the host with system DNS and cannot use fixed addresses
return if (provider.bootstrap.isEmpty()) http3
else BootstrapFallbackTransport(http3) { OkHttpTransport(context, provider) }
} catch (e: Throwable) { } catch (e: Throwable) {
Log.w(TAG, "HttpEngine unavailable, falling back to HTTP/2", e) Log.w(TAG, "HttpEngine unavailable, falling back to HTTP/2", e)
} }
@@ -89,6 +93,38 @@ private interface Transport {
fun shutdown() fun shutdown()
} }
private class UnresolvedHostException(cause: Throwable) : IOException("Host not resolved", cause)
// HTTP/3 first; once the network fails to resolve the server's name, the rest of the session
// goes over OkHttp, which connects to the bootstrap addresses instead
private class BootstrapFallbackTransport(
private val http3: Transport,
createFallback: () -> Transport,
) : Transport {
private val fallback = lazy(createFallback)
@Volatile private var useFallback = false
override val lastProtocol: String?
get() = if (useFallback) fallback.value.lastProtocol else http3.lastProtocol
override fun query(message: ByteArray): ByteArray {
if (!useFallback) {
try {
return http3.query(message)
} catch (e: UnresolvedHostException) {
Log.w(TAG, "Server name not resolved over HTTP/3, using bootstrap addresses")
useFallback = true
}
}
return fallback.value.query(message)
}
override fun shutdown() {
http3.shutdown()
if (fallback.isInitialized()) fallback.value.shutdown()
}
}
private class OkHttpTransport(context: Context, private val provider: DohProvider) : Transport { private class OkHttpTransport(context: Context, private val provider: DohProvider) : Transport {
private val url by lazy { provider.url.toHttpUrl() } private val url by lazy { provider.url.toHttpUrl() }
@@ -140,7 +176,11 @@ private class OkHttpTransport(context: Context, private val provider: DohProvide
if (!response.isSuccessful) throw IOException("HTTP ${response.code}") if (!response.isSuccessful) throw IOException("HTTP ${response.code}")
val body = response.body ?: throw IOException("Empty response") val body = response.body ?: throw IOException("Empty response")
if (body.contentLength() > MAX_DNS_SIZE) throw IOException("Response too large") if (body.contentLength() > MAX_DNS_SIZE) throw IOException("Response too large")
return body.bytes() // Without Content-Length the size is only known while reading: stop one byte past the limit
val source = body.source()
source.request(MAX_DNS_SIZE + 1L)
if (source.buffer.size > MAX_DNS_SIZE) throw IOException("Response too large")
return source.buffer.readByteArray()
} }
} }
@@ -220,7 +260,11 @@ private class Http3Transport(context: Context, private val provider: DohProvider
} }
override fun onFailed(request: UrlRequest, info: UrlResponseInfo?, error: HttpException) { override fun onFailed(request: UrlRequest, info: UrlResponseInfo?, error: HttpException) {
result.completeExceptionally(IOException(error.message, error)) val unresolved = error is NetworkException &&
error.errorCode == NetworkException.ERROR_HOSTNAME_NOT_RESOLVED
result.completeExceptionally(
if (unresolved) UnresolvedHostException(error) else IOException(error.message, error)
)
} }
override fun onCanceled(request: UrlRequest, info: UrlResponseInfo?) { override fun onCanceled(request: UrlRequest, info: UrlResponseInfo?) {
@@ -1,5 +1,6 @@
package cc.skysparkle.sparkledns package cc.skysparkle.sparkledns
import okio.Buffer
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull import org.junit.Assert.assertNull
import org.junit.Test import org.junit.Test
@@ -51,4 +52,22 @@ class BlocklistParserTest {
assertEquals(DomainHash.of(name), DomainHash.of(name, 0)) assertEquals(DomainHash.of(name), DomainHash.of(name, 0))
assertEquals(DomainHash.of("b.example"), DomainHash.of(name, 2, name.indexOf(".com"))) assertEquals(DomainHash.of("b.example"), DomainHash.of(name, 2, name.indexOf(".com")))
} }
@Test
fun rulesWithAPathAreSkippedNotTurnedIntoTheWholeDomain() {
assertNull(BlocklistParser.parseLine("||example.com/ads/"))
assertNull(BlocklistParser.parseLine("|http://example.com/x|"))
assertNull(BlocklistParser.parseLine("https://example.com/banner.js"))
assertEquals("example.com", BlocklistParser.parseLine("||example.com/^"))
assertEquals("example.com", BlocklistParser.parseLine("https://example.com/"))
}
@Test
fun overlongLinesAreSkippedAndTheRestIsRead() {
val long = "x".repeat(BlocklistParser.MAX_LINE_BYTES.toInt() * 3)
val source = Buffer().writeUtf8("a.example\r\n$long\nb.example\nc.example")
val lines = mutableListOf<String>()
BlocklistParser.readLimitedLines(source) { lines += it.trim() }
assertEquals(listOf("a.example", "b.example", "c.example"), lines)
}
} }