Encode statistics on the writer thread and read them outside the lock

This commit is contained in:
2026-10-31 11:40:43 +02:00
parent 60c43df3c5
commit 6444a81f95
2 changed files with 281 additions and 94 deletions
@@ -4,12 +4,11 @@ import android.content.Context
import android.util.Log import android.util.Log
import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.StateFlow
import java.io.BufferedInputStream import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.io.DataInputStream import java.io.DataInputStream
import java.io.DataOutputStream import java.io.DataOutputStream
import java.io.File import java.io.File
import java.io.FileInputStream
import java.util.concurrent.Executors import java.util.concurrent.Executors
class StatsSummary( class StatsSummary(
@@ -26,24 +25,11 @@ class HourPoint(val total: Long, val blocked: Long)
// Hourly counters of the last 7 days for the charts. Only numbers are kept, never domains, // Hourly counters of the last 7 days for the charts. Only numbers are kept, never domains,
// and they never leave the device. // and they never leave the device.
object StatsStore { object StatsStore {
private const val HOURS = 24 * 7
private const val HOUR_MS = 3_600_000L private const val HOUR_MS = 3_600_000L
private const val FORMAT = 1
private const val FILE = "stats.bin" private const val FILE = "stats.bin"
private const val SAVE_INTERVAL_MS = 60_000L private const val SAVE_INTERVAL_MS = 60_000L
// Per hour: total, blocked, cached, latency sum and count, then one counter per company private val counters = HourlyCounters()
private const val TOTAL = 0
private const val BLOCKED = 1
private const val CACHED = 2
private const val LATENCY_SUM = 3
private const val LATENCY_COUNT = 4
private const val FIRST_COMPANY = 5
private val FIELDS = FIRST_COMPANY + Company.entries.size
// A ring of slots; each remembers which hour (since the epoch) it currently holds
private val hourIds = LongArray(HOURS) { -1L }
private val counters = Array(HOURS) { LongArray(FIELDS) }
private var file: File? = null private var file: File? = null
// Bumped when the numbers change without a new query: loaded from disk // Bumped when the numbers change without a new query: loaded from disk
@@ -52,21 +38,110 @@ object StatsStore {
private var lastSave = 0L private var lastSave = 0L
private var dirty = false private var dirty = false
// Counts every change, so summary() can hand out its last result while nothing changed
private var changes = 0L
private var lastSummary: StatsSummary? = null
private var summaryChanges = 0L
private var summaryHours = 0
private var summaryHour = 0L
// Saves happen off the VPN threads, one at a time // Saves happen off the VPN threads, one at a time
private val writer = Executors.newSingleThreadExecutor { Thread(it, "stats-writer").apply { isDaemon = true } } private val writer = Executors.newSingleThreadExecutor { Thread(it, "stats-writer").apply { isDaemon = true } }
// The file is read outside the lock, so neither the DNS threads nor the screens wait for
// storage. Queries counted before it was read are added to the saved numbers.
fun init(context: Context) { fun init(context: Context) {
val target = File(context.filesDir, FILE) val target = File(context.filesDir, FILE)
val saved = read(target)
synchronized(this) { synchronized(this) {
file = target file = target
load(target) if (saved != null) counters.merge(saved)
changes++
} }
_version.value++ _version.value++
} }
fun record(status: QueryStatus, latencyMs: Int, company: Company?, now: Long = System.currentTimeMillis()) { fun record(status: QueryStatus, latencyMs: Int, company: Company?, now: Long = System.currentTimeMillis()) {
val snapshot = synchronized(this) { val save = synchronized(this) {
val row = row(now / HOUR_MS) counters.record(status, latencyMs, company, now / HOUR_MS)
changes++
dirty = true
if (file != null && now - lastSave >= SAVE_INTERVAL_MS) {
lastSave = now
true
} else {
false
}
}
// The writer thread encodes the numbers itself, so this DNS thread only hands the job over
if (save) writer.execute(::save)
}
// Totals of the last [hours] hours, the current one included. While nothing changed the same
// object comes back, so a screen refreshed with it has nothing to recompose.
fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) {
val current = now / HOUR_MS
lastSummary?.takeIf { summaryChanges == changes && summaryHours == hours && summaryHour == current }
?: counters.summary(hours, current).also {
lastSummary = it
summaryChanges = changes
summaryHours = hours
summaryHour = current
}
}
// One point per hour, oldest first, ending with the current hour
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> =
synchronized(this) { counters.hourly(hours, now / HOUR_MS) }
// Saves what is not on disk yet; called when the VPN stops
fun flush() {
val save = synchronized(this) {
if (file != null && dirty) {
lastSave = System.currentTimeMillis()
true
} else {
false
}
}
if (save) writer.execute(::save)
}
// Runs on the writer thread
private fun save() {
val (target, bytes) = synchronized(this) {
val current = file ?: return
dirty = false
current to counters.encode()
}
try {
val tmp = File(target.parentFile, "$FILE.tmp")
tmp.writeBytes(bytes)
if (!tmp.renameTo(target)) tmp.delete()
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot save statistics", e)
}
}
private fun read(source: File): HourlyCounters? {
if (!source.exists()) return null
return try {
HourlyCounters.decode(source.readBytes())
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot read statistics", e)
null
}
}
}
// The numbers behind StatsStore: a ring of hourly rows, each remembering which hour (since the
// epoch) it holds. Plain data without locks or Android; StatsStore takes care of both.
internal class HourlyCounters {
val hourIds = LongArray(HOURS) { -1L }
val rows = Array(HOURS) { LongArray(FIELDS) }
fun record(status: QueryStatus, latencyMs: Int, company: Company?, hour: Long) {
val row = row(hour)
row[TOTAL]++ row[TOTAL]++
when (status) { when (status) {
QueryStatus.BLOCKED -> row[BLOCKED]++ QueryStatus.BLOCKED -> row[BLOCKED]++
@@ -78,30 +153,19 @@ object StatsStore {
QueryStatus.FAILED -> Unit QueryStatus.FAILED -> Unit
} }
if (company != null) row[FIRST_COMPANY + company.ordinal]++ if (company != null) row[FIRST_COMPANY + company.ordinal]++
dirty = true
if (file != null && now - lastSave >= SAVE_INTERVAL_MS) {
lastSave = now
dirty = false
encode()
} else {
null
}
}
snapshot?.let(::save)
} }
// Totals of the last [hours] hours, the current one included // Totals of the last [hours] hours up to [currentHour]
fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) { fun summary(hours: Int, currentHour: Long): StatsSummary {
val current = now / HOUR_MS
val sums = LongArray(FIELDS) val sums = LongArray(FIELDS)
for (back in 0 until hours.coerceIn(1, HOURS)) { for (back in 0 until hours.coerceIn(1, HOURS)) {
val id = current - back val id = currentHour - back
val index = slot(id) val index = slot(id)
if (hourIds[index] != id) continue if (hourIds[index] != id) continue
val row = counters[index] val row = rows[index]
for (field in 0 until FIELDS) sums[field] += row[field] for (field in 0 until FIELDS) sums[field] += row[field]
} }
StatsSummary( return StatsSummary(
total = sums[TOTAL], total = sums[TOTAL],
blocked = sums[BLOCKED], blocked = sums[BLOCKED],
cached = sums[CACHED], cached = sums[CACHED],
@@ -110,29 +174,43 @@ object StatsStore {
) )
} }
// One point per hour, oldest first, ending with the current hour // One point per hour, oldest first, ending with [currentHour]
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> = synchronized(this) { fun hourly(hours: Int, currentHour: Long): List<HourPoint> {
val count = hours.coerceIn(1, HOURS) val count = hours.coerceIn(1, HOURS)
val current = now / HOUR_MS return List(count) { position ->
List(count) { position -> val id = currentHour - (count - 1 - position)
val id = current - (count - 1 - position)
val index = slot(id) val index = slot(id)
if (hourIds[index] == id) HourPoint(counters[index][TOTAL], counters[index][BLOCKED]) else HourPoint(0, 0) if (hourIds[index] == id) HourPoint(rows[index][TOTAL], rows[index][BLOCKED]) else HourPoint(0, 0)
} }
} }
// Saves what is not on disk yet; called when the VPN stops // Saved hours are added to the ones counted since the start; a slot keeps the newer hour
fun flush() { fun merge(saved: HourlyCounters) {
val snapshot = synchronized(this) { for (index in 0 until HOURS) {
if (file != null && dirty) { val id = saved.hourIds[index]
dirty = false when {
lastSave = System.currentTimeMillis() id < 0 || id < hourIds[index] -> Unit
encode() id == hourIds[index] -> for (field in 0 until FIELDS) rows[index][field] += saved.rows[index][field]
} else { else -> {
null hourIds[index] = id
saved.rows[index].copyInto(rows[index])
} }
} }
snapshot?.let(::save) }
}
fun encode(): ByteArray {
val bytes = ByteArrayOutputStream(12 + HOURS * (FIELDS + 1) * 8)
DataOutputStream(bytes).use { out ->
out.writeInt(FORMAT)
out.writeInt(FIELDS)
out.writeInt(HOURS)
for (index in 0 until HOURS) {
out.writeLong(hourIds[index])
rows[index].forEach { out.writeLong(it) }
}
}
return bytes.toByteArray()
} }
private fun slot(id: Long): Int = (((id % HOURS) + HOURS) % HOURS).toInt() private fun slot(id: Long): Int = (((id % HOURS) + HOURS) % HOURS).toInt()
@@ -141,58 +219,41 @@ object StatsStore {
val index = slot(id) val index = slot(id)
if (hourIds[index] != id) { if (hourIds[index] != id) {
hourIds[index] = id hourIds[index] = id
counters[index].fill(0L) rows[index].fill(0L)
} }
return counters[index] return rows[index]
} }
private fun encode(): ByteArray { companion object {
val bytes = ByteArrayOutputStream() const val HOURS = 24 * 7
DataOutputStream(bytes).use { out -> private const val FORMAT = 1
out.writeInt(FORMAT)
out.writeInt(FIELDS)
out.writeInt(HOURS)
for (index in 0 until HOURS) {
out.writeLong(hourIds[index])
counters[index].forEach { out.writeLong(it) }
}
}
return bytes.toByteArray()
}
private fun save(bytes: ByteArray) { // Per hour: total, blocked, cached, latency sum and count, then one counter per company
val target = file ?: return private const val TOTAL = 0
writer.execute { private const val BLOCKED = 1
try { private const val CACHED = 2
val tmp = File(target.parentFile, "$FILE.tmp") private const val LATENCY_SUM = 3
tmp.writeBytes(bytes) private const val LATENCY_COUNT = 4
if (!tmp.renameTo(target)) tmp.delete() const val FIRST_COMPANY = 5
} catch (e: Exception) { val FIELDS = FIRST_COMPANY + Company.entries.size
Log.w("SparkleDNS", "Cannot save statistics", e)
}
}
}
// Files from a version with fewer companies load fine: the new counters start at zero // Files from a version with fewer companies load fine: the new counters start at zero.
private fun load(source: File) { // Null for a file of another format.
if (!source.exists()) return fun decode(bytes: ByteArray): HourlyCounters? {
try { DataInputStream(ByteArrayInputStream(bytes)).use { input ->
DataInputStream(BufferedInputStream(FileInputStream(source))).use { input -> if (input.readInt() != FORMAT) return null
if (input.readInt() != FORMAT) return
val fields = input.readInt() val fields = input.readInt()
if (input.readInt() != HOURS || fields < FIRST_COMPANY) return if (input.readInt() != HOURS || fields < FIRST_COMPANY) return null
val result = HourlyCounters()
for (index in 0 until HOURS) { for (index in 0 until HOURS) {
hourIds[index] = input.readLong() result.hourIds[index] = input.readLong()
for (field in 0 until fields) { for (field in 0 until fields) {
val value = input.readLong() val value = input.readLong()
if (field < FIELDS) counters[index][field] = value if (field < FIELDS) result.rows[index][field] = value
}
}
return result
} }
} }
} }
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot read statistics", e)
hourIds.fill(-1L)
counters.forEach { it.fill(0L) }
}
}
} }
@@ -0,0 +1,126 @@
package cc.skysparkle.sparkledns
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.DataInputStream
import java.io.DataOutputStream
class HourlyCountersTest {
private val hours = HourlyCounters.HOURS
@Test
fun countsQueriesPerHour() {
val counters = HourlyCounters()
counters.record(QueryStatus.OK, 40, Company.GOOGLE, hour = 100)
counters.record(QueryStatus.OK, 20, null, hour = 100)
counters.record(QueryStatus.BLOCKED, 0, Company.META, hour = 100)
counters.record(QueryStatus.CACHED, 0, null, hour = 99)
counters.record(QueryStatus.FAILED, 5000, null, hour = 99)
val day = counters.summary(24, currentHour = 100)
assertEquals(5L, day.total)
assertEquals(1L, day.blocked)
assertEquals(1L, day.cached)
assertEquals(30, day.avgLatencyMs)
assertEquals(1L, day.companies[Company.GOOGLE.ordinal])
assertEquals(1L, day.companies[Company.META.ordinal])
assertEquals(3L, counters.summary(1, currentHour = 100).total)
val points = counters.hourly(3, currentHour = 100)
assertEquals(listOf(0L, 2L, 3L), points.map { it.total })
assertEquals(listOf(0L, 0L, 1L), points.map { it.blocked })
}
@Test
fun aSlotStartsOverAWeekLater() {
val counters = HourlyCounters()
counters.record(QueryStatus.OK, 10, null, hour = 5)
counters.record(QueryStatus.OK, 10, null, hour = 5L + hours)
assertEquals(1L, counters.summary(hours, currentHour = 5L + hours).total)
assertEquals(0L, counters.summary(1, currentHour = 5).total)
}
@Test
fun savedNumbersReadBackUnchanged() {
val counters = HourlyCounters()
counters.record(QueryStatus.BLOCKED, 0, Company.XIAOMI, hour = 1000)
counters.record(QueryStatus.OK, 12, Company.APPLE, hour = 1001)
val copy = requireNotNull(HourlyCounters.decode(counters.encode()))
assertArrayEquals(counters.hourIds, copy.hourIds)
for (index in 0 until hours) assertArrayEquals(counters.rows[index], copy.rows[index])
}
@Test
fun writesFormatOne() {
val bytes = HourlyCounters().encode()
val input = DataInputStream(ByteArrayInputStream(bytes))
assertEquals(1, input.readInt())
assertEquals(HourlyCounters.FIELDS, input.readInt())
assertEquals(hours, input.readInt())
assertEquals(-1L, input.readLong())
assertEquals(12 + hours * (HourlyCounters.FIELDS + 1) * 8, bytes.size)
}
@Test
fun readsFilesFromVersionsWithFewerCompanies() {
// A file from a version that knew two companies: field n of hour 7 holds n + 1
val fields = HourlyCounters.FIRST_COMPANY + 2
val bytes = ByteArrayOutputStream()
DataOutputStream(bytes).use { out ->
out.writeInt(1)
out.writeInt(fields)
out.writeInt(hours)
for (index in 0 until hours) {
out.writeLong(if (index == 7) 7L else -1L)
for (field in 0 until fields) out.writeLong(if (index == 7) field + 1L else 0L)
}
}
val summary = requireNotNull(HourlyCounters.decode(bytes.toByteArray())).summary(1, currentHour = 7)
assertEquals(1L, summary.total)
assertEquals(2L, summary.blocked)
assertEquals(3L, summary.cached)
assertEquals(0, summary.avgLatencyMs) // latency sum 4, count 5
assertArrayEquals(longArrayOf(6, 7) + LongArray(Company.entries.size - 2), summary.companies)
}
@Test
fun refusesOtherFormats() {
val bytes = HourlyCounters().encode()
bytes[3] = 2
assertNull(HourlyCounters.decode(bytes))
}
@Test
fun mergeKeepsWhatWasCountedBeforeTheFileWasRead() {
val saved = HourlyCounters()
saved.record(QueryStatus.OK, 10, null, hour = 50)
saved.record(QueryStatus.OK, 10, null, hour = 49)
val live = HourlyCounters()
live.record(QueryStatus.BLOCKED, 0, null, hour = 50)
live.merge(saved)
assertEquals(2L, live.summary(1, currentHour = 50).total)
assertEquals(1L, live.summary(1, currentHour = 50).blocked)
assertEquals(3L, live.summary(2, currentHour = 50).total)
// A slot already holding a newer hour keeps it
val later = HourlyCounters()
later.record(QueryStatus.OK, 10, null, hour = 50L + hours)
later.merge(saved)
assertEquals(1L, later.summary(1, currentHour = 50L + hours).total)
}
}