Encode statistics on the writer thread and read them outside the lock

This commit is contained in:
2026-10-31 11:40:43 +02:00
parent 60c43df3c5
commit 6444a81f95
2 changed files with 281 additions and 94 deletions
@@ -4,12 +4,11 @@ import android.content.Context
import android.util.Log
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import java.io.BufferedInputStream
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.DataInputStream
import java.io.DataOutputStream
import java.io.File
import java.io.FileInputStream
import java.util.concurrent.Executors
class StatsSummary(
@@ -26,24 +25,11 @@ class HourPoint(val total: Long, val blocked: Long)
// Hourly counters of the last 7 days for the charts. Only numbers are kept, never domains,
// and they never leave the device.
object StatsStore {
private const val HOURS = 24 * 7
private const val HOUR_MS = 3_600_000L
private const val FORMAT = 1
private const val FILE = "stats.bin"
private const val SAVE_INTERVAL_MS = 60_000L
// Per hour: total, blocked, cached, latency sum and count, then one counter per company
private const val TOTAL = 0
private const val BLOCKED = 1
private const val CACHED = 2
private const val LATENCY_SUM = 3
private const val LATENCY_COUNT = 4
private const val FIRST_COMPANY = 5
private val FIELDS = FIRST_COMPANY + Company.entries.size
// A ring of slots; each remembers which hour (since the epoch) it currently holds
private val hourIds = LongArray(HOURS) { -1L }
private val counters = Array(HOURS) { LongArray(FIELDS) }
private val counters = HourlyCounters()
private var file: File? = null
// Bumped when the numbers change without a new query: loaded from disk
@@ -52,21 +38,110 @@ object StatsStore {
private var lastSave = 0L
private var dirty = false
// Counts every change, so summary() can hand out its last result while nothing changed
private var changes = 0L
private var lastSummary: StatsSummary? = null
private var summaryChanges = 0L
private var summaryHours = 0
private var summaryHour = 0L
// Saves happen off the VPN threads, one at a time
private val writer = Executors.newSingleThreadExecutor { Thread(it, "stats-writer").apply { isDaemon = true } }
// The file is read outside the lock, so neither the DNS threads nor the screens wait for
// storage. Queries counted before it was read are added to the saved numbers.
fun init(context: Context) {
val target = File(context.filesDir, FILE)
val saved = read(target)
synchronized(this) {
file = target
load(target)
if (saved != null) counters.merge(saved)
changes++
}
_version.value++
}
fun record(status: QueryStatus, latencyMs: Int, company: Company?, now: Long = System.currentTimeMillis()) {
val snapshot = synchronized(this) {
val row = row(now / HOUR_MS)
val save = synchronized(this) {
counters.record(status, latencyMs, company, now / HOUR_MS)
changes++
dirty = true
if (file != null && now - lastSave >= SAVE_INTERVAL_MS) {
lastSave = now
true
} else {
false
}
}
// The writer thread encodes the numbers itself, so this DNS thread only hands the job over
if (save) writer.execute(::save)
}
// Totals of the last [hours] hours, the current one included. While nothing changed the same
// object comes back, so a screen refreshed with it has nothing to recompose.
fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) {
val current = now / HOUR_MS
lastSummary?.takeIf { summaryChanges == changes && summaryHours == hours && summaryHour == current }
?: counters.summary(hours, current).also {
lastSummary = it
summaryChanges = changes
summaryHours = hours
summaryHour = current
}
}
// One point per hour, oldest first, ending with the current hour
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> =
synchronized(this) { counters.hourly(hours, now / HOUR_MS) }
// Saves what is not on disk yet; called when the VPN stops
fun flush() {
val save = synchronized(this) {
if (file != null && dirty) {
lastSave = System.currentTimeMillis()
true
} else {
false
}
}
if (save) writer.execute(::save)
}
// Runs on the writer thread
private fun save() {
val (target, bytes) = synchronized(this) {
val current = file ?: return
dirty = false
current to counters.encode()
}
try {
val tmp = File(target.parentFile, "$FILE.tmp")
tmp.writeBytes(bytes)
if (!tmp.renameTo(target)) tmp.delete()
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot save statistics", e)
}
}
private fun read(source: File): HourlyCounters? {
if (!source.exists()) return null
return try {
HourlyCounters.decode(source.readBytes())
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot read statistics", e)
null
}
}
}
// The numbers behind StatsStore: a ring of hourly rows, each remembering which hour (since the
// epoch) it holds. Plain data without locks or Android; StatsStore takes care of both.
internal class HourlyCounters {
val hourIds = LongArray(HOURS) { -1L }
val rows = Array(HOURS) { LongArray(FIELDS) }
fun record(status: QueryStatus, latencyMs: Int, company: Company?, hour: Long) {
val row = row(hour)
row[TOTAL]++
when (status) {
QueryStatus.BLOCKED -> row[BLOCKED]++
@@ -78,30 +153,19 @@ object StatsStore {
QueryStatus.FAILED -> Unit
}
if (company != null) row[FIRST_COMPANY + company.ordinal]++
dirty = true
if (file != null && now - lastSave >= SAVE_INTERVAL_MS) {
lastSave = now
dirty = false
encode()
} else {
null
}
}
snapshot?.let(::save)
}
// Totals of the last [hours] hours, the current one included
fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) {
val current = now / HOUR_MS
// Totals of the last [hours] hours up to [currentHour]
fun summary(hours: Int, currentHour: Long): StatsSummary {
val sums = LongArray(FIELDS)
for (back in 0 until hours.coerceIn(1, HOURS)) {
val id = current - back
val id = currentHour - back
val index = slot(id)
if (hourIds[index] != id) continue
val row = counters[index]
val row = rows[index]
for (field in 0 until FIELDS) sums[field] += row[field]
}
StatsSummary(
return StatsSummary(
total = sums[TOTAL],
blocked = sums[BLOCKED],
cached = sums[CACHED],
@@ -110,29 +174,43 @@ object StatsStore {
)
}
// One point per hour, oldest first, ending with the current hour
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> = synchronized(this) {
// One point per hour, oldest first, ending with [currentHour]
fun hourly(hours: Int, currentHour: Long): List<HourPoint> {
val count = hours.coerceIn(1, HOURS)
val current = now / HOUR_MS
List(count) { position ->
val id = current - (count - 1 - position)
return List(count) { position ->
val id = currentHour - (count - 1 - position)
val index = slot(id)
if (hourIds[index] == id) HourPoint(counters[index][TOTAL], counters[index][BLOCKED]) else HourPoint(0, 0)
if (hourIds[index] == id) HourPoint(rows[index][TOTAL], rows[index][BLOCKED]) else HourPoint(0, 0)
}
}
// Saves what is not on disk yet; called when the VPN stops
fun flush() {
val snapshot = synchronized(this) {
if (file != null && dirty) {
dirty = false
lastSave = System.currentTimeMillis()
encode()
} else {
null
// Saved hours are added to the ones counted since the start; a slot keeps the newer hour
fun merge(saved: HourlyCounters) {
for (index in 0 until HOURS) {
val id = saved.hourIds[index]
when {
id < 0 || id < hourIds[index] -> Unit
id == hourIds[index] -> for (field in 0 until FIELDS) rows[index][field] += saved.rows[index][field]
else -> {
hourIds[index] = id
saved.rows[index].copyInto(rows[index])
}
}
snapshot?.let(::save)
}
}
fun encode(): ByteArray {
val bytes = ByteArrayOutputStream(12 + HOURS * (FIELDS + 1) * 8)
DataOutputStream(bytes).use { out ->
out.writeInt(FORMAT)
out.writeInt(FIELDS)
out.writeInt(HOURS)
for (index in 0 until HOURS) {
out.writeLong(hourIds[index])
rows[index].forEach { out.writeLong(it) }
}
}
return bytes.toByteArray()
}
private fun slot(id: Long): Int = (((id % HOURS) + HOURS) % HOURS).toInt()
@@ -141,58 +219,41 @@ object StatsStore {
val index = slot(id)
if (hourIds[index] != id) {
hourIds[index] = id
counters[index].fill(0L)
rows[index].fill(0L)
}
return counters[index]
return rows[index]
}
private fun encode(): ByteArray {
val bytes = ByteArrayOutputStream()
DataOutputStream(bytes).use { out ->
out.writeInt(FORMAT)
out.writeInt(FIELDS)
out.writeInt(HOURS)
for (index in 0 until HOURS) {
out.writeLong(hourIds[index])
counters[index].forEach { out.writeLong(it) }
}
}
return bytes.toByteArray()
}
companion object {
const val HOURS = 24 * 7
private const val FORMAT = 1
private fun save(bytes: ByteArray) {
val target = file ?: return
writer.execute {
try {
val tmp = File(target.parentFile, "$FILE.tmp")
tmp.writeBytes(bytes)
if (!tmp.renameTo(target)) tmp.delete()
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot save statistics", e)
}
}
}
// Per hour: total, blocked, cached, latency sum and count, then one counter per company
private const val TOTAL = 0
private const val BLOCKED = 1
private const val CACHED = 2
private const val LATENCY_SUM = 3
private const val LATENCY_COUNT = 4
const val FIRST_COMPANY = 5
val FIELDS = FIRST_COMPANY + Company.entries.size
// Files from a version with fewer companies load fine: the new counters start at zero
private fun load(source: File) {
if (!source.exists()) return
try {
DataInputStream(BufferedInputStream(FileInputStream(source))).use { input ->
if (input.readInt() != FORMAT) return
// Files from a version with fewer companies load fine: the new counters start at zero.
// Null for a file of another format.
fun decode(bytes: ByteArray): HourlyCounters? {
DataInputStream(ByteArrayInputStream(bytes)).use { input ->
if (input.readInt() != FORMAT) return null
val fields = input.readInt()
if (input.readInt() != HOURS || fields < FIRST_COMPANY) return
if (input.readInt() != HOURS || fields < FIRST_COMPANY) return null
val result = HourlyCounters()
for (index in 0 until HOURS) {
hourIds[index] = input.readLong()
result.hourIds[index] = input.readLong()
for (field in 0 until fields) {
val value = input.readLong()
if (field < FIELDS) counters[index][field] = value
if (field < FIELDS) result.rows[index][field] = value
}
}
return result
}
} catch (e: Exception) {
Log.w("SparkleDNS", "Cannot read statistics", e)
hourIds.fill(-1L)
counters.forEach { it.fill(0L) }
}
}
}
@@ -0,0 +1,126 @@
package cc.skysparkle.sparkledns
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Test
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.DataInputStream
import java.io.DataOutputStream
class HourlyCountersTest {
private val hours = HourlyCounters.HOURS
@Test
fun countsQueriesPerHour() {
val counters = HourlyCounters()
counters.record(QueryStatus.OK, 40, Company.GOOGLE, hour = 100)
counters.record(QueryStatus.OK, 20, null, hour = 100)
counters.record(QueryStatus.BLOCKED, 0, Company.META, hour = 100)
counters.record(QueryStatus.CACHED, 0, null, hour = 99)
counters.record(QueryStatus.FAILED, 5000, null, hour = 99)
val day = counters.summary(24, currentHour = 100)
assertEquals(5L, day.total)
assertEquals(1L, day.blocked)
assertEquals(1L, day.cached)
assertEquals(30, day.avgLatencyMs)
assertEquals(1L, day.companies[Company.GOOGLE.ordinal])
assertEquals(1L, day.companies[Company.META.ordinal])
assertEquals(3L, counters.summary(1, currentHour = 100).total)
val points = counters.hourly(3, currentHour = 100)
assertEquals(listOf(0L, 2L, 3L), points.map { it.total })
assertEquals(listOf(0L, 0L, 1L), points.map { it.blocked })
}
@Test
fun aSlotStartsOverAWeekLater() {
val counters = HourlyCounters()
counters.record(QueryStatus.OK, 10, null, hour = 5)
counters.record(QueryStatus.OK, 10, null, hour = 5L + hours)
assertEquals(1L, counters.summary(hours, currentHour = 5L + hours).total)
assertEquals(0L, counters.summary(1, currentHour = 5).total)
}
@Test
fun savedNumbersReadBackUnchanged() {
val counters = HourlyCounters()
counters.record(QueryStatus.BLOCKED, 0, Company.XIAOMI, hour = 1000)
counters.record(QueryStatus.OK, 12, Company.APPLE, hour = 1001)
val copy = requireNotNull(HourlyCounters.decode(counters.encode()))
assertArrayEquals(counters.hourIds, copy.hourIds)
for (index in 0 until hours) assertArrayEquals(counters.rows[index], copy.rows[index])
}
@Test
fun writesFormatOne() {
val bytes = HourlyCounters().encode()
val input = DataInputStream(ByteArrayInputStream(bytes))
assertEquals(1, input.readInt())
assertEquals(HourlyCounters.FIELDS, input.readInt())
assertEquals(hours, input.readInt())
assertEquals(-1L, input.readLong())
assertEquals(12 + hours * (HourlyCounters.FIELDS + 1) * 8, bytes.size)
}
@Test
fun readsFilesFromVersionsWithFewerCompanies() {
// A file from a version that knew two companies: field n of hour 7 holds n + 1
val fields = HourlyCounters.FIRST_COMPANY + 2
val bytes = ByteArrayOutputStream()
DataOutputStream(bytes).use { out ->
out.writeInt(1)
out.writeInt(fields)
out.writeInt(hours)
for (index in 0 until hours) {
out.writeLong(if (index == 7) 7L else -1L)
for (field in 0 until fields) out.writeLong(if (index == 7) field + 1L else 0L)
}
}
val summary = requireNotNull(HourlyCounters.decode(bytes.toByteArray())).summary(1, currentHour = 7)
assertEquals(1L, summary.total)
assertEquals(2L, summary.blocked)
assertEquals(3L, summary.cached)
assertEquals(0, summary.avgLatencyMs) // latency sum 4, count 5
assertArrayEquals(longArrayOf(6, 7) + LongArray(Company.entries.size - 2), summary.companies)
}
@Test
fun refusesOtherFormats() {
val bytes = HourlyCounters().encode()
bytes[3] = 2
assertNull(HourlyCounters.decode(bytes))
}
@Test
fun mergeKeepsWhatWasCountedBeforeTheFileWasRead() {
val saved = HourlyCounters()
saved.record(QueryStatus.OK, 10, null, hour = 50)
saved.record(QueryStatus.OK, 10, null, hour = 49)
val live = HourlyCounters()
live.record(QueryStatus.BLOCKED, 0, null, hour = 50)
live.merge(saved)
assertEquals(2L, live.summary(1, currentHour = 50).total)
assertEquals(1L, live.summary(1, currentHour = 50).blocked)
assertEquals(3L, live.summary(2, currentHour = 50).total)
// A slot already holding a newer hour keeps it
val later = HourlyCounters()
later.record(QueryStatus.OK, 10, null, hour = 50L + hours)
later.merge(saved)
assertEquals(1L, later.summary(1, currentHour = 50L + hours).total)
}
}