diff --git a/app/src/main/java/cc/skysparkle/sparkledns/StatsStore.kt b/app/src/main/java/cc/skysparkle/sparkledns/StatsStore.kt index 3192cc2..c00fe95 100644 --- a/app/src/main/java/cc/skysparkle/sparkledns/StatsStore.kt +++ b/app/src/main/java/cc/skysparkle/sparkledns/StatsStore.kt @@ -4,12 +4,11 @@ import android.content.Context import android.util.Log import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow -import java.io.BufferedInputStream +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.DataInputStream import java.io.DataOutputStream import java.io.File -import java.io.FileInputStream import java.util.concurrent.Executors class StatsSummary( @@ -26,24 +25,11 @@ class HourPoint(val total: Long, val blocked: Long) // Hourly counters of the last 7 days for the charts. Only numbers are kept, never domains, // and they never leave the device. object StatsStore { - private const val HOURS = 24 * 7 private const val HOUR_MS = 3_600_000L - private const val FORMAT = 1 private const val FILE = "stats.bin" private const val SAVE_INTERVAL_MS = 60_000L - // Per hour: total, blocked, cached, latency sum and count, then one counter per company - private const val TOTAL = 0 - private const val BLOCKED = 1 - private const val CACHED = 2 - private const val LATENCY_SUM = 3 - private const val LATENCY_COUNT = 4 - private const val FIRST_COMPANY = 5 - private val FIELDS = FIRST_COMPANY + Company.entries.size - - // A ring of slots; each remembers which hour (since the epoch) it currently holds - private val hourIds = LongArray(HOURS) { -1L } - private val counters = Array(HOURS) { LongArray(FIELDS) } + private val counters = HourlyCounters() private var file: File? = null // Bumped when the numbers change without a new query: loaded from disk @@ -52,56 +38,134 @@ object StatsStore { private var lastSave = 0L private var dirty = false + // Counts every change, so summary() can hand out its last result while nothing changed + private var changes = 0L + private var lastSummary: StatsSummary? = null + private var summaryChanges = 0L + private var summaryHours = 0 + private var summaryHour = 0L + // Saves happen off the VPN threads, one at a time private val writer = Executors.newSingleThreadExecutor { Thread(it, "stats-writer").apply { isDaemon = true } } + // The file is read outside the lock, so neither the DNS threads nor the screens wait for + // storage. Queries counted before it was read are added to the saved numbers. fun init(context: Context) { val target = File(context.filesDir, FILE) + val saved = read(target) synchronized(this) { file = target - load(target) + if (saved != null) counters.merge(saved) + changes++ } _version.value++ } fun record(status: QueryStatus, latencyMs: Int, company: Company?, now: Long = System.currentTimeMillis()) { - val snapshot = synchronized(this) { - val row = row(now / HOUR_MS) - row[TOTAL]++ - when (status) { - QueryStatus.BLOCKED -> row[BLOCKED]++ - QueryStatus.CACHED -> row[CACHED]++ - QueryStatus.OK -> { - row[LATENCY_SUM] += latencyMs.toLong() - row[LATENCY_COUNT]++ - } - QueryStatus.FAILED -> Unit - } - if (company != null) row[FIRST_COMPANY + company.ordinal]++ + val save = synchronized(this) { + counters.record(status, latencyMs, company, now / HOUR_MS) + changes++ dirty = true if (file != null && now - lastSave >= SAVE_INTERVAL_MS) { lastSave = now - dirty = false - encode() + true } else { - null + false } } - snapshot?.let(::save) + // The writer thread encodes the numbers itself, so this DNS thread only hands the job over + if (save) writer.execute(::save) } - // Totals of the last [hours] hours, the current one included + // Totals of the last [hours] hours, the current one included. While nothing changed the same + // object comes back, so a screen refreshed with it has nothing to recompose. fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) { val current = now / HOUR_MS + lastSummary?.takeIf { summaryChanges == changes && summaryHours == hours && summaryHour == current } + ?: counters.summary(hours, current).also { + lastSummary = it + summaryChanges = changes + summaryHours = hours + summaryHour = current + } + } + + // One point per hour, oldest first, ending with the current hour + fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List = + synchronized(this) { counters.hourly(hours, now / HOUR_MS) } + + // Saves what is not on disk yet; called when the VPN stops + fun flush() { + val save = synchronized(this) { + if (file != null && dirty) { + lastSave = System.currentTimeMillis() + true + } else { + false + } + } + if (save) writer.execute(::save) + } + + // Runs on the writer thread + private fun save() { + val (target, bytes) = synchronized(this) { + val current = file ?: return + dirty = false + current to counters.encode() + } + try { + val tmp = File(target.parentFile, "$FILE.tmp") + tmp.writeBytes(bytes) + if (!tmp.renameTo(target)) tmp.delete() + } catch (e: Exception) { + Log.w("SparkleDNS", "Cannot save statistics", e) + } + } + + private fun read(source: File): HourlyCounters? { + if (!source.exists()) return null + return try { + HourlyCounters.decode(source.readBytes()) + } catch (e: Exception) { + Log.w("SparkleDNS", "Cannot read statistics", e) + null + } + } +} + +// The numbers behind StatsStore: a ring of hourly rows, each remembering which hour (since the +// epoch) it holds. Plain data without locks or Android; StatsStore takes care of both. +internal class HourlyCounters { + val hourIds = LongArray(HOURS) { -1L } + val rows = Array(HOURS) { LongArray(FIELDS) } + + fun record(status: QueryStatus, latencyMs: Int, company: Company?, hour: Long) { + val row = row(hour) + row[TOTAL]++ + when (status) { + QueryStatus.BLOCKED -> row[BLOCKED]++ + QueryStatus.CACHED -> row[CACHED]++ + QueryStatus.OK -> { + row[LATENCY_SUM] += latencyMs.toLong() + row[LATENCY_COUNT]++ + } + QueryStatus.FAILED -> Unit + } + if (company != null) row[FIRST_COMPANY + company.ordinal]++ + } + + // Totals of the last [hours] hours up to [currentHour] + fun summary(hours: Int, currentHour: Long): StatsSummary { val sums = LongArray(FIELDS) for (back in 0 until hours.coerceIn(1, HOURS)) { - val id = current - back + val id = currentHour - back val index = slot(id) if (hourIds[index] != id) continue - val row = counters[index] + val row = rows[index] for (field in 0 until FIELDS) sums[field] += row[field] } - StatsSummary( + return StatsSummary( total = sums[TOTAL], blocked = sums[BLOCKED], cached = sums[CACHED], @@ -110,29 +174,43 @@ object StatsStore { ) } - // One point per hour, oldest first, ending with the current hour - fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List = synchronized(this) { + // One point per hour, oldest first, ending with [currentHour] + fun hourly(hours: Int, currentHour: Long): List { val count = hours.coerceIn(1, HOURS) - val current = now / HOUR_MS - List(count) { position -> - val id = current - (count - 1 - position) + return List(count) { position -> + val id = currentHour - (count - 1 - position) val index = slot(id) - if (hourIds[index] == id) HourPoint(counters[index][TOTAL], counters[index][BLOCKED]) else HourPoint(0, 0) + if (hourIds[index] == id) HourPoint(rows[index][TOTAL], rows[index][BLOCKED]) else HourPoint(0, 0) } } - // Saves what is not on disk yet; called when the VPN stops - fun flush() { - val snapshot = synchronized(this) { - if (file != null && dirty) { - dirty = false - lastSave = System.currentTimeMillis() - encode() - } else { - null + // Saved hours are added to the ones counted since the start; a slot keeps the newer hour + fun merge(saved: HourlyCounters) { + for (index in 0 until HOURS) { + val id = saved.hourIds[index] + when { + id < 0 || id < hourIds[index] -> Unit + id == hourIds[index] -> for (field in 0 until FIELDS) rows[index][field] += saved.rows[index][field] + else -> { + hourIds[index] = id + saved.rows[index].copyInto(rows[index]) + } } } - snapshot?.let(::save) + } + + fun encode(): ByteArray { + val bytes = ByteArrayOutputStream(12 + HOURS * (FIELDS + 1) * 8) + DataOutputStream(bytes).use { out -> + out.writeInt(FORMAT) + out.writeInt(FIELDS) + out.writeInt(HOURS) + for (index in 0 until HOURS) { + out.writeLong(hourIds[index]) + rows[index].forEach { out.writeLong(it) } + } + } + return bytes.toByteArray() } private fun slot(id: Long): Int = (((id % HOURS) + HOURS) % HOURS).toInt() @@ -141,58 +219,41 @@ object StatsStore { val index = slot(id) if (hourIds[index] != id) { hourIds[index] = id - counters[index].fill(0L) + rows[index].fill(0L) } - return counters[index] + return rows[index] } - private fun encode(): ByteArray { - val bytes = ByteArrayOutputStream() - DataOutputStream(bytes).use { out -> - out.writeInt(FORMAT) - out.writeInt(FIELDS) - out.writeInt(HOURS) - for (index in 0 until HOURS) { - out.writeLong(hourIds[index]) - counters[index].forEach { out.writeLong(it) } - } - } - return bytes.toByteArray() - } + companion object { + const val HOURS = 24 * 7 + private const val FORMAT = 1 - private fun save(bytes: ByteArray) { - val target = file ?: return - writer.execute { - try { - val tmp = File(target.parentFile, "$FILE.tmp") - tmp.writeBytes(bytes) - if (!tmp.renameTo(target)) tmp.delete() - } catch (e: Exception) { - Log.w("SparkleDNS", "Cannot save statistics", e) - } - } - } + // Per hour: total, blocked, cached, latency sum and count, then one counter per company + private const val TOTAL = 0 + private const val BLOCKED = 1 + private const val CACHED = 2 + private const val LATENCY_SUM = 3 + private const val LATENCY_COUNT = 4 + const val FIRST_COMPANY = 5 + val FIELDS = FIRST_COMPANY + Company.entries.size - // Files from a version with fewer companies load fine: the new counters start at zero - private fun load(source: File) { - if (!source.exists()) return - try { - DataInputStream(BufferedInputStream(FileInputStream(source))).use { input -> - if (input.readInt() != FORMAT) return + // Files from a version with fewer companies load fine: the new counters start at zero. + // Null for a file of another format. + fun decode(bytes: ByteArray): HourlyCounters? { + DataInputStream(ByteArrayInputStream(bytes)).use { input -> + if (input.readInt() != FORMAT) return null val fields = input.readInt() - if (input.readInt() != HOURS || fields < FIRST_COMPANY) return + if (input.readInt() != HOURS || fields < FIRST_COMPANY) return null + val result = HourlyCounters() for (index in 0 until HOURS) { - hourIds[index] = input.readLong() + result.hourIds[index] = input.readLong() for (field in 0 until fields) { val value = input.readLong() - if (field < FIELDS) counters[index][field] = value + if (field < FIELDS) result.rows[index][field] = value } } + return result } - } catch (e: Exception) { - Log.w("SparkleDNS", "Cannot read statistics", e) - hourIds.fill(-1L) - counters.forEach { it.fill(0L) } } } } diff --git a/app/src/test/java/cc/skysparkle/sparkledns/HourlyCountersTest.kt b/app/src/test/java/cc/skysparkle/sparkledns/HourlyCountersTest.kt new file mode 100644 index 0000000..6c57c1e --- /dev/null +++ b/app/src/test/java/cc/skysparkle/sparkledns/HourlyCountersTest.kt @@ -0,0 +1,126 @@ +package cc.skysparkle.sparkledns + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.DataInputStream +import java.io.DataOutputStream + +class HourlyCountersTest { + + private val hours = HourlyCounters.HOURS + + @Test + fun countsQueriesPerHour() { + val counters = HourlyCounters() + counters.record(QueryStatus.OK, 40, Company.GOOGLE, hour = 100) + counters.record(QueryStatus.OK, 20, null, hour = 100) + counters.record(QueryStatus.BLOCKED, 0, Company.META, hour = 100) + counters.record(QueryStatus.CACHED, 0, null, hour = 99) + counters.record(QueryStatus.FAILED, 5000, null, hour = 99) + + val day = counters.summary(24, currentHour = 100) + assertEquals(5L, day.total) + assertEquals(1L, day.blocked) + assertEquals(1L, day.cached) + assertEquals(30, day.avgLatencyMs) + assertEquals(1L, day.companies[Company.GOOGLE.ordinal]) + assertEquals(1L, day.companies[Company.META.ordinal]) + assertEquals(3L, counters.summary(1, currentHour = 100).total) + + val points = counters.hourly(3, currentHour = 100) + assertEquals(listOf(0L, 2L, 3L), points.map { it.total }) + assertEquals(listOf(0L, 0L, 1L), points.map { it.blocked }) + } + + @Test + fun aSlotStartsOverAWeekLater() { + val counters = HourlyCounters() + counters.record(QueryStatus.OK, 10, null, hour = 5) + counters.record(QueryStatus.OK, 10, null, hour = 5L + hours) + + assertEquals(1L, counters.summary(hours, currentHour = 5L + hours).total) + assertEquals(0L, counters.summary(1, currentHour = 5).total) + } + + @Test + fun savedNumbersReadBackUnchanged() { + val counters = HourlyCounters() + counters.record(QueryStatus.BLOCKED, 0, Company.XIAOMI, hour = 1000) + counters.record(QueryStatus.OK, 12, Company.APPLE, hour = 1001) + + val copy = requireNotNull(HourlyCounters.decode(counters.encode())) + + assertArrayEquals(counters.hourIds, copy.hourIds) + for (index in 0 until hours) assertArrayEquals(counters.rows[index], copy.rows[index]) + } + + @Test + fun writesFormatOne() { + val bytes = HourlyCounters().encode() + val input = DataInputStream(ByteArrayInputStream(bytes)) + + assertEquals(1, input.readInt()) + assertEquals(HourlyCounters.FIELDS, input.readInt()) + assertEquals(hours, input.readInt()) + assertEquals(-1L, input.readLong()) + assertEquals(12 + hours * (HourlyCounters.FIELDS + 1) * 8, bytes.size) + } + + @Test + fun readsFilesFromVersionsWithFewerCompanies() { + // A file from a version that knew two companies: field n of hour 7 holds n + 1 + val fields = HourlyCounters.FIRST_COMPANY + 2 + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { out -> + out.writeInt(1) + out.writeInt(fields) + out.writeInt(hours) + for (index in 0 until hours) { + out.writeLong(if (index == 7) 7L else -1L) + for (field in 0 until fields) out.writeLong(if (index == 7) field + 1L else 0L) + } + } + + val summary = requireNotNull(HourlyCounters.decode(bytes.toByteArray())).summary(1, currentHour = 7) + + assertEquals(1L, summary.total) + assertEquals(2L, summary.blocked) + assertEquals(3L, summary.cached) + assertEquals(0, summary.avgLatencyMs) // latency sum 4, count 5 + assertArrayEquals(longArrayOf(6, 7) + LongArray(Company.entries.size - 2), summary.companies) + } + + @Test + fun refusesOtherFormats() { + val bytes = HourlyCounters().encode() + bytes[3] = 2 + + assertNull(HourlyCounters.decode(bytes)) + } + + @Test + fun mergeKeepsWhatWasCountedBeforeTheFileWasRead() { + val saved = HourlyCounters() + saved.record(QueryStatus.OK, 10, null, hour = 50) + saved.record(QueryStatus.OK, 10, null, hour = 49) + + val live = HourlyCounters() + live.record(QueryStatus.BLOCKED, 0, null, hour = 50) + live.merge(saved) + + assertEquals(2L, live.summary(1, currentHour = 50).total) + assertEquals(1L, live.summary(1, currentHour = 50).blocked) + assertEquals(3L, live.summary(2, currentHour = 50).total) + + // A slot already holding a newer hour keeps it + val later = HourlyCounters() + later.record(QueryStatus.OK, 10, null, hour = 50L + hours) + later.merge(saved) + + assertEquals(1L, later.summary(1, currentHour = 50L + hours).total) + } +}