mirror of
https://github.com/kalarina-bit/Sparkledns.git
synced 2026-11-22 07:36:09 +00:00
Encode statistics on the writer thread and read them outside the lock
This commit is contained in:
@@ -4,12 +4,11 @@ import android.content.Context
|
||||
import android.util.Log
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import java.io.BufferedInputStream
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.DataInputStream
|
||||
import java.io.DataOutputStream
|
||||
import java.io.File
|
||||
import java.io.FileInputStream
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class StatsSummary(
|
||||
@@ -26,24 +25,11 @@ class HourPoint(val total: Long, val blocked: Long)
|
||||
// Hourly counters of the last 7 days for the charts. Only numbers are kept, never domains,
|
||||
// and they never leave the device.
|
||||
object StatsStore {
|
||||
private const val HOURS = 24 * 7
|
||||
private const val HOUR_MS = 3_600_000L
|
||||
private const val FORMAT = 1
|
||||
private const val FILE = "stats.bin"
|
||||
private const val SAVE_INTERVAL_MS = 60_000L
|
||||
|
||||
// Per hour: total, blocked, cached, latency sum and count, then one counter per company
|
||||
private const val TOTAL = 0
|
||||
private const val BLOCKED = 1
|
||||
private const val CACHED = 2
|
||||
private const val LATENCY_SUM = 3
|
||||
private const val LATENCY_COUNT = 4
|
||||
private const val FIRST_COMPANY = 5
|
||||
private val FIELDS = FIRST_COMPANY + Company.entries.size
|
||||
|
||||
// A ring of slots; each remembers which hour (since the epoch) it currently holds
|
||||
private val hourIds = LongArray(HOURS) { -1L }
|
||||
private val counters = Array(HOURS) { LongArray(FIELDS) }
|
||||
private val counters = HourlyCounters()
|
||||
private var file: File? = null
|
||||
|
||||
// Bumped when the numbers change without a new query: loaded from disk
|
||||
@@ -52,56 +38,134 @@ object StatsStore {
|
||||
private var lastSave = 0L
|
||||
private var dirty = false
|
||||
|
||||
// Counts every change, so summary() can hand out its last result while nothing changed
|
||||
private var changes = 0L
|
||||
private var lastSummary: StatsSummary? = null
|
||||
private var summaryChanges = 0L
|
||||
private var summaryHours = 0
|
||||
private var summaryHour = 0L
|
||||
|
||||
// Saves happen off the VPN threads, one at a time
|
||||
private val writer = Executors.newSingleThreadExecutor { Thread(it, "stats-writer").apply { isDaemon = true } }
|
||||
|
||||
// The file is read outside the lock, so neither the DNS threads nor the screens wait for
|
||||
// storage. Queries counted before it was read are added to the saved numbers.
|
||||
fun init(context: Context) {
|
||||
val target = File(context.filesDir, FILE)
|
||||
val saved = read(target)
|
||||
synchronized(this) {
|
||||
file = target
|
||||
load(target)
|
||||
if (saved != null) counters.merge(saved)
|
||||
changes++
|
||||
}
|
||||
_version.value++
|
||||
}
|
||||
|
||||
fun record(status: QueryStatus, latencyMs: Int, company: Company?, now: Long = System.currentTimeMillis()) {
|
||||
val snapshot = synchronized(this) {
|
||||
val row = row(now / HOUR_MS)
|
||||
row[TOTAL]++
|
||||
when (status) {
|
||||
QueryStatus.BLOCKED -> row[BLOCKED]++
|
||||
QueryStatus.CACHED -> row[CACHED]++
|
||||
QueryStatus.OK -> {
|
||||
row[LATENCY_SUM] += latencyMs.toLong()
|
||||
row[LATENCY_COUNT]++
|
||||
}
|
||||
QueryStatus.FAILED -> Unit
|
||||
}
|
||||
if (company != null) row[FIRST_COMPANY + company.ordinal]++
|
||||
val save = synchronized(this) {
|
||||
counters.record(status, latencyMs, company, now / HOUR_MS)
|
||||
changes++
|
||||
dirty = true
|
||||
if (file != null && now - lastSave >= SAVE_INTERVAL_MS) {
|
||||
lastSave = now
|
||||
dirty = false
|
||||
encode()
|
||||
true
|
||||
} else {
|
||||
null
|
||||
false
|
||||
}
|
||||
}
|
||||
snapshot?.let(::save)
|
||||
// The writer thread encodes the numbers itself, so this DNS thread only hands the job over
|
||||
if (save) writer.execute(::save)
|
||||
}
|
||||
|
||||
// Totals of the last [hours] hours, the current one included
|
||||
// Totals of the last [hours] hours, the current one included. While nothing changed the same
|
||||
// object comes back, so a screen refreshed with it has nothing to recompose.
|
||||
fun summary(hours: Int, now: Long = System.currentTimeMillis()): StatsSummary = synchronized(this) {
|
||||
val current = now / HOUR_MS
|
||||
lastSummary?.takeIf { summaryChanges == changes && summaryHours == hours && summaryHour == current }
|
||||
?: counters.summary(hours, current).also {
|
||||
lastSummary = it
|
||||
summaryChanges = changes
|
||||
summaryHours = hours
|
||||
summaryHour = current
|
||||
}
|
||||
}
|
||||
|
||||
// One point per hour, oldest first, ending with the current hour
|
||||
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> =
|
||||
synchronized(this) { counters.hourly(hours, now / HOUR_MS) }
|
||||
|
||||
// Saves what is not on disk yet; called when the VPN stops
|
||||
fun flush() {
|
||||
val save = synchronized(this) {
|
||||
if (file != null && dirty) {
|
||||
lastSave = System.currentTimeMillis()
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
if (save) writer.execute(::save)
|
||||
}
|
||||
|
||||
// Runs on the writer thread
|
||||
private fun save() {
|
||||
val (target, bytes) = synchronized(this) {
|
||||
val current = file ?: return
|
||||
dirty = false
|
||||
current to counters.encode()
|
||||
}
|
||||
try {
|
||||
val tmp = File(target.parentFile, "$FILE.tmp")
|
||||
tmp.writeBytes(bytes)
|
||||
if (!tmp.renameTo(target)) tmp.delete()
|
||||
} catch (e: Exception) {
|
||||
Log.w("SparkleDNS", "Cannot save statistics", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun read(source: File): HourlyCounters? {
|
||||
if (!source.exists()) return null
|
||||
return try {
|
||||
HourlyCounters.decode(source.readBytes())
|
||||
} catch (e: Exception) {
|
||||
Log.w("SparkleDNS", "Cannot read statistics", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The numbers behind StatsStore: a ring of hourly rows, each remembering which hour (since the
|
||||
// epoch) it holds. Plain data without locks or Android; StatsStore takes care of both.
|
||||
internal class HourlyCounters {
|
||||
val hourIds = LongArray(HOURS) { -1L }
|
||||
val rows = Array(HOURS) { LongArray(FIELDS) }
|
||||
|
||||
fun record(status: QueryStatus, latencyMs: Int, company: Company?, hour: Long) {
|
||||
val row = row(hour)
|
||||
row[TOTAL]++
|
||||
when (status) {
|
||||
QueryStatus.BLOCKED -> row[BLOCKED]++
|
||||
QueryStatus.CACHED -> row[CACHED]++
|
||||
QueryStatus.OK -> {
|
||||
row[LATENCY_SUM] += latencyMs.toLong()
|
||||
row[LATENCY_COUNT]++
|
||||
}
|
||||
QueryStatus.FAILED -> Unit
|
||||
}
|
||||
if (company != null) row[FIRST_COMPANY + company.ordinal]++
|
||||
}
|
||||
|
||||
// Totals of the last [hours] hours up to [currentHour]
|
||||
fun summary(hours: Int, currentHour: Long): StatsSummary {
|
||||
val sums = LongArray(FIELDS)
|
||||
for (back in 0 until hours.coerceIn(1, HOURS)) {
|
||||
val id = current - back
|
||||
val id = currentHour - back
|
||||
val index = slot(id)
|
||||
if (hourIds[index] != id) continue
|
||||
val row = counters[index]
|
||||
val row = rows[index]
|
||||
for (field in 0 until FIELDS) sums[field] += row[field]
|
||||
}
|
||||
StatsSummary(
|
||||
return StatsSummary(
|
||||
total = sums[TOTAL],
|
||||
blocked = sums[BLOCKED],
|
||||
cached = sums[CACHED],
|
||||
@@ -110,29 +174,43 @@ object StatsStore {
|
||||
)
|
||||
}
|
||||
|
||||
// One point per hour, oldest first, ending with the current hour
|
||||
fun hourly(hours: Int, now: Long = System.currentTimeMillis()): List<HourPoint> = synchronized(this) {
|
||||
// One point per hour, oldest first, ending with [currentHour]
|
||||
fun hourly(hours: Int, currentHour: Long): List<HourPoint> {
|
||||
val count = hours.coerceIn(1, HOURS)
|
||||
val current = now / HOUR_MS
|
||||
List(count) { position ->
|
||||
val id = current - (count - 1 - position)
|
||||
return List(count) { position ->
|
||||
val id = currentHour - (count - 1 - position)
|
||||
val index = slot(id)
|
||||
if (hourIds[index] == id) HourPoint(counters[index][TOTAL], counters[index][BLOCKED]) else HourPoint(0, 0)
|
||||
if (hourIds[index] == id) HourPoint(rows[index][TOTAL], rows[index][BLOCKED]) else HourPoint(0, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// Saves what is not on disk yet; called when the VPN stops
|
||||
fun flush() {
|
||||
val snapshot = synchronized(this) {
|
||||
if (file != null && dirty) {
|
||||
dirty = false
|
||||
lastSave = System.currentTimeMillis()
|
||||
encode()
|
||||
} else {
|
||||
null
|
||||
// Saved hours are added to the ones counted since the start; a slot keeps the newer hour
|
||||
fun merge(saved: HourlyCounters) {
|
||||
for (index in 0 until HOURS) {
|
||||
val id = saved.hourIds[index]
|
||||
when {
|
||||
id < 0 || id < hourIds[index] -> Unit
|
||||
id == hourIds[index] -> for (field in 0 until FIELDS) rows[index][field] += saved.rows[index][field]
|
||||
else -> {
|
||||
hourIds[index] = id
|
||||
saved.rows[index].copyInto(rows[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
snapshot?.let(::save)
|
||||
}
|
||||
|
||||
fun encode(): ByteArray {
|
||||
val bytes = ByteArrayOutputStream(12 + HOURS * (FIELDS + 1) * 8)
|
||||
DataOutputStream(bytes).use { out ->
|
||||
out.writeInt(FORMAT)
|
||||
out.writeInt(FIELDS)
|
||||
out.writeInt(HOURS)
|
||||
for (index in 0 until HOURS) {
|
||||
out.writeLong(hourIds[index])
|
||||
rows[index].forEach { out.writeLong(it) }
|
||||
}
|
||||
}
|
||||
return bytes.toByteArray()
|
||||
}
|
||||
|
||||
private fun slot(id: Long): Int = (((id % HOURS) + HOURS) % HOURS).toInt()
|
||||
@@ -141,58 +219,41 @@ object StatsStore {
|
||||
val index = slot(id)
|
||||
if (hourIds[index] != id) {
|
||||
hourIds[index] = id
|
||||
counters[index].fill(0L)
|
||||
rows[index].fill(0L)
|
||||
}
|
||||
return counters[index]
|
||||
return rows[index]
|
||||
}
|
||||
|
||||
private fun encode(): ByteArray {
|
||||
val bytes = ByteArrayOutputStream()
|
||||
DataOutputStream(bytes).use { out ->
|
||||
out.writeInt(FORMAT)
|
||||
out.writeInt(FIELDS)
|
||||
out.writeInt(HOURS)
|
||||
for (index in 0 until HOURS) {
|
||||
out.writeLong(hourIds[index])
|
||||
counters[index].forEach { out.writeLong(it) }
|
||||
}
|
||||
}
|
||||
return bytes.toByteArray()
|
||||
}
|
||||
companion object {
|
||||
const val HOURS = 24 * 7
|
||||
private const val FORMAT = 1
|
||||
|
||||
private fun save(bytes: ByteArray) {
|
||||
val target = file ?: return
|
||||
writer.execute {
|
||||
try {
|
||||
val tmp = File(target.parentFile, "$FILE.tmp")
|
||||
tmp.writeBytes(bytes)
|
||||
if (!tmp.renameTo(target)) tmp.delete()
|
||||
} catch (e: Exception) {
|
||||
Log.w("SparkleDNS", "Cannot save statistics", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Per hour: total, blocked, cached, latency sum and count, then one counter per company
|
||||
private const val TOTAL = 0
|
||||
private const val BLOCKED = 1
|
||||
private const val CACHED = 2
|
||||
private const val LATENCY_SUM = 3
|
||||
private const val LATENCY_COUNT = 4
|
||||
const val FIRST_COMPANY = 5
|
||||
val FIELDS = FIRST_COMPANY + Company.entries.size
|
||||
|
||||
// Files from a version with fewer companies load fine: the new counters start at zero
|
||||
private fun load(source: File) {
|
||||
if (!source.exists()) return
|
||||
try {
|
||||
DataInputStream(BufferedInputStream(FileInputStream(source))).use { input ->
|
||||
if (input.readInt() != FORMAT) return
|
||||
// Files from a version with fewer companies load fine: the new counters start at zero.
|
||||
// Null for a file of another format.
|
||||
fun decode(bytes: ByteArray): HourlyCounters? {
|
||||
DataInputStream(ByteArrayInputStream(bytes)).use { input ->
|
||||
if (input.readInt() != FORMAT) return null
|
||||
val fields = input.readInt()
|
||||
if (input.readInt() != HOURS || fields < FIRST_COMPANY) return
|
||||
if (input.readInt() != HOURS || fields < FIRST_COMPANY) return null
|
||||
val result = HourlyCounters()
|
||||
for (index in 0 until HOURS) {
|
||||
hourIds[index] = input.readLong()
|
||||
result.hourIds[index] = input.readLong()
|
||||
for (field in 0 until fields) {
|
||||
val value = input.readLong()
|
||||
if (field < FIELDS) counters[index][field] = value
|
||||
if (field < FIELDS) result.rows[index][field] = value
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w("SparkleDNS", "Cannot read statistics", e)
|
||||
hourIds.fill(-1L)
|
||||
counters.forEach { it.fill(0L) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package cc.skysparkle.sparkledns
|
||||
|
||||
import org.junit.Assert.assertArrayEquals
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Test
|
||||
import java.io.ByteArrayInputStream
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.DataInputStream
|
||||
import java.io.DataOutputStream
|
||||
|
||||
class HourlyCountersTest {
|
||||
|
||||
private val hours = HourlyCounters.HOURS
|
||||
|
||||
@Test
|
||||
fun countsQueriesPerHour() {
|
||||
val counters = HourlyCounters()
|
||||
counters.record(QueryStatus.OK, 40, Company.GOOGLE, hour = 100)
|
||||
counters.record(QueryStatus.OK, 20, null, hour = 100)
|
||||
counters.record(QueryStatus.BLOCKED, 0, Company.META, hour = 100)
|
||||
counters.record(QueryStatus.CACHED, 0, null, hour = 99)
|
||||
counters.record(QueryStatus.FAILED, 5000, null, hour = 99)
|
||||
|
||||
val day = counters.summary(24, currentHour = 100)
|
||||
assertEquals(5L, day.total)
|
||||
assertEquals(1L, day.blocked)
|
||||
assertEquals(1L, day.cached)
|
||||
assertEquals(30, day.avgLatencyMs)
|
||||
assertEquals(1L, day.companies[Company.GOOGLE.ordinal])
|
||||
assertEquals(1L, day.companies[Company.META.ordinal])
|
||||
assertEquals(3L, counters.summary(1, currentHour = 100).total)
|
||||
|
||||
val points = counters.hourly(3, currentHour = 100)
|
||||
assertEquals(listOf(0L, 2L, 3L), points.map { it.total })
|
||||
assertEquals(listOf(0L, 0L, 1L), points.map { it.blocked })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aSlotStartsOverAWeekLater() {
|
||||
val counters = HourlyCounters()
|
||||
counters.record(QueryStatus.OK, 10, null, hour = 5)
|
||||
counters.record(QueryStatus.OK, 10, null, hour = 5L + hours)
|
||||
|
||||
assertEquals(1L, counters.summary(hours, currentHour = 5L + hours).total)
|
||||
assertEquals(0L, counters.summary(1, currentHour = 5).total)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun savedNumbersReadBackUnchanged() {
|
||||
val counters = HourlyCounters()
|
||||
counters.record(QueryStatus.BLOCKED, 0, Company.XIAOMI, hour = 1000)
|
||||
counters.record(QueryStatus.OK, 12, Company.APPLE, hour = 1001)
|
||||
|
||||
val copy = requireNotNull(HourlyCounters.decode(counters.encode()))
|
||||
|
||||
assertArrayEquals(counters.hourIds, copy.hourIds)
|
||||
for (index in 0 until hours) assertArrayEquals(counters.rows[index], copy.rows[index])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun writesFormatOne() {
|
||||
val bytes = HourlyCounters().encode()
|
||||
val input = DataInputStream(ByteArrayInputStream(bytes))
|
||||
|
||||
assertEquals(1, input.readInt())
|
||||
assertEquals(HourlyCounters.FIELDS, input.readInt())
|
||||
assertEquals(hours, input.readInt())
|
||||
assertEquals(-1L, input.readLong())
|
||||
assertEquals(12 + hours * (HourlyCounters.FIELDS + 1) * 8, bytes.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun readsFilesFromVersionsWithFewerCompanies() {
|
||||
// A file from a version that knew two companies: field n of hour 7 holds n + 1
|
||||
val fields = HourlyCounters.FIRST_COMPANY + 2
|
||||
val bytes = ByteArrayOutputStream()
|
||||
DataOutputStream(bytes).use { out ->
|
||||
out.writeInt(1)
|
||||
out.writeInt(fields)
|
||||
out.writeInt(hours)
|
||||
for (index in 0 until hours) {
|
||||
out.writeLong(if (index == 7) 7L else -1L)
|
||||
for (field in 0 until fields) out.writeLong(if (index == 7) field + 1L else 0L)
|
||||
}
|
||||
}
|
||||
|
||||
val summary = requireNotNull(HourlyCounters.decode(bytes.toByteArray())).summary(1, currentHour = 7)
|
||||
|
||||
assertEquals(1L, summary.total)
|
||||
assertEquals(2L, summary.blocked)
|
||||
assertEquals(3L, summary.cached)
|
||||
assertEquals(0, summary.avgLatencyMs) // latency sum 4, count 5
|
||||
assertArrayEquals(longArrayOf(6, 7) + LongArray(Company.entries.size - 2), summary.companies)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun refusesOtherFormats() {
|
||||
val bytes = HourlyCounters().encode()
|
||||
bytes[3] = 2
|
||||
|
||||
assertNull(HourlyCounters.decode(bytes))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun mergeKeepsWhatWasCountedBeforeTheFileWasRead() {
|
||||
val saved = HourlyCounters()
|
||||
saved.record(QueryStatus.OK, 10, null, hour = 50)
|
||||
saved.record(QueryStatus.OK, 10, null, hour = 49)
|
||||
|
||||
val live = HourlyCounters()
|
||||
live.record(QueryStatus.BLOCKED, 0, null, hour = 50)
|
||||
live.merge(saved)
|
||||
|
||||
assertEquals(2L, live.summary(1, currentHour = 50).total)
|
||||
assertEquals(1L, live.summary(1, currentHour = 50).blocked)
|
||||
assertEquals(3L, live.summary(2, currentHour = 50).total)
|
||||
|
||||
// A slot already holding a newer hour keeps it
|
||||
val later = HourlyCounters()
|
||||
later.record(QueryStatus.OK, 10, null, hour = 50L + hours)
|
||||
later.merge(saved)
|
||||
|
||||
assertEquals(1L, later.summary(1, currentHour = 50L + hours).total)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user