KantSites/src/Content/Statistics.php

<?php

namespace App\Content;

use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Exception\UniqueConstraintViolationException;

/**
 * How often, when and where the website was used.
 *
 * What makes this measurement possible **without asking for consent** is what it does not do: it
 * recognises nobody. There is no cookie, no identifier, no IP address, not even a hashed one - a
 * hash of an IP address is still personal data. Stored is a counter per (day, hour, path, status,
 * referrer host), incremented per request. The table therefore grows with the number of pages and
 * hours, not with the number of visits.
 *
 * The price of that is honest and worth saying out loud: this cannot tell visitors from views, it
 * has no session length, no bounce rate, no path through the site and no country. Anything of that
 * kind needs recognition, and recognition needs consent - it would be a separate component in the
 * `statistics` group, not this one.
 */
final class Statistics
{
    /** Everything that is not a page of this site, or not a visit at all. */
    private const IGNORED = '#^/(media/|assets/|einwilligung$|feed\.xml$|sitemap\.xml$|robots\.txt$|llms\.txt$|favicon)#';

    /**
     * A heuristic, and nothing more: user agents lie. It keeps the obvious crawlers and monitors
     * out of the numbers, and the numbers are therefore "requests that did not announce
     * themselves as a machine", not "humans".
     */
    private const MACHINES = '#bot|crawl|spider|slurp|bingpreview|headless|lighthouse|curl/|wget|python-requests|okhttp|monitor|uptime|preview#i';

    /** A probed address that no page answers is counted, but not with its invented path. */
    public const OTHER = '(sonstige)';

    /** Kept for a little over a year, so a comparison with the same month last year still works. */
    public const KEEP_DAYS = 425;

    /** After two months the hours of a day are merged into one row. */
    public const COMPACT_AFTER_DAYS = 60;

    public function __construct(private Connection $db, private string $adminPath = 'admin')
    {
    }

    /** Whether this request is one of the site's own pages and not an announced machine. */
    public function countable(string $method, string $path, int $status, string $contentType, string $userAgent): bool
    {
        return 'GET' === $method
            && in_array($status, [200, 404], true)
            && str_contains($contentType, 'text/html')
            && 1 !== preg_match(self::IGNORED, $path)
            // Die Verwaltung ist kein Besuch - unabhaengig davon, wie sie heisst.
            && !str_starts_with($path, '/'.$this->adminPath)
            && 1 !== preg_match(self::MACHINES, $userAgent);
    }

    public function record(string $path, int $status, string $referrer, string $siteHost, \DateTimeImmutable $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'))): void
    {
        $path = $this->normalisePath($path, $status);
        $host = $this->referrerHost($referrer, $siteHost);
        $key = [
            'website_id' => 'default',
            'day' => $now->format('Y-m-d'),
            'hour' => (int) $now->format('G'),
            'path' => $path,
            'status' => $status,
            'referrer_host' => $host,
        ];
        // Update first, insert only when there was nothing to update. That is one query in the
        // common case and needs no platform specific upsert syntax, which matters because the
        // target database is MySQL and the development one is SQLite.
        if (1 === $this->increment($key)) {
            return;
        }
        try {
            $this->db->insert('stat_hit', [...$key, 'count' => 1]);
        } catch (UniqueConstraintViolationException) {
            // Another request inserted the same row in between; then incrementing works.
            $this->increment($key);
        }
    }

    /**
     * The numbers for a range of days, ending today.
     *
     * @return array{
     *     days: list<array{day: string, count: int}>,
     *     total: int, pages: list<array{path: string, count: int}>,
     *     referrers: list<array{host: string, count: int}>,
     *     missing: list<array{path: string, count: int}>,
     *     hours: list<array{hour: int, count: int}>,
     *     range: int, from: string, compactedBefore: string, busiest: int
     * }
     */
    public function overview(int $range = 30, \DateTimeImmutable $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'))): array
    {
        $range = max(1, min(365, $range));
        $from = $now->modify(sprintf('-%d days', $range - 1))->format('Y-m-d');
        $where = 'website_id = ? AND day >= ?';
        $params = ['default', $from];

        $perDay = [];
        foreach ($this->db->fetchAllAssociative(
            'SELECT day, SUM(count) AS total FROM stat_hit WHERE '.$where.' AND status = 200 GROUP BY day',
            $params,
        ) as $row) {
            $perDay[$row['day']] = (int) $row['total'];
        }
        // Every day in the range, including the ones with nothing: a gap in a chart is a
        // statement, a missing bar is a puzzle.
        $days = [];
        for ($index = $range - 1; $index >= 0; --$index) {
            $day = $now->modify(sprintf('-%d days', $index))->format('Y-m-d');
            $days[] = ['day' => $day, 'count' => $perDay[$day] ?? 0];
        }

        $rows = fn (string $select, string $extra, string $column): array => array_map(
            static fn (array $row): array => [$column => (string) $row[$column], 'count' => (int) $row['total']],
            $this->db->fetchAllAssociative(
                sprintf('SELECT %s, SUM(count) AS total FROM stat_hit WHERE %s %s GROUP BY %s ORDER BY total DESC LIMIT 15', $select, $where, $extra, $select),
                $params,
            ),
        );

        $hours = [];
        foreach ($this->db->fetchAllAssociative(
            'SELECT hour, SUM(count) AS total FROM stat_hit WHERE '.$where.' AND status = 200 AND hour >= 0 GROUP BY hour',
            $params,
        ) as $row) {
            $hours[(int) $row['hour']] = (int) $row['total'];
        }

        return [
            'days' => $days,
            'total' => array_sum(array_column($days, 'count')),
            'busiest' => max(1, max(array_column($days, 'count'))),
            'pages' => $rows('path', 'AND status = 200', 'path'),
            'referrers' => $rows('referrer_host', "AND status = 200 AND referrer_host <> ''", 'referrer_host'),
            'missing' => $rows('path', 'AND status = 404', 'path'),
            'hours' => array_map(static fn (int $hour): array => ['hour' => $hour, 'count' => $hours[$hour] ?? 0], range(0, 23)),
            'range' => $range,
            'from' => $from,
            'compactedBefore' => $now->modify(sprintf('-%d days', self::COMPACT_AFTER_DAYS))->format('Y-m-d'),
        ];
    }

    /** Every counted row of a range, for the CSV export. */
    public function export(int $range, \DateTimeImmutable $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'))): iterable
    {
        return $this->db->iterateAssociative(
            'SELECT day, hour, path, status, referrer_host, count FROM stat_hit WHERE website_id = ? AND day >= ? ORDER BY day DESC, hour DESC, count DESC',
            ['default', $now->modify(sprintf('-%d days', max(1, min(365, $range)) - 1))->format('Y-m-d')],
        );
    }

    /**
     * Merges the hours of older days into one row each and drops what is beyond the retention.
     *
     * @return array{compacted: int, removed: int}
     */
    public function prune(\DateTimeImmutable $now = new \DateTimeImmutable('now', new \DateTimeZone('UTC'))): array
    {
        $removed = $this->db->executeStatement(
            'DELETE FROM stat_hit WHERE website_id = ? AND day < ?',
            ['default', $now->modify(sprintf('-%d days', self::KEEP_DAYS))->format('Y-m-d')],
        );

        $border = $now->modify(sprintf('-%d days', self::COMPACT_AFTER_DAYS))->format('Y-m-d');
        $compacted = 0;
        $this->db->transactional(function () use ($border, &$compacted): void {
            $groups = $this->db->fetchAllAssociative(
                'SELECT day, path, status, referrer_host, SUM(count) AS total FROM stat_hit
                 WHERE website_id = ? AND day < ? AND hour >= 0
                 GROUP BY day, path, status, referrer_host',
                ['default', $border],
            );
            foreach ($groups as $group) {
                $key = ['website_id' => 'default', 'day' => $group['day'], 'path' => $group['path'], 'status' => (int) $group['status'], 'referrer_host' => $group['referrer_host']];
                $this->db->executeStatement(
                    'DELETE FROM stat_hit WHERE website_id = ? AND day = ? AND path = ? AND status = ? AND referrer_host = ?',
                    array_values($key),
                );
                $this->db->insert('stat_hit', [...$key, 'hour' => -1, 'count' => (int) $group['total']]);
                ++$compacted;
            }
        });

        return ['compacted' => $compacted, 'removed' => $removed];
    }

    private function increment(array $key): int
    {
        return $this->db->executeStatement(
            'UPDATE stat_hit SET count = count + 1 WHERE website_id = ? AND day = ? AND hour = ? AND path = ? AND status = ? AND referrer_host = ?',
            array_values($key),
        );
    }

    /**
     * A 404 keeps its address only if it looks like an address of this site. Crawlers probe
     * thousands of invented paths; each one would otherwise become a row of its own.
     */
    private function normalisePath(string $path, int $status): string
    {
        $path = '/'.trim(parse_url($path, PHP_URL_PATH) ?: '/', '/');
        if (strlen($path) > 200 || (404 === $status && 1 !== preg_match('#^/([a-z0-9]+(?:-[a-z0-9]+)*)(/[a-z0-9]+(?:-[a-z0-9]+)*){0,3}$#D', $path))) {
            return self::OTHER;
        }

        return $path;
    }

    /** The bare host of an external referrer; empty for a direct visit or a link from this site. */
    private function referrerHost(string $referrer, string $siteHost): string
    {
        $host = strtolower((string) parse_url($referrer, PHP_URL_HOST));
        if ('' === $host || $host === strtolower($siteHost) || strlen($host) > 100) {
            return '';
        }

        return 1 === preg_match('/^[a-z0-9.-]+$/D', $host) ? $host : '';
    }
}