data endpoint for entity 90008 (aka. a website)

fix: filter legit page visits properly, dont count api, rss, prefetch

ptr.pet 71cb6e68 c0f19555

verified
Changed files
+101 -51
src
+1 -1
src/app.html
···
<meta name="viewport" content="width=device-width, initial-scale=1" />
%sveltekit.head%
</head>
-
<body class="md:overflow-hidden" data-sveltekit-preload-data="hover">
+
<body class="md:overflow-hidden">
<div style="display: contents">%sveltekit.body%</div>
</body>
</html>
+2 -7
src/components/navButton.svelte
···
iconUri: string;
}
-
let {
-
highlight = false,
-
name,
-
href,
-
iconUri
-
}: Props = $props();
+
let { highlight = false, name, href, iconUri }: Props = $props();
</script>
<a
···
"
title={name}
href="/{href}"
-
data-sveltekit-preload-data="hover"
+
data-sveltekit-preload-data="tap"
>
<img class="max-w-4" style="image-rendering: pixelated;" src={iconUri} alt={name} />
<div
+2 -2
src/components/pet.svelte
···
let bounciness = 0.8; // How much energy is preserved on bounce
const sendBounceMetrics = () => {
-
fetch('/pet/bounce');
+
fetch('/_api/pet/bounce');
};
let deltaTravelled = 0.0;
···
};
const sendTotalDistance = () => {
-
fetch('/pet/distance', {
+
fetch('/_api/pet/distance', {
method: 'POST',
body: deltaTravelledTotal.toString()
});
+62 -1
src/hooks.server.ts
···
import { steamUpdateNowPlaying } from '$lib/steam';
import { updateCommits } from '$lib/activity';
import { cancelJob, scheduleJob, scheduledJobs } from 'node-schedule';
-
import { sendAllMetrics } from '$lib/metrics';
+
import {
+
incrementFakeVisitCount,
+
incrementLegitVisitCount,
+
pushMetric,
+
sendAllMetrics
+
} from '$lib/metrics';
+
import {
+
addLastVisitor,
+
decrementVisitCount,
+
incrementVisitCount,
+
notifyDarkVisitors,
+
removeLastVisitor
+
} from '$lib/visits';
+
import { testUa } from '$lib/robots';
+
import { error } from '@sveltejs/kit';
const UPDATE_LAST_JOB_NAME = 'update steam game, lastfm track, bsky posts, git activity';
···
console.log(`error while running ${UPDATE_LAST_JOB_NAME} job: ${err}`);
}
}).invoke(); // invoke once immediately
+
+
export const handle = async ({ event, resolve }) => {
+
notifyDarkVisitors(event.url, event.request); // no await so it doesnt block
+
+
const isPrefetch = () => {
+
return (
+
event.request.headers.get('Sec-Purpose')?.includes('prefetch') ||
+
event.request.headers.get('Purpose')?.includes('prefetch') ||
+
event.request.headers.get('x-purpose')?.includes('preview') ||
+
event.request.headers.get('x-moz')?.includes('prefetch')
+
);
+
};
+
const isApi = () => {
+
return event.url.pathname.startsWith('/_api');
+
};
+
const isRss = () => {
+
return event.url.pathname.endsWith('/_rss');
+
};
+
+
// block any requests if the user agent is disallowed by our robots txt
+
const isFakeVisit =
+
(await testUa(event.url.toString(), event.request.headers.get('user-agent') ?? '')) === false;
+
if (isFakeVisit) {
+
pushMetric({ gazesys_visit_fake_total: incrementFakeVisitCount() });
+
throw error(403, 'get a better user agent silly');
+
}
+
+
// only push metric if legit page visit (still want rss to count here though)
+
const isPageVisit = !isApi() && !isPrefetch();
+
if (isPageVisit) pushMetric({ gazesys_visit_real_total: incrementLegitVisitCount() });
+
+
// only add visitors if its a "legit" page visit
+
let id = null;
+
let valid = false;
+
if (isPageVisit && !isRss()) {
+
id = addLastVisitor(event.request, event.cookies);
+
valid = incrementVisitCount(event.request, event.cookies);
+
}
+
+
const resp = await resolve(event);
+
// remove visitors if it was a 404
+
if (resp.status === 404) {
+
if (id !== null) removeLastVisitor(id);
+
if (valid) decrementVisitCount();
+
}
+
return resp;
+
};
+26 -14
src/lib/visits.ts
···
import { get, writable } from 'svelte/store';
const visitCountFile = `${env.WEBSITE_DATA_DIR}/visitcount`;
-
const visitCount = writable(
+
export const visitCount = writable(
parseInt(existsSync(visitCountFile) ? readFileSync(visitCountFile).toString() : '0')
);
type Visitor = { visits: number[] };
-
const lastVisitors = writable<Map<string, Visitor>>(new Map());
+
export const lastVisitors = writable<Map<string, Visitor>>(new Map());
const VISITOR_EXPIRY_SECONDS = 60 * 60; // an hour seems reasonable
+
export const decrementVisitCount = () => {
+
visitCount.set(get(visitCount) - 1);
+
};
+
export const incrementVisitCount = (request: Request, cookies: Cookies) => {
let currentVisitCount = get(visitCount);
// check whether the request is from a bot or not (this doesnt need to be accurate we just want to filter out honest bots)
-
if (isBot(request)) {
-
return currentVisitCount;
-
}
+
if (isBot(request)) return false;
const scopedCookies = scopeCookies(cookies, '/');
// parse the last visit timestamp from cookies if it exists
const visitedTimestamp = parseInt(scopedCookies.get('visitedTimestamp') || '0');
···
// write the visit count to a file so we can load it later again
writeFileSync(visitCountFile, currentVisitCount.toString());
}
-
return currentVisitCount;
+
return true;
+
};
+
+
export const removeLastVisitor = (id: string) => {
+
const visitors = get(lastVisitors);
+
if (visitors.has(id)) {
+
const visitor = visitors.get(id) ?? { visits: [] };
+
visitor?.visits.pop();
+
visitors.set(id, visitor);
+
}
+
lastVisitors.set(visitors);
};
export const addLastVisitor = (request: Request, cookies: Cookies) => {
-
let visitors = get(lastVisitors);
-
visitors = _addLastVisitor(visitors, request, cookies);
+
const { visitors, visitorId } = _addLastVisitor(get(lastVisitors), request, cookies);
lastVisitors.set(visitors);
-
return visitors;
+
return visitorId;
};
export const getVisitorId = (cookies: Cookies) => {
···
}
});
// check whether the request is from a bot or not (this doesnt need to be accurate we just want to filter out honest bots)
-
if (isBot(request)) {
-
return visitors;
-
}
+
if (isBot(request)) return { visitors, visitorId: null };
const scopedCookies = scopeCookies(cookies, '/');
// parse the last visit timestamp from cookies if it exists
let visitorId = scopedCookies.get('visitorId') || '';
···
// put new visit in the front
visitorEntry.visits = [currentTime].concat(visitorEntry.visits);
visitors.set(visitorId, visitorEntry);
-
return visitors;
+
return {
+
visitors,
+
visitorId
+
};
};
export const isBot = (request: Request) => {
···
.then(async (resp) => {
if (resp !== null) {
const msg = await resp.json();
-
const host = `(${request.headers.get('host')} ${request.headers.get('x-real-ip')})`;
+
const host = `(${request.headers.get('host')}|${request.headers.get('x-real-ip')}|${request.headers.get('user-agent')})`;
console.log(
`sent visitor analytic to dark visitors: ${resp.statusText}; ${msg.message ?? ''}${host}`
);
+7 -25
src/routes/+layout.server.ts
···
-
import {
-
bounceCount,
-
distanceTravelled,
-
incrementFakeVisitCount,
-
incrementLegitVisitCount,
-
pushMetric
-
} from '$lib/metrics.js';
-
import { testUa } from '$lib/robots.js';
-
import { addLastVisitor, incrementVisitCount, notifyDarkVisitors } from '$lib/visits.js';
-
import { error } from '@sveltejs/kit';
+
import { bounceCount, distanceTravelled } from '$lib/metrics.js';
+
import { lastVisitors, visitCount } from '$lib/visits.js';
import { localDistanceTravelled } from '../components/pet.svelte';
import { get } from 'svelte/store';
···
export const prerender = false;
export const trailingSlash = 'always';
-
export async function load({ request, cookies, url }) {
-
notifyDarkVisitors(url, request); // no await so it doesnt block load
-
-
// block any requests if the user agent is disallowed by our robots txt
-
if ((await testUa(url.toString(), request.headers.get('user-agent') ?? '')) === false) {
-
pushMetric({ gazesys_visit_fake_total: incrementFakeVisitCount() });
-
throw error(403, 'get a better user agent silly');
-
} else {
-
pushMetric({ gazesys_visit_real_total: incrementLegitVisitCount() });
-
}
-
-
const lastVisitors = addLastVisitor(request, cookies);
+
export async function load({ url }) {
+
const visitors = get(lastVisitors);
let recentVisitCount = 0;
-
for (const [, visitor] of lastVisitors) {
+
for (const [, visitor] of visitors) {
recentVisitCount += visitor.visits.length;
}
···
petTotalBounce: bounceCount.get(),
petTotalDistance: distanceTravelled.get(),
petLocalDistance: get(localDistanceTravelled),
-
visitCount: incrementVisitCount(request, cookies),
-
lastVisitors,
+
visitCount: get(visitCount),
+
lastVisitors: visitors,
recentVisitCount
};
}
+1 -1
src/routes/+page.svelte
···
event.preventDefault();
const data = new FormData(event.currentTarget);
try {
-
fetch(`${PUBLIC_BASE_URL}/pushnotif/?content=${data.get('content')}`);
+
fetch(`${PUBLIC_BASE_URL}/_api/pushnotif/?content=${data.get('content')}`);
} catch (err) {
console.log(`failed to send notif: ${err}`);
}
src/routes/pet/bounce/+server.ts src/routes/_api/pet/bounce/+server.ts
src/routes/pet/distance/+server.ts src/routes/_api/pet/distance/+server.ts
src/routes/pushnotif/+server.ts src/routes/_api/pushnotif/+server.ts