Skip to content

Commit

Permalink
add status column to pageviews event and table
Browse files Browse the repository at this point in the history
  • Loading branch information
@negrel
negrel committed Nov 1, 2024
1 parent c35c6cd commit 9183bed
Show file tree
Hide file tree
Showing 12 changed files with 203 additions and 4 deletions.
16 changes: 16 additions & 0 deletions mocks/static/404.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<!doctype html>
<html lang="en">

<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>404 - Page Not Found</title>
<script src="http://prisme.localhost:8000/static/wa.js" data-status="404" defer></script>
</head>

<body>
<h1>Page Not Found</h1>
<a href="/">Go to Home page</a>
</body>

</html>
17 changes: 17 additions & 0 deletions mocks/static/noscript/404.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<!doctype html>
<html lang="en">

<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>404 - Page Not Found</title>
</head>

<body>
<h1>Page Not Found</h1>
<img src="http://prisme.localhost:8000/api/v1/noscript/events/pageviews?status=404" crossorigin=""
style="position:fixed;top:-1px" />
<a href="/">Go to Home page</a>
</body>

</html>
Empty file added pkg/embedded/ch_migrations/20241101202405_add_status_to_pageviews_table.down.sql
View file
Empty file.
16 changes: 16 additions & 0 deletions pkg/embedded/ch_migrations/20241101202405_add_status_to_pageviews_table.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
ALTER TABLE sessions ADD COLUMN exit_status UInt16 DEFAULT 200 AFTER utm_content;
ALTER TABLE pageviews ADD COLUMN status UInt16 DEFAULT 200 AFTER session_uuid;

DROP TABLE pageviews_mv;

CREATE MATERIALIZED VIEW pageviews_mv TO pageviews AS
SELECT
exit_timestamp AS timestamp,
domain,
exit_path AS path,
visitor_id,
session_uuid,
exit_status AS status
FROM sessions
WHERE sign = 1;

2 changes: 1 addition & 1 deletion pkg/embedded/static/wa.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion pkg/event/pageview.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ type PageView struct {
Session Session
Timestamp time.Time
PageUri uri.Uri
Status uint16
}

// MarshalZerologObject implements zerolog.LogObjectMarshaler.
func (pv *PageView) MarshalZerologObject(e *zerolog.Event) {
e.
Object("session", &pv.Session).
Time("timestamp", pv.Timestamp).
Stringer("page_uri", pv.PageUri)
Stringer("page_uri", pv.PageUri).
Uint16("status", pv.Status)
}
16 changes: 14 additions & 2 deletions pkg/handlers/events_pageviews.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/binary"
"fmt"
"strconv"
"time"

"github.com/gofiber/fiber/v2"
Expand Down Expand Up @@ -51,6 +52,7 @@ func ProvidePostEventsPageViews(
c.Request().Header.Peek("X-Prisme-Document-Referrer"),
c.Context().UserAgent(),
utils.UnsafeBytes(c.IP()),
utils.UnsafeString(c.Request().Header.Peek("X-Prisme-Status")),
utils.UnsafeString(c.Request().Header.Peek("X-Prisme-Visitor-Id")),
)
}
Expand All @@ -66,17 +68,27 @@ func eventsPageviewsHandler(
headers *fasthttp.RequestHeader,
requestReferrer uri.Uri,
documentReferrer, userAgent, ipAddr []byte,
visitorId string,
status, visitorId string,
) (err error) {
var referrerUri event.ReferrerUri
pageView := event.PageView{
PageUri: requestReferrer,
Status: fiber.StatusOK,
}

// Retrive pageview status code.
if status != "" {
pvStatus, err := strconv.ParseUint(status, 10, 16)
if err != nil {
return fiber.NewError(fiber.StatusBadRequest, "invalid pageview status")
}
pageView.Status = uint16(pvStatus)
}

// Parse referrer URI.
referrerUri, err = event.ParseReferrerUri(documentReferrer)
if err != nil {
return fiber.NewError(fiber.StatusBadRequest, "invalid X-Prisme-Document-Referrer")
return fiber.NewError(fiber.StatusBadRequest, "invalid document referrer")
}

// Compute device id.
Expand Down
1 change: 1 addition & 0 deletions pkg/handlers/noscript_events_pageviews.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ func ProvideGetNoscriptEventsPageviews(
utils.UnsafeBytes(c.Query("document-referrer")),
c.Context().UserAgent(),
utils.UnsafeBytes(c.IP()),
c.Query("status"),
c.Query("visitor-id"),
)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/services/eventstore/clickhouse_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ func (cs *clickhouseService) batchPageViewLoop(batchDone chan<- struct{}) {
ev.Session.Utm.Campaign,
ev.Session.Utm.Term,
ev.Session.Utm.Content,
ev.Status,
ev.Session.PageviewCount-1, // Cancel previous version.
-1,
)
Expand All @@ -227,6 +228,7 @@ func (cs *clickhouseService) batchPageViewLoop(batchDone chan<- struct{}) {
ev.Session.Utm.Campaign,
ev.Session.Utm.Term,
ev.Session.Utm.Content,
ev.Status,
ev.Session.PageviewCount,
1,
)
Expand Down
64 changes: 64 additions & 0 deletions tests/bun/events/events_pageviews.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,53 @@ test('valid internal pageview traffic with no session associated', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'mywebsite.localhost',
path: '/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
})
})

test('valid pageview with status 404', async () => {
const response = await fetch(PRISME_PAGEVIEWS_URL, {
method: 'POST',
headers: {
Origin: 'http://mywebsite.localhost',
'X-Forwarded-For': faker.internet.ip(),
Referer: 'https://mywebsite.localhost/foo?bar=baz#qux',
'X-Prisme-Status': '404'
}
})
expect(response.status).toBe(200)

const data = await getLatestPageview()

expect(data).toMatchObject({
session: {
domain: 'mywebsite.localhost',
entry_path: '/foo',
exit_timestamp: expect.stringMatching(TIMESTAMP_REGEX),
exit_path: '/foo',
exit_status: 404,
operating_system: 'Other',
browser_family: 'Other',
device: 'Other',
referrer_domain: 'direct',
country_code: expect.stringMatching(COUNTRY_CODE_REGEX),
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX),
utm_source: '',
utm_medium: '',
utm_campaign: '',
utm_term: '',
utm_content: '',
version: 1
},
pageview: {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'mywebsite.localhost',
path: '/foo',
status: 404,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -173,6 +220,7 @@ test('registered domain in Origin header and valid referrer is accepted', async
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'mywebsite.localhost',
path: '/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -217,6 +265,7 @@ test('registered domain in Origin header and valid X-Prisme-Referrer is accepted
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'mywebsite.localhost',
path: '/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -261,6 +310,7 @@ test('valid URL with registered domain in Origin header is accepted', async () =
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/another/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -306,6 +356,7 @@ test('valid pageview with Windows + Chrome user agent', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/another/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -349,6 +400,7 @@ test('valid pageview without X-Prisme-Document-Referrer', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/another/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -394,6 +446,7 @@ test('valid pageview with different Referer and X-Prisme-Referrer headers define
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'bar.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -437,6 +490,7 @@ test('valid pageview without trailing slash in referrer', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -480,6 +534,7 @@ test('valid pageview with US IP address', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/us/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -524,6 +579,7 @@ test('valid pageview with dirty path', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/another/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -568,6 +624,7 @@ test('valid pageview with UTM parameters', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -611,6 +668,7 @@ test('valid pageview with ref query parameter', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -655,6 +713,7 @@ test('valid consecutive pageviews', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -697,6 +756,7 @@ test('valid consecutive pageviews', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/foo',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -742,6 +802,7 @@ test('valid pageview with custom visitor id', async () => {
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: visitorId,
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -786,6 +847,7 @@ test('valid pageview with empty visitor id fallback to auto generated visitor id
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -830,6 +892,7 @@ test('valid consecutive pageviews with visitor id defined on second event', asyn
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/',
status: 200,
visitor_id: expect.stringMatching(PRISME_VISITOR_ID_REGEX),
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down Expand Up @@ -874,6 +937,7 @@ test('valid consecutive pageviews with visitor id defined on second event', asyn
timestamp: expect.stringMatching(TIMESTAMP_REGEX),
domain: 'foo.mywebsite.localhost',
path: '/foo',
status: 200,
visitor_id: visitorId,
session_uuid: expect.stringMatching(UUID_V7_REGEX)
}
Expand Down
Loading

0 comments on commit 9183bed

Please sign in to comment.