perf: paginate ListEvents, gzip the HTTP responses, cache the assets

cmd/serv.go

@@ -28,6 +28,7 @@ import (
 	"github.com/bananaops/tracker/server"
 	"github.com/go-openapi/runtime/server-middleware/docui"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/klauspost/compress/gzhttp"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/spf13/cobra"
@@ -306,6 +307,7 @@ var serv = &cobra.Command{
 				_, err := os.Stat(path)
 				if os.IsNotExist(err) {
 					// File does not exist, serve index.html for SPA routing
+					w.Header().Set("Cache-Control", "no-cache")
 					http.ServeFile(w, r, filepath.Join(frontendDir, "index.html"))
 					return
 				} else if err != nil {
@@ -314,6 +316,14 @@ var serv = &cobra.Command{
 					return
 				}
 
+				// Vite names the bundled assets after their content: they never
+				// change, everything else (index.html) is revalidated.
+				if strings.HasPrefix(r.URL.Path, "/assets/") {
+					w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
+				} else {
+					w.Header().Set("Cache-Control", "no-cache")
+				}
+
 				// File exists, serve it
 				fileServer.ServeHTTP(w, r)
 			})
@@ -322,6 +332,25 @@ var serv = &cobra.Command{
 			httpHandler = mux
 		}
 
+		// Compress the SPA and the API responses. The MCP endpoint streams
+		// (text/event-stream) and is left as is. The random jitter hides the
+		// exact compressed size of responses that hold secrets (BREACH).
+		gzipWrapper, err := gzhttp.NewWrapper(
+			gzhttp.RandomJitter(32, 0, false),
+			gzhttp.ExceptContentTypes([]string{"text/event-stream"}),
+		)
+		if err != nil {
+			log.Fatalf("cannot create the gzip handler: %v", err)
+		}
+		uncompressed, compressed := httpHandler, gzipWrapper(httpHandler)
+		httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == server.MCPPath {
+				uncompressed.ServeHTTP(w, r)
+				return
+			}
+			compressed.ServeHTTP(w, r)
+		})
+
 		// Resolve the principal of every HTTP request (SPA, API and custom handlers)
 		httpHandler = auth.HTTPMiddleware(resolver, authCfg)(httpHandler)
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.21.0
 	github.com/go-openapi/runtime/server-middleware v0.33.3
 	github.com/golang-jwt/jwt/v5 v5.3.1
+	github.com/klauspost/compress v1.20.1
 	github.com/modelcontextprotocol/go-sdk v1.8.0
 	golang.org/x/crypto v0.58.0
 	golang.org/x/oauth2 v0.37.0
@@ -21,7 +22,6 @@ require (
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/jsonschema-go v0.4.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/klauspost/compress v1.20.1 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

internal/stores/event.go

@@ -45,6 +45,28 @@ func (c *EventStoreClient) List(ctx context.Context, scope auth.Scope) (results
 	return
 }
 
+// ListPage returns one page of events, newest first, and the number of
+// events in scope. page starts at 1.
+func (c *EventStoreClient) ListPage(ctx context.Context, scope auth.Scope, perPage, page int64) (results []*v1alpha1.Event, total int64, err error) {
+	filter := scopedFilter(bson.D{}, scope, eventServiceField)
+	total, err = c.collection.CountDocuments(ctx, filter)
+	if err != nil {
+		return nil, 0, err
+	}
+	opts := options.Find().
+		SetSort(bson.D{{Key: "metadata.createdat.seconds", Value: -1}, {Key: "_id", Value: -1}}).
+		SetSkip((page - 1) * perPage).
+		SetLimit(perPage)
+	cursor, err := c.collection.Find(ctx, filter, opts)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err = cursor.All(ctx, &results); err != nil {
+		return nil, 0, err
+	}
+	return results, total, nil
+}
+
 // Create takes the representation of an Event and creates it.  Returns the server's representation of the Event, and an error, if there is any.
 func (c *EventStoreClient) Create(ctx context.Context, eventInsert *v1alpha1.Event) (result *v1alpha1.Event, err error) {
 	result = &v1alpha1.Event{}

server/event.go

@@ -81,6 +81,7 @@ func addChangelogEntry(event *v1alpha1.Event, changeType v1alpha1.ChangeType, us
 const (
 	changelogDefaultPerPage = 50
 	changelogMaxPerPage     = 1000
+	listEventsMaxPerPage    = 1000
 )
 
 // shouldCreateLock détermine si un lock doit être créé pour cet événement
@@ -359,6 +360,18 @@ func (e *Event) ListEvents(
 	var eventsResult = &v1alpha1.ListEventsResponse{}
 	var err error
 
+	// Without per_page the whole collection is returned, as before.
+	if perPage := i.GetPerPage().GetValue(); perPage > 0 {
+		page := max(int64(i.GetPage().GetValue()), 1)
+		var total int64
+		eventsResult.Events, total, err = e.store.ListPage(ctx, authz.ScopeFromContext(ctx), int64(min(perPage, listEventsMaxPerPage)), page)
+		if err != nil {
+			return nil, err
+		}
+		eventsResult.TotalCount = uint32(total) // #nosec G115 -- a collection never holds 2^32 events
+		return eventsResult, nil
+	}
+
 	eventsResult.Events, err = e.store.List(context.Background(), authz.ScopeFromContext(ctx))
 	if err != nil {
 		return nil, err

server/event_scope_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 const eventSvc = "/tracker.event.v1alpha1.EventService/"
@@ -140,6 +141,27 @@ func TestListEventsScope(t *testing.T) {
 	require.Len(t, list(x.all).Events, 3)
 }
 
+func TestListEventsPagination(t *testing.T) {
+	x := newScopeEnv(t)
+	page := func(p auth.Principal, perPage uint32, n int32) *eventv1.ListEventsResponse {
+		r, err := x.s.events.ListEvents(x.ctx(p, "ListEvents"), &eventv1.ListEventsRequest{
+			PerPage: wrapperspb.UInt32(perPage), Page: wrapperspb.Int32(n),
+		})
+		require.NoError(t, err)
+		return r
+	}
+	first, second := page(x.all, 2, 1), page(x.all, 2, 2)
+	require.Len(t, first.Events, 2)
+	require.Len(t, second.Events, 1)
+	require.EqualValues(t, 3, first.TotalCount)
+	require.ElementsMatch(t, []string{x.a, x.b, x.e}, append(eventIDs(first.Events), eventIDs(second.Events)...))
+	require.Empty(t, page(x.all, 2, 3).Events)
+
+	scoped := page(x.pa, 2, 1)
+	require.ElementsMatch(t, []string{x.a}, eventIDs(scoped.Events))
+	require.EqualValues(t, 1, scoped.TotalCount)
+}
+
 func TestSearchEventsScope(t *testing.T) {
 	x := newScopeEnv(t)
 	search := func(p auth.Principal, req *eventv1.SearchEventsRequest) []*eventv1.Event {

web/src/components/LinksSearch.tsx

@@ -5,7 +5,7 @@ import { load as yamlLoad } from 'js-yaml'
 import { useNavigate } from 'react-router-dom'
 import { Dialog, DialogContent } from '@/components/ui/dialog'
 import { getLinksConfig, getHomerUrl, type LinkItem, type LinkGroup } from '../config'
-import { catalogApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import type { Catalog } from '../types/api'
 
 interface HomerLink { name: string; url: string; icon?: string }
@@ -90,8 +90,7 @@ export default function LinksSearch({ collapsed = false }: { collapsed?: boolean
   })
 
   const { data: catalogData } = useQuery({
-    queryKey: ['catalog', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 500 }),
+    ...catalogListQuery,
     staleTime: 60_000,
   })
 

web/src/lib/queries.ts

@@ -0,0 +1,9 @@
+import { catalogApi } from './api'
+
+// The whole catalog, shared by every page under one cache key. The backend
+// returns every catalog whatever the page size, and mutations invalidate the
+// ['catalog'] prefix.
+export const catalogListQuery = {
+  queryKey: ['catalog', 'list'],
+  queryFn: () => catalogApi.list({ perPage: 1000 }),
+} as const

web/src/main.tsx

@@ -43,6 +43,9 @@ const queryClient = new QueryClient({
     queries: {
       refetchOnWindowFocus: false,
       retry: 1,
+      // Data fetched less than 30s ago is reused when a page mounts instead of
+      // refetched; mutations still invalidate it right away.
+      staleTime: 30_000,
     },
   },
 })

web/src/pages/CatalogDependencies.tsx

@@ -1,6 +1,6 @@
 import { useQuery } from '@tanstack/react-query'
 import { useNavigate } from 'react-router-dom'
-import { catalogApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { SLALevel, CatalogType, Platform, type Catalog } from '../types/api'
 import { ArrowLeft, GitBranch, Search, X, Grid, List, Maximize2, ZoomIn, ZoomOut, Server, Zap, Activity, Package } from 'lucide-react'
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
@@ -217,8 +217,7 @@ export default function CatalogDependencies() {
   }
 
   const { data: allCatalogs, isLoading } = useQuery({
-    queryKey: ['catalog', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   // Build complete dependency graph with improved layouts

web/src/pages/CatalogDetail.tsx

@@ -1,6 +1,7 @@
 import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
 import { useParams, useNavigate, Link } from 'react-router-dom'
 import { catalogApi, eventsApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { SLALevel, CatalogType, Language, Platform, CommunicationType, type Catalog, type InfrastructureType, Status } from '../types/api'
 import { Package, GitBranch, Activity, ExternalLink, Github, Code, Server, Edit, Trash2, AlertTriangle, X, Mail, Zap, Plus, Database, HardDrive, Network, MessageSquare, Shield, Cloud, Check, Clock, Rocket, Maximize2, Minimize2 } from 'lucide-react'
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
@@ -466,8 +467,7 @@ export default function CatalogDetail({ serviceNameProp, onClose }: CatalogDetai
   }, [])
 
   const { data: allCatalogs, isLoading: catalogsLoading } = useQuery({
-    queryKey: ['catalog', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   // Fetch deployments for this service (last 30 days)

web/src/pages/CatalogTable.tsx

@@ -1,5 +1,5 @@
 import { useQuery } from '@tanstack/react-query'
-import { catalogApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { CatalogType, Language, SLALevel, Platform, CommunicationType, DashboardType, type Catalog } from '../types/api'
 import { Package, BookOpen, Search, X, Server, Cloud, Zap, Globe, Shield, HardDrive, Activity, Mail, Filter, SlidersHorizontal } from 'lucide-react'
 import { useState, useMemo } from 'react'
@@ -26,8 +26,7 @@ export default function CatalogTable() {
 
 
   const { data, isLoading } = useQuery({
-    queryKey: ['catalog', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 100 }),
+    ...catalogListQuery,
   })
 
 

web/src/pages/CreateCatalog.tsx

@@ -2,6 +2,7 @@ import { useState, useEffect, useMemo, type FormEvent, type ReactNode } from 're
 import { useMutation, useQueryClient, useQuery } from '@tanstack/react-query'
 import { useNavigate, useParams } from 'react-router-dom'
 import { catalogApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { CatalogType, Language, SLALevel, Platform, type Catalog, type SLA, type UsedDeliverable, type CommunicationChannel, type DashboardLink, type VulnerabilitySummary } from '../types/api'
 import { Package, X, GitBranch, Shield, Plus } from 'lucide-react'
 import DependencySelector from '../components/DependencySelector'
@@ -55,8 +56,7 @@ export default function CreateCatalog({ asPanel = false, onClose, onSuccess }: C
 
   // Fetch all catalogs for dependency selection
   const { data: allCatalogs } = useQuery({
-    queryKey: ['catalog', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   // Get list of available service names (excluding current service)

web/src/pages/CreateDrift.tsx

@@ -1,6 +1,7 @@
 import { useState, type FormEvent, type ReactNode } from 'react'
 import { useMutation, useQueryClient, useQuery } from '@tanstack/react-query'
-import { eventsApi, catalogApi } from '../lib/api'
+import { eventsApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { useNavigate } from 'react-router-dom'
 import { EventType, Priority, Status, Environment } from '../types/api'
 import type { CreateEventRequest } from '../types/api'
@@ -27,8 +28,7 @@ export default function CreateDrift({ asPanel = false, onClose, onSuccess }: Cre
   const [showToast, setShowToast] = useState(false)
 
   const { data: catalogData, isLoading: catalogLoading } = useQuery({
-    queryKey: ['catalogs', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   const catalogServices = catalogData?.catalogs.map((c) => c.name).sort() || []

web/src/pages/CreateEvent.tsx

@@ -1,7 +1,8 @@
 import { useState, type FormEvent, type ReactNode } from 'react'
 import { apiErrorMessage } from '../lib/errors'
 import { useMutation, useQueryClient, useQuery } from '@tanstack/react-query'
-import { eventsApi, catalogApi } from '../lib/api'
+import { eventsApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { useNavigate } from 'react-router-dom'
 import { EventType, Priority, Status, Environment } from '../types/api'
 import type { CreateEventRequest } from '../types/api'
@@ -27,8 +28,7 @@ export default function CreateEvent({ asPanel = false, onClose, onSuccess }: Cre
   const [showToast, setShowToast] = useState(false)
 
   const { data: catalogData, isLoading: catalogLoading } = useQuery({
-    queryKey: ['catalogs', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   const catalogServices = catalogData?.catalogs.map((c) => c.name).sort() || []

web/src/pages/CreateRpaOperation.tsx

@@ -1,6 +1,7 @@
 import { useState, type FormEvent, type ReactNode } from 'react'
 import { useMutation, useQueryClient, useQuery } from '@tanstack/react-query'
-import { eventsApi, catalogApi } from '../lib/api'
+import { eventsApi } from '../lib/api'
+import { catalogListQuery } from '../lib/queries'
 import { useNavigate } from 'react-router-dom'
 import { EventType, Priority, Status, Environment } from '../types/api'
 import type { CreateEventRequest } from '../types/api'
@@ -26,8 +27,7 @@ export default function CreateRpaOperation({ asPanel = false, onClose, onSuccess
   const [showToast, setShowToast] = useState(false)
 
   const { data: catalogData, isLoading: catalogLoading } = useQuery({
-    queryKey: ['catalogs', 'list'],
-    queryFn: () => catalogApi.list({ perPage: 1000 }),
+    ...catalogListQuery,
   })
 
   const catalogServices = catalogData?.catalogs.map((c) => c.name).sort() || []

web/src/pages/Dashboard.tsx

@@ -49,6 +49,9 @@ const T = Object.fromEntries(
 ) as Record<keyof typeof V, string>
 const a = (key: keyof typeof V, opacity: number) => `rgb(var(${V[key]}) / ${opacity})`
 
+// How far ahead the dashboard looks for the next planned event.
+const UPCOMING_HORIZON_DAYS = 90
+
 const C = {
   primary: '#1B3575',
   accent: '#E8580A',
@@ -299,17 +302,24 @@ export default function Dashboard() {
     refetchInterval: 30000,
   })
 
-  const { data: allEventsData } = useQuery({
-    queryKey: ['events', 'dashboard-all'],
-    queryFn: () => eventsApi.list({ perPage: 1000, page: 1 }),
+  // Only what is still ahead feeds "next": the whole collection is not needed.
+  const { data: upcomingEventsData } = useQuery({
+    queryKey: ['events', 'dashboard-upcoming'],
+    queryFn: () => {
+      const now = new Date()
+      return eventsApi.search({
+        startDate: now.toISOString(),
+        endDate: new Date(now.getTime() + UPCOMING_HORIZON_DAYS * 86_400_000).toISOString(),
+      })
+    },
     refetchInterval: 30000,
   })
 
   // Update lastUpdated timestamp whenever data changes
   useEffect(() => {
     setLastUpdated(new Date())
     setCountdown(30)
-  }, [todayEvents, allEventsData])
+  }, [todayEvents, upcomingEventsData])
 
   // Countdown timer
   useEffect(() => {
@@ -320,7 +330,7 @@ export default function Dashboard() {
   }, [])
 
   const events: Event[] = useMemo(() => (todayEvents?.events || []) as Event[], [todayEvents])
-  const allEvents: Event[] = useMemo(() => (allEventsData?.events || []) as Event[], [allEventsData])
+  const upcomingEvents: Event[] = useMemo(() => (upcomingEventsData?.events || []) as Event[], [upcomingEventsData])
 
   const derived = useMemo(() => {
     const toDerivedRow = (event: Event): DerivedRow => {
@@ -344,7 +354,7 @@ export default function Dashboard() {
     }, {})
 
     const now = new Date()
-    const nextRows: DerivedRow[] = (allEvents.length > 0 ? allEvents : events)
+    const nextRows: DerivedRow[] = (upcomingEvents.length > 0 ? upcomingEvents : events)
       .map((event: Event) => toDerivedRow(event))
       .filter((row: DerivedRow) => row.start >= now)
       .sort((aRow: DerivedRow, bRow: DerivedRow) => {
@@ -403,7 +413,7 @@ export default function Dashboard() {
       services: (Object.values(serviceCount) as ServiceSummary[]).sort((aRow: ServiceSummary, bRow: ServiceSummary) => bRow.count - aRow.count).slice(0, 5),
       timeline,
     }
-  }, [events, allEvents])
+  }, [events, upcomingEvents])
 
   const nowHour = new Date().getHours() + new Date().getMinutes() / 60