refactor(convex): separar heartbeat em tabela dedicada para evitar OOM

- Criar tabela machineHeartbeats para armazenar lastHeartbeatAt separadamente
- Modificar heartbeat para so atualizar machines quando ha mudancas reais
- Atualizar queries listByTenant e getById para usar nova tabela
- Reducao drastica de versoes de documentos criadas a cada heartbeat

Antes: ~54 versoes por maquina (3524 linhas para 65 maquinas)
Agora: heartbeat atualiza documento leve, machines so muda com dados novos

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
esdrasrenan 2025-12-09 23:26:30 -03:00
parent cf28ad2ee4
commit 4fbd521fa8
5 changed files with 111 additions and 51 deletions

View file

@ -49,14 +49,6 @@ import type {
FunctionReference, FunctionReference,
} from "convex/server"; } from "convex/server";
/**
* A utility for referencing Convex functions in your app's API.
*
* Usage:
* ```js
* const myFunctionReference = api.myModule.myFunction;
* ```
*/
declare const fullApi: ApiFromModules<{ declare const fullApi: ApiFromModules<{
alerts: typeof alerts; alerts: typeof alerts;
bootstrap: typeof bootstrap; bootstrap: typeof bootstrap;
@ -93,14 +85,30 @@ declare const fullApi: ApiFromModules<{
usbPolicy: typeof usbPolicy; usbPolicy: typeof usbPolicy;
users: typeof users; users: typeof users;
}>; }>;
declare const fullApiWithMounts: typeof fullApi;
/**
* A utility for referencing Convex functions in your app's public API.
*
* Usage:
* ```js
* const myFunctionReference = api.myModule.myFunction;
* ```
*/
export declare const api: FilterApi< export declare const api: FilterApi<
typeof fullApiWithMounts, typeof fullApi,
FunctionReference<any, "public"> FunctionReference<any, "public">
>; >;
/**
* A utility for referencing Convex functions in your app's internal API.
*
* Usage:
* ```js
* const myFunctionReference = internal.myModule.myFunction;
* ```
*/
export declare const internal: FilterApi< export declare const internal: FilterApi<
typeof fullApiWithMounts, typeof fullApi,
FunctionReference<any, "internal"> FunctionReference<any, "internal">
>; >;

View file

@ -10,7 +10,6 @@
import { import {
ActionBuilder, ActionBuilder,
AnyComponents,
HttpActionBuilder, HttpActionBuilder,
MutationBuilder, MutationBuilder,
QueryBuilder, QueryBuilder,
@ -19,15 +18,9 @@ import {
GenericQueryCtx, GenericQueryCtx,
GenericDatabaseReader, GenericDatabaseReader,
GenericDatabaseWriter, GenericDatabaseWriter,
FunctionReference,
} from "convex/server"; } from "convex/server";
import type { DataModel } from "./dataModel.js"; import type { DataModel } from "./dataModel.js";
type GenericCtx =
| GenericActionCtx<DataModel>
| GenericMutationCtx<DataModel>
| GenericQueryCtx<DataModel>;
/** /**
* Define a query in this Convex app's public API. * Define a query in this Convex app's public API.
* *
@ -92,11 +85,12 @@ export declare const internalAction: ActionBuilder<DataModel, "internal">;
/** /**
* Define an HTTP action. * Define an HTTP action.
* *
* This function will be used to respond to HTTP requests received by a Convex * The wrapped function will be used to respond to HTTP requests received
* deployment if the requests matches the path and method where this action * by a Convex deployment if the requests matches the path and method where
* is routed. Be sure to route your action in `convex/http.js`. * this action is routed. Be sure to route your httpAction in `convex/http.js`.
* *
* @param func - The function. It receives an {@link ActionCtx} as its first argument. * @param func - The function. It receives an {@link ActionCtx} as its first argument
* and a Fetch API `Request` object as its second.
* @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up.
*/ */
export declare const httpAction: HttpActionBuilder; export declare const httpAction: HttpActionBuilder;

View file

@ -16,7 +16,6 @@ import {
internalActionGeneric, internalActionGeneric,
internalMutationGeneric, internalMutationGeneric,
internalQueryGeneric, internalQueryGeneric,
componentsGeneric,
} from "convex/server"; } from "convex/server";
/** /**
@ -81,10 +80,14 @@ export const action = actionGeneric;
export const internalAction = internalActionGeneric; export const internalAction = internalActionGeneric;
/** /**
* Define a Convex HTTP action. * Define an HTTP action.
* *
* @param func - The function. It receives an {@link ActionCtx} as its first argument, and a `Request` object * The wrapped function will be used to respond to HTTP requests received
* as its second. * by a Convex deployment if the requests matches the path and method where
* @returns The wrapped endpoint function. Route a URL path to this function in `convex/http.js`. * this action is routed. Be sure to route your httpAction in `convex/http.js`.
*
* @param func - The function. It receives an {@link ActionCtx} as its first argument
* and a Fetch API `Request` object as its second.
* @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up.
*/ */
export const httpAction = httpActionGeneric; export const httpAction = httpActionGeneric;

View file

@ -251,6 +251,20 @@ function isObject(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value) return Boolean(value) && typeof value === "object" && !Array.isArray(value)
} }
// Busca o lastHeartbeatAt da tabela machineHeartbeats (fonte de verdade)
// Fallback para machine.lastHeartbeatAt para retrocompatibilidade durante migracao
async function getMachineLastHeartbeat(
ctx: QueryCtx | MutationCtx,
machineId: Id<"machines">,
fallback?: number | null
): Promise<number | null> {
const hb = await ctx.db
.query("machineHeartbeats")
.withIndex("by_machine", (q) => q.eq("machineId", machineId))
.first()
return hb?.lastHeartbeatAt ?? fallback ?? null
}
// Campos do inventory que sao muito grandes e nao devem ser persistidos // Campos do inventory que sao muito grandes e nao devem ser persistidos
// para evitar OOM no Convex (documentos de ~100KB cada) // para evitar OOM no Convex (documentos de ~100KB cada)
const INVENTORY_BLOCKLIST = new Set(["software", "extended"]) const INVENTORY_BLOCKLIST = new Set(["software", "extended"])
@ -827,6 +841,20 @@ export const heartbeat = mutation({
const { machine, token } = await getActiveToken(ctx, args.machineToken) const { machine, token } = await getActiveToken(ctx, args.machineToken)
const now = Date.now() const now = Date.now()
// 1. SEMPRE atualizar machineHeartbeats (documento pequeno, upsert)
// Isso evita criar versoes do documento machines a cada heartbeat
const existingHeartbeat = await ctx.db
.query("machineHeartbeats")
.withIndex("by_machine", (q) => q.eq("machineId", machine._id))
.first()
if (existingHeartbeat) {
await ctx.db.patch(existingHeartbeat._id, { lastHeartbeatAt: now })
} else {
await ctx.db.insert("machineHeartbeats", { machineId: machine._id, lastHeartbeatAt: now })
}
// 2. Preparar patch de metadata (se houver mudancas)
const metadataPatch: Record<string, unknown> = {} const metadataPatch: Record<string, unknown> = {}
if (args.metadata && typeof args.metadata === "object") { if (args.metadata && typeof args.metadata === "object") {
Object.assign(metadataPatch, args.metadata as Record<string, unknown>) Object.assign(metadataPatch, args.metadata as Record<string, unknown>)
@ -841,7 +869,22 @@ export const heartbeat = mutation({
if (args.metrics && typeof args.metrics === "object") { if (args.metrics && typeof args.metrics === "object") {
metadataPatch.metrics = args.metrics as Record<string, unknown> metadataPatch.metrics = args.metrics as Record<string, unknown>
} }
const mergedMetadata = Object.keys(metadataPatch).length ? mergeMetadata(machine.metadata, metadataPatch) : machine.metadata
// 3. Verificar se ha mudancas reais nos dados que justifiquem atualizar o documento machines
const hasMetadataChanges = Object.keys(metadataPatch).length > 0
const hasHostnameChange = args.hostname && args.hostname !== machine.hostname
const hasOsChange = args.os && (
args.os.name !== machine.osName ||
args.os.version !== machine.osVersion ||
args.os.architecture !== machine.architecture
)
const hasStatusChange = args.status && args.status !== machine.status
const needsMachineUpdate = hasMetadataChanges || hasHostnameChange || hasOsChange || hasStatusChange
// 4. So atualizar machines se houver mudancas reais (evita criar versoes desnecessarias)
// NOTA: lastHeartbeatAt agora vive na tabela machineHeartbeats, nao atualizamos mais aqui
if (needsMachineUpdate) {
const mergedMetadata = hasMetadataChanges ? mergeMetadata(machine.metadata, metadataPatch) : machine.metadata
await ctx.db.patch(machine._id, { await ctx.db.patch(machine._id, {
hostname: args.hostname ?? machine.hostname, hostname: args.hostname ?? machine.hostname,
@ -852,11 +895,11 @@ export const heartbeat = mutation({
devicePlatform: args.os?.name ?? machine.devicePlatform, devicePlatform: args.os?.name ?? machine.devicePlatform,
deviceType: machine.deviceType ?? "desktop", deviceType: machine.deviceType ?? "desktop",
managementMode: machine.managementMode ?? "agent", managementMode: machine.managementMode ?? "agent",
lastHeartbeatAt: now,
updatedAt: now, updatedAt: now,
status: args.status ?? "online", status: args.status ?? "online",
metadata: mergedMetadata, metadata: mergedMetadata,
}) })
}
if (remoteAccessSnapshot) { if (remoteAccessSnapshot) {
await upsertRemoteAccessSnapshotFromHeartbeat(ctx, machine, remoteAccessSnapshot, now) await upsertRemoteAccessSnapshotFromHeartbeat(ctx, machine, remoteAccessSnapshot, now)
@ -869,7 +912,7 @@ export const heartbeat = mutation({
}) })
// Evaluate posture/alerts & optionally create ticket // Evaluate posture/alerts & optionally create ticket
const fresh = (await ctx.db.get(machine._id)) as Doc<"machines"> const fresh = needsMachineUpdate ? (await ctx.db.get(machine._id)) as Doc<"machines"> : machine
await evaluatePostureAndMaybeRaise(ctx, fresh, { metrics: args.metrics, inventory: args.inventory, metadata: args.metadata }) await evaluatePostureAndMaybeRaise(ctx, fresh, { metrics: args.metrics, inventory: args.inventory, metadata: args.metadata })
return { return {
@ -958,6 +1001,8 @@ export const listByTenant = query({
return Promise.all( return Promise.all(
machines.map(async (machine) => { machines.map(async (machine) => {
const activeToken = await findActiveMachineToken(ctx, machine._id, now) const activeToken = await findActiveMachineToken(ctx, machine._id, now)
// Busca heartbeat da tabela separada (fonte de verdade), fallback para legado
const lastHeartbeatAt = await getMachineLastHeartbeat(ctx, machine._id, machine.lastHeartbeatAt)
const offlineThresholdMs = getOfflineThresholdMs() const offlineThresholdMs = getOfflineThresholdMs()
const staleThresholdMs = getStaleThresholdMs(offlineThresholdMs) const staleThresholdMs = getStaleThresholdMs(offlineThresholdMs)
const manualStatus = (machine.status ?? "").toLowerCase() const manualStatus = (machine.status ?? "").toLowerCase()
@ -966,8 +1011,8 @@ export const listByTenant = query({
derivedStatus = "deactivated" derivedStatus = "deactivated"
} else if (["maintenance", "blocked"].includes(manualStatus)) { } else if (["maintenance", "blocked"].includes(manualStatus)) {
derivedStatus = manualStatus derivedStatus = manualStatus
} else if (machine.lastHeartbeatAt) { } else if (lastHeartbeatAt) {
const age = now - machine.lastHeartbeatAt const age = now - lastHeartbeatAt
if (age <= offlineThresholdMs) { if (age <= offlineThresholdMs) {
derivedStatus = "online" derivedStatus = "online"
} else if (age <= staleThresholdMs) { } else if (age <= staleThresholdMs) {
@ -1050,8 +1095,8 @@ export const listByTenant = query({
linkedUsers, linkedUsers,
status: derivedStatus, status: derivedStatus,
isActive: machine.isActive ?? true, isActive: machine.isActive ?? true,
lastHeartbeatAt: machine.lastHeartbeatAt ?? null, lastHeartbeatAt: lastHeartbeatAt,
heartbeatAgeMs: machine.lastHeartbeatAt ? now - machine.lastHeartbeatAt : null, heartbeatAgeMs: lastHeartbeatAt ? now - lastHeartbeatAt : null,
registeredBy: machine.registeredBy ?? null, registeredBy: machine.registeredBy ?? null,
createdAt: machine.createdAt, createdAt: machine.createdAt,
updatedAt: machine.updatedAt, updatedAt: machine.updatedAt,
@ -1100,6 +1145,8 @@ export async function getByIdHandler(
const resolvedCompany = companyFromId ?? companyFromSlug const resolvedCompany = companyFromId ?? companyFromSlug
const activeToken = await findActiveMachineToken(ctx, machine._id, now) const activeToken = await findActiveMachineToken(ctx, machine._id, now)
// Busca heartbeat da tabela separada (fonte de verdade), fallback para legado
const lastHeartbeatAt = await getMachineLastHeartbeat(ctx, machine._id, machine.lastHeartbeatAt)
const offlineThresholdMs = getOfflineThresholdMs() const offlineThresholdMs = getOfflineThresholdMs()
const staleThresholdMs = getStaleThresholdMs(offlineThresholdMs) const staleThresholdMs = getStaleThresholdMs(offlineThresholdMs)
@ -1109,8 +1156,8 @@ export async function getByIdHandler(
derivedStatus = "deactivated" derivedStatus = "deactivated"
} else if (["maintenance", "blocked"].includes(manualStatus)) { } else if (["maintenance", "blocked"].includes(manualStatus)) {
derivedStatus = manualStatus derivedStatus = manualStatus
} else if (machine.lastHeartbeatAt) { } else if (lastHeartbeatAt) {
const age = now - machine.lastHeartbeatAt const age = now - lastHeartbeatAt
if (age <= offlineThresholdMs) { if (age <= offlineThresholdMs) {
derivedStatus = "online" derivedStatus = "online"
} else if (age <= staleThresholdMs) { } else if (age <= staleThresholdMs) {
@ -1179,8 +1226,8 @@ export async function getByIdHandler(
linkedUsers, linkedUsers,
status: derivedStatus, status: derivedStatus,
isActive: machine.isActive ?? true, isActive: machine.isActive ?? true,
lastHeartbeatAt: machine.lastHeartbeatAt ?? null, lastHeartbeatAt: lastHeartbeatAt,
heartbeatAgeMs: machine.lastHeartbeatAt ? now - machine.lastHeartbeatAt : null, heartbeatAgeMs: lastHeartbeatAt ? now - lastHeartbeatAt : null,
registeredBy: machine.registeredBy ?? null, registeredBy: machine.registeredBy ?? null,
createdAt: machine.createdAt, createdAt: machine.createdAt,
updatedAt: machine.updatedAt, updatedAt: machine.updatedAt,

View file

@ -707,6 +707,14 @@ export default defineSchema({
.index("by_tenant_created", ["tenantId", "createdAt"]) .index("by_tenant_created", ["tenantId", "createdAt"])
.index("by_tenant_machine", ["tenantId", "machineId"]), .index("by_tenant_machine", ["tenantId", "machineId"]),
// Tabela separada para heartbeats - evita criar versoes do documento machines a cada heartbeat
// O documento machines so e atualizado quando ha mudancas reais nos dados (metadata, inventory, etc)
machineHeartbeats: defineTable({
machineId: v.id("machines"),
lastHeartbeatAt: v.number(),
})
.index("by_machine", ["machineId"]),
machineTokens: defineTable({ machineTokens: defineTable({
tenantId: v.string(), tenantId: v.string(),
machineId: v.id("machines"), machineId: v.id("machines"),