diff --git a/libs/common/src/tools/log/ecs-format/core.ts b/libs/common/src/tools/log/ecs-format/core.ts new file mode 100644 index 00000000000..3667be590d5 --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/core.ts @@ -0,0 +1,22 @@ +/** Elastic Common Schema log format - core fields. + */ +export interface EcsFormat { + "@timestamp": Date, + + /** custom key/value pairs */ + labels?: Record, + + /** system message related to the event */ + message?: string, + + /** keywords tagging the event */ + tags?: Array, + + /** describe the event; it is recommended that all events have these. */ + event: { + kind?: "alert" | "enrichment" | "event" | "metric" | "state", + category?: "api" | "authentication" | "iam" | "process" | "session" , + type?: "access" | "admin" | "allowed" | "creation" | "deletion" | "denied" | "end" | "error" | "info" | "start" | "user", + outcome?: "failure" | "success" | "unknown", + } +}; diff --git a/libs/common/src/tools/log/ecs-format/ecs-classifier.ts b/libs/common/src/tools/log/ecs-format/ecs-classifier.ts new file mode 100644 index 00000000000..9da65db1d0c --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/ecs-classifier.ts @@ -0,0 +1,17 @@ +import { Jsonify } from "type-fest"; + +import { Classifier } from "../../state/classifier"; + +import { EcsFormat } from "./core"; + +/** Removes all properties except ECS properties and the listed properties. +*/ +export class EcsClassifier implements Classifier { + classify(value: LogFormat): { disclosed: never; secret: never; } { + throw new Error("Method not implemented."); + } + declassify(disclosed: never, secret: never): Jsonify { + throw new Error("Method not implemented."); + } + +} diff --git a/libs/common/src/tools/log/ecs-format/error.ts b/libs/common/src/tools/log/ecs-format/error.ts new file mode 100644 index 00000000000..03b0e0623fa --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/error.ts @@ -0,0 +1,15 @@ +import { EcsFormat } from "./core"; + +export type ErrorFormat = EcsFormat & { + /** Error indicators collected by the provider */ + error: { + /** content from the message field of the error */ + message: string, + + /** content from the error's stack trace */ + stack_trace: string, + + /** the type of the error, for example the error's class name */ + type: string + }, +}; diff --git a/libs/common/src/tools/log/ecs-format/event.ts b/libs/common/src/tools/log/ecs-format/event.ts new file mode 100644 index 00000000000..7b677496b9b --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/event.ts @@ -0,0 +1,29 @@ +import { LogLevelType } from "../../../platform/enums"; + +import { EcsFormat } from "./core"; + +/** extends core event logs with additional information */ +export type EventFormat = EcsFormat & { + + event: Partial & Partial & { + /** event severity as a number */ + severity?: LogLevelType, + }, +} + +export type ProcessEvent = { + start: Date, + duration: number, + end: Date, +}; + +export type ApplicationEvent = { + /** source of the event; this is usually a client type or service name */ + provider: string, + + /** reason why the event occurred, according to the source */ + reason: string, + + /** reference URL for the event */ + reference: string, +}; diff --git a/libs/common/src/tools/log/ecs-format/index.ts b/libs/common/src/tools/log/ecs-format/index.ts new file mode 100644 index 00000000000..2c8b20f7a98 --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/index.ts @@ -0,0 +1,5 @@ +export { EcsFormat } from "./core"; +export { ErrorFormat } from "./error"; +export { EventFormat } from "./event"; +export { LogFormat } from "./log"; +export { UserFormat } from "./user"; diff --git a/libs/common/src/tools/log/ecs-format/log.ts b/libs/common/src/tools/log/ecs-format/log.ts new file mode 100644 index 00000000000..b91fa5fe814 --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/log.ts @@ -0,0 +1,16 @@ +import { EcsFormat } from "./core"; + +export type LogFormat = EcsFormat & { + /** Log metadata */ + log: { + /** original log level of the event */ + level: "debug" | "info" | "warn" | "error", + + /** source of the event; this is usually a type name */ + logger: string, + + // FIXME: if it becomes possible to include line/file numbers, + // add the origin fields from here: + // https://www.elastic.co/guide/en/ecs/current/ecs-log.html + } +} diff --git a/libs/common/src/tools/log/ecs-format/user.ts b/libs/common/src/tools/log/ecs-format/user.ts new file mode 100644 index 00000000000..61aa766ba93 --- /dev/null +++ b/libs/common/src/tools/log/ecs-format/user.ts @@ -0,0 +1,13 @@ +import { UserId } from "../../../types/guid" + +import { EcsFormat } from "./core"; + +export type UserFormat = EcsFormat & { + /** Account indicators collected by the provider + * WARNING: `UserFormat` should be used sparingly; it is PII. + */ + user: { + id: UserId, + email?: string, + } +}; diff --git a/libs/common/src/tools/log/log-key.ts b/libs/common/src/tools/log/log-key.ts new file mode 100644 index 00000000000..1a37cbb1307 --- /dev/null +++ b/libs/common/src/tools/log/log-key.ts @@ -0,0 +1,22 @@ +// eslint-disable-next-line -- `StateDefinition` used as a type +import { StateDefinition } from "../../platform/state/state-definition"; +import { Classifier } from "../state/classifier"; + +import { EcsFormat } from "./ecs-format"; + +export type LogKey = { + + target: "log", + format: "classified_circular_buffer", + size: N, + key: string, + state: StateDefinition; + cleanupDelayMs?: number; + classifier: Classifier, + + /** For encrypted outputs, determines how much padding is applied to + * encoded inputs. When this isn't specified, each frame is 32 bytes + * long. + */ + frame?: number; +}; diff --git a/libs/common/src/tools/log/log-subject-dependency-provider.ts b/libs/common/src/tools/log/log-subject-dependency-provider.ts new file mode 100644 index 00000000000..9b862409bf5 --- /dev/null +++ b/libs/common/src/tools/log/log-subject-dependency-provider.ts @@ -0,0 +1,16 @@ +import { LogService } from "../../platform/abstractions/log.service"; +import { StateProvider } from "../../platform/state"; +import { LegacyEncryptorProvider } from "../cryptography/legacy-encryptor-provider"; + +export abstract class LogSubjectDependencyProvider { + /** Provides objects that encrypt and decrypt user and organization data */ + abstract encryptor: LegacyEncryptorProvider; + + /** Provides local object persistence */ + abstract state: StateProvider; + + /** `LogSubject` uses the log service instead of semantic logging + * to avoid creating a loop where it logs its own actions. + */ + abstract log: LogService; +} diff --git a/libs/common/src/tools/log/log-subject.ts b/libs/common/src/tools/log/log-subject.ts new file mode 100644 index 00000000000..07283aac568 --- /dev/null +++ b/libs/common/src/tools/log/log-subject.ts @@ -0,0 +1,79 @@ +import { Observable, Observer, ReplaySubject, SubjectLike, Subscription, Unsubscribable, map } from "rxjs"; + +import { EcsFormat } from "./ecs-format"; +import { LogKey } from "./log-key"; +import { LogSubjectDependencyProvider } from "./log-subject-dependency-provider"; + +/** A subject that captures the last N values it observes. + * Subscribers use one of several endpoints to retrieve values. + * + * `LogSubject$`: monitoring the LogSubject directly emits all captured + * values individually (like `ReplaySubject`). + * `LogSubject.window$(size:number)`: emit captured values in blocks less than or equal + * to the size of the capture buffer. + * `LogSubject.new$`: emit values received after the subscription occurs + */ +export class LogSubject + extends Observable + implements SubjectLike +{ + constructor( + private key: LogKey, + private providers: LogSubjectDependencyProvider + ) { + super(); + } + + // window$(size:number) : Observable; + // new$(size:number) : Observable; + + next(value: LogFormat) { + this.input?.next(value); + } + + error(err: any) { + this.input?.error(err); + } + + complete() { + this.input?.complete(); + } + + /** Subscribe to the subject's event stream + * @param observer listening for events + * @returns the subscription + */ + subscribe(observer?: Partial> | ((value: LogFormat) => void) | null): Subscription { + return this.output.pipe(map((log) => log)).subscribe(observer); + } + + // using subjects to ensure the right semantics are followed; + // if greater efficiency becomes desirable, consider implementing + // `SubjectLike` directly + private input? = new ReplaySubject(this.key.size); + private readonly output = new ReplaySubject(this.key.size); + + private inputSubscription?: Unsubscribable; + private outputSubscription?: Unsubscribable; + + private get isDisposed() { + return this.input === null; + } + + private dispose() { + if (!this.isDisposed) { + this.providers.log.debug("disposing LogSubject"); + + // clean up internal subscriptions + this.inputSubscription?.unsubscribe(); + this.outputSubscription?.unsubscribe(); + this.inputSubscription = undefined; + this.outputSubscription = undefined; + + // drop input to ensure its value is removed from memory + this.input = undefined; + + this.providers.log.debug("disposed LogSubject"); + } + } +}