rough in logging and ECS schema

libs/common/src/tools/log/ecs-format/core.ts

@@ -0,0 +1,22 @@
+/** Elastic Common Schema log format - core fields.
+ */
+export interface EcsFormat {
+  "@timestamp": Date,
+
+  /** custom key/value pairs */
+  labels?: Record<string, string>,
+
+  /** system message related to the event */
+  message?: string,
+
+  /** keywords tagging the event */
+  tags?: Array<string>,
+
+  /** describe the event; it is recommended that all events have these. */
+  event: {
+    kind?: "alert" | "enrichment" | "event" | "metric" | "state",
+    category?: "api" | "authentication" | "iam" | "process" | "session" ,
+    type?: "access" | "admin" | "allowed" | "creation" | "deletion" | "denied" | "end" | "error" | "info" | "start" | "user",
+    outcome?: "failure" | "success" | "unknown",
+  }
+};

libs/common/src/tools/log/ecs-format/ecs-classifier.ts

@@ -0,0 +1,17 @@
+import { Jsonify } from "type-fest";
+
+import { Classifier } from "../../state/classifier";
+
+import { EcsFormat } from "./core";
+
+/** Removes all properties except ECS properties and the listed properties.
+*/
+export class EcsClassifier<LogFormat extends EcsFormat> implements Classifier<LogFormat, unknown, unknown> {
+  classify(value: LogFormat): { disclosed: never; secret: never; } {
+    throw new Error("Method not implemented.");
+  }
+  declassify(disclosed: never, secret: never): Jsonify<LogFormat> {
+    throw new Error("Method not implemented.");
+  }
+
+}

libs/common/src/tools/log/ecs-format/error.ts

@@ -0,0 +1,15 @@
+import { EcsFormat } from "./core";
+
+export type ErrorFormat = EcsFormat & {
+  /** Error indicators collected by the provider */
+  error: {
+    /** content from the message field of the error */
+    message: string,
+
+    /** content from the error's stack trace */
+    stack_trace: string,
+
+    /** the type of the error, for example the error's class name */
+    type: string
+  },
+};

libs/common/src/tools/log/ecs-format/event.ts

@@ -0,0 +1,29 @@
+import { LogLevelType } from "../../../platform/enums";
+
+import { EcsFormat } from "./core";
+
+/** extends core event logs with additional information */
+export type EventFormat = EcsFormat & {
+
+  event: Partial<ProcessEvent> & Partial<ApplicationEvent> & {
+    /** event severity as a number */
+    severity?: LogLevelType,
+  },
+}
+
+export type ProcessEvent = {
+  start: Date,
+  duration: number,
+  end: Date,
+};
+
+export type ApplicationEvent = {
+    /** source of the event; this is usually a client type or service name */
+    provider: string,
+
+    /** reason why the event occurred, according to the source */
+    reason: string,
+
+    /** reference URL for the event */
+    reference: string,
+};

libs/common/src/tools/log/ecs-format/index.ts

@@ -0,0 +1,5 @@
+export { EcsFormat } from "./core";
+export { ErrorFormat } from "./error";
+export { EventFormat } from "./event";
+export { LogFormat } from "./log";
+export { UserFormat } from "./user";

libs/common/src/tools/log/ecs-format/log.ts

@@ -0,0 +1,16 @@
+import { EcsFormat } from "./core";
+
+export type LogFormat = EcsFormat & {
+  /** Log metadata */
+  log: {
+    /** original log level of the event */
+    level: "debug" | "info" | "warn" | "error",
+
+    /** source of the event; this is usually a type name */
+    logger: string,
+
+    // FIXME: if it becomes possible to include line/file numbers,
+    // add the origin fields from here:
+    //   https://www.elastic.co/guide/en/ecs/current/ecs-log.html
+  }
+}

libs/common/src/tools/log/ecs-format/user.ts

@@ -0,0 +1,13 @@
+import { UserId } from "../../../types/guid"
+
+import { EcsFormat } from "./core";
+
+export type UserFormat = EcsFormat & {
+  /** Account indicators collected by the provider
+   *  WARNING: `UserFormat` should be used sparingly; it is PII.
+  */
+  user: {
+    id: UserId,
+    email?: string,
+  }
+};

libs/common/src/tools/log/log-key.ts

@@ -0,0 +1,22 @@
+// eslint-disable-next-line -- `StateDefinition` used as a type
+import { StateDefinition } from "../../platform/state/state-definition";
+import { Classifier } from "../state/classifier";
+
+import { EcsFormat } from "./ecs-format";
+
+export type LogKey<LogFormat extends EcsFormat, N extends number = 100> = {
+
+  target: "log",
+  format: "classified_circular_buffer",
+  size: N,
+  key: string,
+  state: StateDefinition;
+  cleanupDelayMs?: number;
+  classifier: Classifier<LogFormat, unknown, unknown>,
+
+  /** For encrypted outputs, determines how much padding is applied to
+   *  encoded inputs. When this isn't specified, each frame is 32 bytes
+   *  long.
+   */
+  frame?: number;
+};

libs/common/src/tools/log/log-subject-dependency-provider.ts

@@ -0,0 +1,16 @@
+import { LogService } from "../../platform/abstractions/log.service";
+import { StateProvider } from "../../platform/state";
+import { LegacyEncryptorProvider } from "../cryptography/legacy-encryptor-provider";
+
+export abstract class LogSubjectDependencyProvider {
+  /** Provides objects that encrypt and decrypt user and organization data */
+  abstract encryptor: LegacyEncryptorProvider;
+
+  /** Provides local object persistence */
+  abstract state: StateProvider;
+
+  /** `LogSubject` uses the log service instead of semantic logging
+   *   to avoid creating a loop where it logs its own actions.
+   */
+  abstract log: LogService;
+}

libs/common/src/tools/log/log-subject.ts

@@ -0,0 +1,79 @@
+import { Observable, Observer, ReplaySubject, SubjectLike, Subscription, Unsubscribable, map } from "rxjs";
+
+import { EcsFormat } from "./ecs-format";
+import { LogKey } from "./log-key";
+import { LogSubjectDependencyProvider } from "./log-subject-dependency-provider";
+
+/** A subject that captures the last N values it observes.
+ *  Subscribers use one of several endpoints to retrieve values.
+ *
+ *  `LogSubject$`: monitoring the LogSubject directly emits all captured
+ *     values individually (like `ReplaySubject`).
+ *  `LogSubject.window$(size:number)`: emit captured values in blocks less than or equal
+ *    to the size of the capture buffer.
+ *  `LogSubject.new$`: emit values received after the subscription occurs
+ */
+export class LogSubject<LogFormat extends EcsFormat>
+  extends Observable<LogFormat>
+  implements SubjectLike<LogFormat>
+{
+  constructor(
+    private key: LogKey<LogFormat>,
+    private providers: LogSubjectDependencyProvider
+  ) {
+    super();
+  }
+
+  // window$(size:number) : Observable<LogFormat[]>;
+  // new$(size:number) : Observable<LogFormat>;
+
+  next(value: LogFormat) {
+    this.input?.next(value);
+  }
+
+  error(err: any) {
+    this.input?.error(err);
+  }
+
+  complete() {
+    this.input?.complete();
+  }
+
+  /** Subscribe to the subject's event stream
+   * @param observer listening for events
+   * @returns the subscription
+   */
+  subscribe(observer?: Partial<Observer<LogFormat>> | ((value: LogFormat) => void) | null): Subscription {
+    return this.output.pipe(map((log) => log)).subscribe(observer);
+  }
+
+  // using subjects to ensure the right semantics are followed;
+  // if greater efficiency becomes desirable, consider implementing
+  // `SubjectLike` directly
+  private input? = new ReplaySubject<LogFormat>(this.key.size);
+  private readonly output = new ReplaySubject<LogFormat>(this.key.size);
+
+  private inputSubscription?: Unsubscribable;
+  private outputSubscription?: Unsubscribable;
+
+  private get isDisposed() {
+    return this.input === null;
+  }
+
+  private dispose() {
+    if (!this.isDisposed) {
+      this.providers.log.debug("disposing LogSubject");
+
+      // clean up internal subscriptions
+      this.inputSubscription?.unsubscribe();
+      this.outputSubscription?.unsubscribe();
+      this.inputSubscription = undefined;
+      this.outputSubscription = undefined;
+
+      // drop input to ensure its value is removed from memory
+      this.input = undefined;
+
+      this.providers.log.debug("disposed LogSubject");
+    }
+  }
+}