Skip to content

Commit

Permalink
[Security Solution] Resolver children pagination (elastic#74603)
Browse files Browse the repository at this point in the history
* Handle info and change events for children

* Adding sequence

* Fixing children pagination

* Fixing tests

* Adding docs
  • Loading branch information
jonathan-buttner committed Aug 10, 2020
1 parent 09a6c9d commit 4cdea7b
Show file tree
Hide file tree
Showing 12 changed files with 337 additions and 54 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ describe('data generator', () => {
generator = new EndpointDocGenerator('seed');
});

it('creates events with a numerically increasing sequence value', () => {
const event1 = generator.generateEvent();
const event2 = generator.generateEvent();

expect(event2.event.sequence).toBe(event1.event.sequence + 1);
});

it('creates the same documents with same random seed', () => {
const generator1 = new EndpointDocGenerator('seed');
const generator2 = new EndpointDocGenerator('seed');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ export function getTreeOptionsWithDef(options?: TreeOptions): TreeOptionDefaults
export class EndpointDocGenerator {
commonInfo: HostInfo;
random: seedrandom.prng;
sequence: number = 0;
constructor(seed: string | seedrandom.prng = Math.random().toString()) {
if (typeof seed === 'string') {
this.random = seedrandom(seed);
Expand Down Expand Up @@ -440,6 +441,7 @@ export class EndpointDocGenerator {
dataset: 'endpoint',
module: 'endpoint',
type: 'creation',
sequence: this.sequence++,
},
file: {
owner: 'SYSTEM',
Expand Down Expand Up @@ -586,6 +588,7 @@ export class EndpointDocGenerator {
kind: 'event',
type: options.eventType ? options.eventType : ['start'],
id: this.seededUUIDv4(),
sequence: this.sequence++,
},
host: this.commonInfo.host,
process: {
Expand Down
14 changes: 14 additions & 0 deletions x-pack/plugins/security_solution/common/endpoint/models/event.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,20 @@ export function eventId(event: ResolverEvent): number | undefined | string {
return event.event.id;
}

export function eventSequence(event: ResolverEvent): number | undefined {
if (isLegacyEvent(event)) {
return firstNonNullValue(event.endgame.serial_event_id);
}
return firstNonNullValue(event.event?.sequence);
}

export function eventSequenceSafeVersion(event: SafeResolverEvent): number | undefined {
if (isLegacyEventSafeVersion(event)) {
return firstNonNullValue(event.endgame.serial_event_id);
}
return firstNonNullValue(event.event?.sequence);
}

export function eventIDSafeVersion(event: SafeResolverEvent): number | undefined | string {
return firstNonNullValue(
isLegacyEventSafeVersion(event) ? event.endgame?.serial_event_id : event.event?.id
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/security_solution/common/endpoint/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ export interface AlertEvent {
dataset: string;
module: string;
type: string;
sequence: number;
};
Endpoint: {
policy: {
Expand Down Expand Up @@ -515,6 +516,7 @@ export interface EndpointEvent {
type: string | string[];
id: string;
kind: string;
sequence: number;
};
host: Host;
network?: {
Expand Down Expand Up @@ -591,6 +593,7 @@ export type SafeEndpointEvent = Partial<{
type: ECSField<string>;
id: ECSField<string>;
kind: ECSField<string>;
sequence: ECSField<number>;
}>;
host: Partial<{
id: ECSField<string>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,16 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ChildrenQuery } from './children';
import { PaginationBuilder } from '../utils/pagination';
import { ChildrenPaginationBuilder } from '../utils/children_pagination';
import { legacyEventIndexPattern } from './legacy_event_index_pattern';

describe('Children query', () => {
it('constructs a legacy multi search query', () => {
const query = new ChildrenQuery(new PaginationBuilder(1), 'index-pattern', 'endpointID');
const query = new ChildrenQuery(
new ChildrenPaginationBuilder(1),
'index-pattern',
'endpointID'
);
// using any here because otherwise ts complains that it doesn't know what bool and filter are
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const msearch: any = query.buildMSearch('1234');
Expand All @@ -20,7 +24,7 @@ describe('Children query', () => {
});

it('constructs a non-legacy multi search query', () => {
const query = new ChildrenQuery(new PaginationBuilder(1), 'index-pattern');
const query = new ChildrenQuery(new ChildrenPaginationBuilder(1), 'index-pattern');
// using any here because otherwise ts complains that it doesn't know what bool and filter are
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const msearch: any = query.buildMSearch(['1234', '5678']);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@
import { SearchResponse } from 'elasticsearch';
import { ResolverEvent } from '../../../../../common/endpoint/types';
import { ResolverQuery } from './base';
import { PaginationBuilder } from '../utils/pagination';
import { ChildrenPaginationBuilder } from '../utils/children_pagination';
import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';

/**
* Builds a query for retrieving descendants of a node.
*/
export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
constructor(
private readonly pagination: PaginationBuilder,
private readonly pagination: ChildrenPaginationBuilder,
indexPattern: string | string[],
endpointID?: string
) {
Expand All @@ -32,6 +32,7 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
query: {
bool: {
filter: [
...paginationFields.filters,
{
terms: { 'endgame.unique_ppid': uniquePIDs },
},
Expand Down Expand Up @@ -63,7 +64,7 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
}

protected query(entityIDs: string[]): JsonObject {
const paginationFields = this.pagination.buildQueryFieldsAsInterface('event.id');
const paginationFields = this.pagination.buildQueryFields('event.id');
return {
/**
* Using collapse here will only return a single event per occurrence of a process.entity_id. The events are sorted
Expand All @@ -80,12 +81,12 @@ export class ChildrenQuery extends ResolverQuery<ResolverEvent[]> {
collapse: {
field: 'process.entity_id',
},
// do not set the search_after field because collapse does not work with it
size: paginationFields.size,
sort: paginationFields.sort,
query: {
bool: {
filter: [
...paginationFields.filters,
{
bool: {
should: [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
ResolverChildren,
} from '../../../../../common/endpoint/types';
import { createChild } from './node';
import { PaginationBuilder } from './pagination';
import { ChildrenPaginationBuilder } from './children_pagination';

/**
* This class helps construct the children structure when building a resolver tree.
Expand Down Expand Up @@ -162,7 +162,7 @@ export class ChildrenNodesHelper {
for (const nodeEntityID of nodes.values()) {
const cachedNode = this.entityToNodeCache.get(nodeEntityID);
if (cachedNode) {
cachedNode.nextChild = PaginationBuilder.buildCursor(startEvents);
cachedNode.nextChild = ChildrenPaginationBuilder.buildCursor(startEvents);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { ResolverEvent } from '../../../../../common/endpoint/types';
import { eventSequence } from '../../../../../common/endpoint/models/event';
import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common';
import { urlEncodeCursor, SortFields, urlDecodeCursor } from './pagination';

/**
* Pagination information for the children class.
*/
export interface ChildrenPaginationCursor {
timestamp: number;
sequence: number;
}

/**
* Interface for defining the returned pagination information.
*/
export interface ChildrenPaginationFields {
sort: SortFields;
size: number;
filters: JsonObject[];
}

/**
* This class handles constructing pagination cursors that resolver can use to return additional events in subsequent
* queries.
*/
export class ChildrenPaginationBuilder {
constructor(
/**
* upper limit of how many results should be returned by the parent query.
*/
private readonly size: number,
/**
* timestamp that will be used in the search_after section
*/
private readonly timestamp?: number,
/**
* unique sequence number for the event
*/
private readonly sequence?: number
) {}

/**
* This function validates that the parsed cursor is a ChildrenPaginationCursor.
*
* @param parsed an object parsed from an encoded cursor.
*/
static decode(
parsed: ChildrenPaginationCursor | undefined
): ChildrenPaginationCursor | undefined {
if (parsed && parsed.timestamp && parsed.sequence) {
const { timestamp, sequence } = parsed;
return { timestamp, sequence };
}
}

/**
* Construct a cursor to use in subsequent queries.
*
* @param results the events that were returned by the ES query
*/
static buildCursor(results: ResolverEvent[]): string | null {
const lastResult = results[results.length - 1];
const sequence = eventSequence(lastResult);
const cursor = {
timestamp: lastResult['@timestamp'],
sequence: sequence === undefined ? 0 : sequence,
};
return urlEncodeCursor(cursor);
}

/**
* Creates a PaginationBuilder with an upper bound limit of results and a specific cursor to use to retrieve the next
* set of results.
*
* @param limit upper bound for the number of results to return within this query
* @param after a cursor to retrieve the next set of results
*/
static createBuilder(limit: number, after?: string): ChildrenPaginationBuilder {
if (after) {
try {
const cursor = urlDecodeCursor(after, ChildrenPaginationBuilder.decode);
if (cursor && cursor.timestamp && cursor.sequence) {
return new ChildrenPaginationBuilder(limit, cursor.timestamp, cursor.sequence);
}
} catch (err) {
/* tslint:disable:no-empty */
} // ignore invalid cursor values
}
return new ChildrenPaginationBuilder(limit);
}

/**
* Helper for creates an object for adding the pagination fields to a query
*
* @param tiebreaker a unique field to use as the tiebreaker for the search_after
* @returns an object containing the pagination information
*/
buildQueryFields(tiebreaker: string): ChildrenPaginationFields {
const sort: SortFields = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }];
const filters: JsonObject[] = [];
if (this.timestamp && this.sequence) {
filters.push(
{
range: {
'@timestamp': {
gte: this.timestamp,
},
},
},
{
range: {
'event.sequence': {
gt: this.sequence,
},
},
}
);
}

return { sort, size: this.size, filters };
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { ChildrenQuery } from '../queries/children';
import { QueryInfo } from '../queries/multi_searcher';
import { QueryHandler } from './fetch';
import { ChildrenNodesHelper } from './children_helper';
import { PaginationBuilder } from './pagination';
import { ChildrenPaginationBuilder } from './children_pagination';

/**
* Retrieve the start lifecycle events for the children of a resolver tree.
Expand All @@ -32,7 +32,7 @@ export class ChildrenStartQueryHandler implements QueryHandler<ChildrenNodesHelp
private readonly legacyEndpointID: string | undefined
) {
this.query = new ChildrenQuery(
PaginationBuilder.createBuilder(limit, after),
ChildrenPaginationBuilder.createBuilder(limit, after),
indexPattern,
legacyEndpointID
);
Expand All @@ -56,8 +56,13 @@ export class ChildrenStartQueryHandler implements QueryHandler<ChildrenNodesHelp
}

this.limitLeft = this.limit - this.childrenHelper.getNumNodes();

if (this.limitLeft < 0) {
this.limitLeft = 0;
}

this.query = new ChildrenQuery(
PaginationBuilder.createBuilder(this.limitLeft),
ChildrenPaginationBuilder.createBuilder(this.limitLeft),
this.indexPattern,
this.legacyEndpointID
);
Expand Down
Loading

0 comments on commit 4cdea7b

Please sign in to comment.