From 444d841d460a138f640206a7d14189b9bca90514 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 14 Apr 2021 16:06:08 -0700 Subject: [PATCH 01/41] WIP --- src/js/flows/magic.ts | 41 +++ .../-from 1583768524.415634000 -to 158377486 | 11 + src/pkg/brimcap/index.ts | 5 + src/pkg/brimcap/load.test.ts | 5 + test/data/count-by-typeof.zson | 2 + test/data/sample.zson | 80 ++++++ zealot/zed/count_by.json | 73 +++++ zealot/zed/star.json | 48 ++++ zealot/zed/zjson.test.ts | 23 ++ zealot/zed/zjson.ts | 272 ++++++++++++++++++ 10 files changed, 560 insertions(+) create mode 100644 src/js/flows/magic.ts create mode 100644 src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 create mode 100644 src/pkg/brimcap/index.ts create mode 100644 src/pkg/brimcap/load.test.ts create mode 100644 test/data/count-by-typeof.zson create mode 100644 test/data/sample.zson create mode 100644 zealot/zed/count_by.json create mode 100644 zealot/zed/star.json create mode 100644 zealot/zed/zjson.test.ts create mode 100644 zealot/zed/zjson.ts diff --git a/src/js/flows/magic.ts b/src/js/flows/magic.ts new file mode 100644 index 0000000000..4c4a7b3331 --- /dev/null +++ b/src/js/flows/magic.ts @@ -0,0 +1,41 @@ +// import brimcap from "src/pkg/brimcap" +// import {file} from "tmp" +// import brim from "../brim" +// import space from "../brim/space" +// import transaction from "../lib/transaction" + +// function loadFile(file, space) { +// return brimcap // promise +// .load({ +// space, +// file, +// progress: trackProgress +// }) +// } + +// function trackProgress() { +// // do someting here +// } + +// function isPcap() {} + +// // pcap format +// function activate() { +// brim.importers.add({ +// name: "PCAP Loader", +// match: isPcap, +// import: loadFile +// }) +// } + +// // default loading +// brim.importers.add({ +// name: "Default Loader", +// match: () => true, +// import: ({files, space, progress}) => { +// await brim.backend.post(file, space) +// progress(0.5) +// } +// }) + +// const importer = brim.importers.matching(file) diff --git a/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 b/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 new file mode 100644 index 0000000000..876a280360 --- /dev/null +++ b/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 @@ -0,0 +1,11 @@ +-from 1583768524.415634000 -to 1583774869.009422 -p tcp 192.168.10.120:62458 34.232.129.83:443 + +# Successful brimcap launch +build/dist/brimcap launch -root ~/work/pcaps/root -ts "2020-03-09T15:42:04.415Z" -duration "6344.593788000s" -proto=tcp -src.ip 192.168.10.120 -src.port 62458 -dst.ip 34.232.129.83 -dst.port 443 + + +ts_sec=1583768524 +ts_ns=415634000 +duration_sec=6344 +duration_ns=593788000 +proto=tcp&src_host=192.168.10.120&src_port=62458&dst_host=34.232.129.83&dst_port=443 \ No newline at end of file diff --git a/src/pkg/brimcap/index.ts b/src/pkg/brimcap/index.ts new file mode 100644 index 0000000000..c8223c6aaa --- /dev/null +++ b/src/pkg/brimcap/index.ts @@ -0,0 +1,5 @@ +function load() {} + +export default { + load +} diff --git a/src/pkg/brimcap/load.test.ts b/src/pkg/brimcap/load.test.ts new file mode 100644 index 0000000000..82f2dcec55 --- /dev/null +++ b/src/pkg/brimcap/load.test.ts @@ -0,0 +1,5 @@ +import brimcap from "./" + +test("load function", async () => { + await brimcap.load() +}) diff --git a/test/data/count-by-typeof.zson b/test/data/count-by-typeof.zson new file mode 100644 index 0000000000..92c395e193 --- /dev/null +++ b/test/data/count-by-typeof.zson @@ -0,0 +1,2 @@ +{ first_name: "James" } (=person_schema) +{ score: 100 } diff --git a/test/data/sample.zson b/test/data/sample.zson new file mode 100644 index 0000000000..50ee241261 --- /dev/null +++ b/test/data/sample.zson @@ -0,0 +1,80 @@ +{ + _path: "conn", + ts: 2018-03-24T17:15:20.600725Z, + uid: "C1zOivgBT6dBmknqk" (bstring), + id: { + orig_h: 10.47.1.152, + orig_p: 49562 (port=(uint16)), + resp_h: 23.217.103.245, + resp_p: 80 (port) + } (=0), + proto: "tcp" (=zenum), + service: null (bstring), + duration: 9.698493s (duration), + orig_bytes: 0 (uint64), + resp_bytes: 90453565 (uint64), + conn_state: "SF" (bstring), + local_orig: null, + local_resp: null, + missed_bytes: 0 (uint64), + history: "^dtAttttFf" (bstring), + orig_pkts: 57490 (uint64), + orig_ip_bytes: 2358856 (uint64), + resp_pkts: 123713 (uint64), + resp_ip_bytes: 185470730 (uint64), + tunnel_parents: null (=1) +} (=2) +{ + _path: "conn", + ts: 2018-03-24T17:15:20.6008Z, + uid: "CfbnHCmClhWXY99ui", + id: { + orig_h: 10.128.0.207, + orig_p: 13, + resp_h: 10.47.19.254, + resp_p: 14 + }, + proto: "icmp", + service: null, + duration: 1.278ms, + orig_bytes: 336, + resp_bytes: 0, + conn_state: "OTH", + local_orig: null, + local_resp: null, + missed_bytes: 0, + history: null, + orig_pkts: 28, + orig_ip_bytes: 1120, + resp_pkts: 0, + resp_ip_bytes: 0, + tunnel_parents: null +} (2) +{ + array: ["1", "2", "3"], + set: |["a", "b", "c"]|, + union: "hi james" ((string, int64)), + map: |{ {"best", 1}, {"worst", 0} }|, + null: null +} +{ city: "Berkeley", state: "CA", population: 121643 (uint32) } (=city_schema) +{ city: "Broad Cove", state: "ME", population: 806 } (city_schema) +{ city: "Baton Rouge", state: "LA", population: 221599 } (city_schema) +{ + info: "Connection Example", + src: { addr: 10.1.1.2, port: 80 (uint16) } (=socket), + dst: { addr: 10.0.1.2, port: 20130 } (socket) +} (=conn) +{ + info: "Connection Example 2", + src: { addr: 10.1.1.8, port: 80 }, + dst: { addr: 10.1.2.88, port: 19801 } +} (conn) +{ + info: "Access List Example", + nets: [ 10.1.1.0/24, 10.1.2.0/24 ] +} (=access_list) +{ metric: "A", ts: 2020-11-24T08:44:09.586441-08:00, value: 120 } +{ metric: "B", ts: 2020-11-24T08:44:20.726057-08:00, value: 0.86 } +{ metric: "A", ts: 2020-11-24T08:44:32.201458-08:00, value: 126 } +{ metric: "C", ts: 2020-11-24T08:44:43.547506-08:00, value: { x:10, y:101 } } diff --git a/zealot/zed/count_by.json b/zealot/zed/count_by.json new file mode 100644 index 0000000000..1b0cd2542f --- /dev/null +++ b/zealot/zed/count_by.json @@ -0,0 +1,73 @@ +{ + "schema": "26", + "types": [ + { + "kind": "typedef", + "name": "26", + "type": { + "kind": "record", + "fields": [ + { + "name": "typeof", + "type": { + "kind": "primitive", + "name": "type" + } + }, + { + "name": "count", + "type": { + "kind": "primitive", + "name": "uint64" + } + } + ] + } + }, + { + "kind": "typedef", + "name": "person_schema", + "type": { + "kind": "record", + "fields": [ + { + "name": "first_name", + "type": { + "kind": "primitive", + "name": "string" + } + } + ] + } + } + ], + "values": [ + "24", + "1" + ] +} +{ + "schema": "26", + "types": [ + { + "kind": "typedef", + "name": "25", + "type": { + "kind": "record", + "fields": [ + { + "name": "score", + "type": { + "kind": "primitive", + "name": "int64" + } + } + ] + } + } + ], + "values": [ + "25", + "1" + ] +} diff --git a/zealot/zed/star.json b/zealot/zed/star.json new file mode 100644 index 0000000000..8d62f1d020 --- /dev/null +++ b/zealot/zed/star.json @@ -0,0 +1,48 @@ +{ + "schema": "person_schema", + "types": [ + { + "kind": "typedef", + "name": "person_schema", + "type": { + "kind": "record", + "fields": [ + { + "name": "first_name", + "type": { + "kind": "primitive", + "name": "string" + } + } + ] + } + } + ], + "values": [ + "James" + ] +} +{ + "schema": "25", + "types": [ + { + "kind": "typedef", + "name": "25", + "type": { + "kind": "record", + "fields": [ + { + "name": "score", + "type": { + "kind": "primitive", + "name": "int64" + } + } + ] + } + } + ], + "values": [ + "100" + ] +} diff --git a/zealot/zed/zjson.test.ts b/zealot/zed/zjson.test.ts new file mode 100644 index 0000000000..f90ea7b2f4 --- /dev/null +++ b/zealot/zed/zjson.test.ts @@ -0,0 +1,23 @@ +import {execSync} from "child_process" +import {parse} from "./zjson" + +function zq(q, file) { + const zed = "/Users/jkerr/tools/go/bin/zed" + const cmd = `${zed} query -f zjson "${q}" ${file}` + return execSync(cmd, {encoding: "utf-8"}) + .trim() + .split("\n") + .map((s) => JSON.parse(s)) +} + +test("can correlate?", () => { + const file = "test/data/count-by-typeof.zson" + const list = parse(zq("*", file)) + const chart = parse(zq("count() by typeof(.)", file)) + + // I need the name of the typedef + console.log(list.zjson.map((o) => o.types)) + console.log(chart.zjson.map((o) => o.types)) + // console.log(list.rows.map((r) => r._name)) + // console.log(chart.rows.map((r) => r.fields[0])) +}) diff --git a/zealot/zed/zjson.ts b/zealot/zed/zjson.ts new file mode 100644 index 0000000000..9a8fa3a577 --- /dev/null +++ b/zealot/zed/zjson.ts @@ -0,0 +1,272 @@ +type ID = string | number + +type Value = string | null | Value[] + +type Field = { + name: string + type: Type +} + +type PrimitiveType = { + kind: "primitive" + name: string +} + +type RecordType = { + kind: "record" + fields: Field[] +} + +type ArrayType = { + kind: "array" + type: Type +} + +type SetType = { + kind: "set" + type: Type +} + +type UnionType = { + kind: "union" + types: Type[] +} + +type EnumType = { + kind: "enum" + symbols: string[] +} + +type MapType = { + kind: "map" + key_type: Type + val_type: Type +} + +type TypeDefType = { + kind: "typedef" + name: ID + type: Type +} + +type TypeNameType = { + kind: "typename" + name: ID +} + +type Type = + | PrimitiveType + | RecordType + | ArrayType + | SetType + | UnionType + | EnumType + | MapType + | TypeDefType + | TypeNameType + +type StreamObject = { + schema: ID + values: Value[] + types: TypeDefType[] +} + +type TypeName = string | number + +class ZedType { + protected _type: T + protected _value: V + protected _name?: TypeName + + constructor(args: {type: T; value: V; name?: TypeName}) { + this._type = args.type + this._value = args.value + this._name = args.name + } + + missing() { + return false + } + + toString() { + return this._value ? this._value.toString() : "" + } +} + +class Missing { + constructor(readonly type: typeof ZedType, readonly name: string) { + return new Proxy(this, { + get(target, prop, receiver) { + if (target[prop]) { + return Reflect.get(target, prop, receiver) + } else { + return receiver + } + } + }) + } + + toString() { + return `Missing: "${this.name}" in ${this.type}` + } + + missing() { + return true + } +} + +type ZedData = + | ZedPrimitive + | ZedRecord + | ZedTypeDef + | ZedArray + | ZedSet + | ZedUnion + | ZedEnum + | ZedMap + | ZedTypeName + +class ZedField { + name: string + data: ZedData + + constructor({name, data}: {name: string; data: ZedData}) { + this.name = name + this.data = data + } +} + +function withQuickAccess(record, accessor) { + return new Proxy(record, { + get(target, property, receiver) { + if (target[property]) { + return Reflect.get(target, property, receiver) + } + const data = accessor(target, property) + return data ? data : new Missing(target, property.toString()) + } + }) +} + +class ZedPrimitive extends ZedType { + constructor(args) { + super(args) + return withQuickAccess(this, () => false) + } +} +class ZedTypeDef extends ZedType { + get name() { + return this._type.name + } +} +class ZedArray extends ZedType {} +class ZedSet extends ZedType {} +class ZedUnion extends ZedType {} +class ZedEnum extends ZedType {} +class ZedMap extends ZedType {} +class ZedTypeName extends ZedType {} + +class ZedRecord extends ZedType { + fields: ZedField[] + constructor(args) { + super(args) + this.fields = this._value.map((value, index) => { + const {name, type} = this._type.fields[index] + return new ZedField({ + name, + data: construct(args.context, type, value, args.name) + }) + }) + return withQuickAccess( + this, + (record, prop) => record.fields.find((f) => f.name === prop)?.data + ) + } + + [Symbol.iterator]() { + let index = 0 + const next = () => + index < this._type.fields.length + ? {done: false, value: this.at(index++)} + : {done: true, value: undefined} + + return {next} + } + + at(index: number) { + return this.fields[index] + } + + get columns() { + return this.fields.map((f) => f.name) + } +} + +class Rows extends Array { + first() { + return this[0] + } + last() { + return this[this.length - 1] + } +} + +function construct( + context: TypeContext, + type: Type, + value: Value, + name?: TypeName +) { + switch (type.kind) { + case "record": + return new ZedRecord({type, value, context, name}) + case "primitive": + return new ZedPrimitive({type, value: value as string, name}) + case "array": + return new ZedArray({type, value: value as Value[], name}) + case "set": + return new ZedSet({type, value: value as Value[], name}) + case "union": + return new ZedUnion({type, value: value as Value[], name}) + case "enum": + return new ZedEnum({type, value: value as string, name}) + case "map": + return new ZedMap({type, value: value as Value[], name}) + case "typename": + var typedef = context.get(type.name) + return construct(context, typedef.type, value, typedef.name) + case "typedef": + context.set(type.name, type) + return construct(context, type.type, value, type.name) + default: + throw new Error(`Unknown ZJSON Type: ${JSON.stringify(type)}`) + } +} + +type TypeContext = Map + +export function parse(zjson: StreamObject[]) { + const rows = new Rows() + const context: TypeContext = new Map() + const schemas = new Map() + + for (const object of zjson) { + if (object.schema) schemas.set(object.schema, true) + if (object.types) { + object.types.forEach((typedef) => context.set(typedef.name, typedef)) + } + const typedef = context.get(object.schema) + + const type = typedef.type + const name = typedef.name + const value = object.values + rows.push(construct(context, type, value, name)) + } + + return { + zjson, + rows, + schemas, + context + } +} From 3022764a5c4a710aa1ec9907ee76e2053b6b46a8 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 15 Apr 2021 13:27:55 -0700 Subject: [PATCH 02/41] Updated to most recent branch --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index d94968317d..e2a57daac1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24944,8 +24944,8 @@ } }, "zed": { - "version": "git+https://github.com/brimdata/zed.git#63c19f71e7afae23254a1eb39db4d1671837c8e6", - "from": "git+https://github.com/brimdata/zed.git#63c19f71e7afae23254a1eb39db4d1671837c8e6" + "version": "git+https://github.com/brimdata/zed.git#61e611d8ee8f52675aca41f890112e0aff09848a", + "from": "git+https://github.com/brimdata/zed.git#61e611d8ee8f52675aca41f890112e0aff09848a" }, "zip-folder": { "version": "1.0.0", diff --git a/package.json b/package.json index 2b08e187cf..854c2e002e 100644 --- a/package.json +++ b/package.json @@ -154,7 +154,7 @@ "styled-components": "^5.1.1", "tree-model": "^1.0.7", "valid-url": "^1.0.9", - "zed": "git+https://github.com/brimdata/zed.git#63c19f71e7afae23254a1eb39db4d1671837c8e6" + "zed": "git+https://github.com/brimdata/zed.git#61e611d8ee8f52675aca41f890112e0aff09848a" }, "optionalDependencies": { "electron-installer-debian": "^3.0.0", From 1aeaebfb7a9fd8fd6ca56a8b777eb6dab128bf1e Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 19 Apr 2021 09:57:37 -0700 Subject: [PATCH 03/41] Work in progress --- app/core/hooks/useSearch.ts | 8 +- app/detail/Fields.tsx | 14 +- app/detail/Pane.tsx | 6 +- app/detail/flows/contextMenu.ts | 4 +- app/search/flows/histogram-search.test.ts | 137 +- app/search/flows/histogram-search.ts | 2 +- .../flows/next-page-viewer-search.test.ts | 47 +- app/search/flows/next-page-viewer-search.ts | 9 +- app/search/flows/viewer-search.test.ts | 213 +- app/search/flows/viewer-search.ts | 30 +- app/tile/viz.tsx | 10 +- port-array.zson | 3 + ppl/detail/Correlation.tsx | 4 +- ppl/detail/RelatedAlerts.tsx | 25 +- ppl/detail/RelatedConns.tsx | 22 +- ppl/detail/flows/fetch.test.ts | 34 +- ppl/detail/flows/fetch.ts | 8 +- .../flows/get-correlation-query.test.ts | 67 +- ppl/detail/flows/get-correlation-query.ts | 12 +- ppl/detail/flows/stubs.ts | 1869 ----------------- ppl/detail/models/BrimEvent.ts | 8 +- ppl/detail/models/Correlation.ts | 8 +- ppl/detail/models/SuricataEvent.ts | 8 +- ppl/detail/models/UnknownEvent.ts | 6 +- ppl/detail/models/ZeekEvent.ts | 10 +- ppl/detail/util/sort.ts | 6 +- ppl/menus/detailFieldContextMenu.ts | 13 +- ppl/menus/searchFieldContextMenu.ts | 17 +- ppl/summary/bar-chart.tsx | 14 +- ppl/summary/horizontal-bar-chart.tsx | 14 +- scripts/rename.js | 23 + src/css/shared/_type-colors.scss | 10 + src/js/brim/ast.ts | 2 +- src/js/brim/cell.test.ts | 100 - src/js/brim/cell.ts | 12 +- src/js/brim/complexCell.ts | 23 +- src/js/brim/primitiveCell.ts | 8 +- src/js/brim/program.test.ts | 54 +- src/js/brim/program.ts | 9 +- src/js/components/ConnVersation.tsx | 33 +- src/js/components/FieldCell.tsx | 11 +- src/js/components/LogCell/CompoundField.tsx | 20 +- src/js/components/LogCell/SingleField.tsx | 11 +- src/js/components/LogCell/index.tsx | 30 +- src/js/components/LogDetails/ConnPanel.tsx | 7 +- src/js/components/LogDetails/Md5Panel.tsx | 29 +- src/js/components/LogRow.test.tsx | 38 - src/js/components/LogRow.tsx | 12 +- src/js/components/RightPane.test.tsx | 4 +- src/js/components/RightPane.tsx | 4 +- .../components/SearchResults/ResultsTable.tsx | 4 +- src/js/components/Tables/HorizontalTable.tsx | 11 +- src/js/components/Tables/Table.tsx | 16 +- src/js/components/Tables/VerticalTable.tsx | 17 +- src/js/components/Viewer/Chunk.tsx | 7 +- src/js/components/Viewer/HeaderCell.tsx | 1 - src/js/components/Viewer/Viewer.tsx | 4 +- src/js/electron/menu/actions/detailActions.ts | 80 +- src/js/electron/menu/actions/searchActions.ts | 124 +- src/js/flows/downloadPcap.ts | 8 +- src/js/flows/rightclick/cellMenu.test.ts | 47 +- src/js/flows/scrollToLog.ts | 6 +- src/js/flows/search/handler.ts | 13 +- src/js/flows/search/response.ts | 3 +- src/js/flows/searchBar/actions.ts | 18 +- src/js/flows/viewLogDetail.ts | 10 +- src/js/models/TableColumns.ts | 19 +- src/js/searches/programs.test.ts | 100 +- src/js/searches/programs.ts | 18 +- src/js/state/Chart/actions.ts | 12 +- src/js/state/Chart/test.ts | 20 +- src/js/state/Columns/models/column.ts | 20 +- src/js/state/Columns/models/columnSet.ts | 23 +- src/js/state/Columns/selectors.ts | 19 +- src/js/state/Columns/touch.test.ts | 49 +- src/js/state/Columns/touch.ts | 4 +- src/js/state/LogDetails/actions.ts | 8 +- src/js/state/LogDetails/selectors.ts | 22 +- src/js/state/LogDetails/test.ts | 24 +- src/js/state/LogDetails/types.ts | 8 +- src/js/state/Packets/flows.ts | 46 +- src/js/state/SearchBar/test.ts | 34 +- src/js/state/Viewer/actions.ts | 6 +- src/js/state/Viewer/reducer.ts | 6 +- src/js/state/Viewer/selectors.ts | 26 +- src/js/state/Viewer/test.ts | 63 +- src/js/state/Viewer/types.ts | 13 +- src/js/test/mockLogs.ts | 234 --- src/js/types/index.ts | 8 +- src/js/zql/toZql.ts | 11 +- test/api/scripts/search.ts | 2 +- test/fixtures/zjson-types.ts | 7 + test/responses/index.ts | 5 + zealot/config/search_args.ts | 3 +- zealot/enhancers/mod.ts | 1 - zealot/enhancers/zngToZeek.ts | 74 - zealot/enhancers/zngToZeek_test.ts | 101 - zealot/fetcher/records_callback.ts | 49 +- zealot/fetcher/stream.ts | 10 +- zealot/zealot_mock.ts | 4 +- zealot/zed/construct.ts | 48 + zealot/zed/count_by.json | 73 - zealot/zed/data-types.ts | 304 +++ zealot/zed/star.json | 48 - zealot/zed/type-def.ts | 41 + zealot/zed/zjson.test.ts | 12 +- zealot/zed/zjson.ts | 241 +-- zealot/zng/types/record.ts | 6 +- zealot/zqd.ts | 4 +- 109 files changed, 1493 insertions(+), 3770 deletions(-) create mode 100644 port-array.zson delete mode 100644 ppl/detail/flows/stubs.ts create mode 100644 scripts/rename.js delete mode 100644 src/js/brim/cell.test.ts delete mode 100644 src/js/components/LogRow.test.tsx delete mode 100644 src/js/test/mockLogs.ts create mode 100644 test/fixtures/zjson-types.ts create mode 100644 test/responses/index.ts delete mode 100644 zealot/enhancers/mod.ts delete mode 100644 zealot/enhancers/zngToZeek.ts delete mode 100644 zealot/enhancers/zngToZeek_test.ts create mode 100644 zealot/zed/construct.ts delete mode 100644 zealot/zed/count_by.json create mode 100644 zealot/zed/data-types.ts delete mode 100644 zealot/zed/star.json create mode 100644 zealot/zed/type-def.ts diff --git a/app/core/hooks/useSearch.ts b/app/core/hooks/useSearch.ts index 0a1505990e..3fa55ca36f 100644 --- a/app/core/hooks/useSearch.ts +++ b/app/core/hooks/useSearch.ts @@ -2,19 +2,19 @@ import {useEffect, useState} from "react" import {useDispatch} from "react-redux" import {search} from "src/js/flows/search/mod" import {AppDispatch} from "src/js/state/types" -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" -type R = [zng.Record[], boolean] +type R = [ZedRecord[], boolean] export default function useSearch(query: string, deps?: any[]): R { const dispatch = useDispatch() - const [records, setRecords] = useState([]) + const [records, setRecords] = useState([]) const [isFetching, setIsFetching] = useState(true) useEffect(() => { setIsFetching(true) const {response, abort} = dispatch(search({query})) - response.chan(0, (records) => setRecords(records)) + response.chan(0, ({rows}) => setRecords(rows)) response.status((status) => setIsFetching(status === "FETCHING")) return abort }, deps) diff --git a/app/detail/Fields.tsx b/app/detail/Fields.tsx index 446d5aa94d..8a46cbfed3 100644 --- a/app/detail/Fields.tsx +++ b/app/detail/Fields.tsx @@ -1,7 +1,6 @@ import {useDispatch} from "react-redux" import React, {memo, useCallback, useMemo, useState} from "react" -import {zng} from "zealot" import {Data, Name, Value} from "app/core/Data" import {createCell} from "src/js/brim/cell" import BrimTooltip from "src/js/components/BrimTooltip" @@ -10,15 +9,16 @@ import ColumnDescription from "src/js/components/LogDetails/ColumnDescription" import PanelHeading from "./PanelHeading" import Panel from "./Panel" import contextMenu from "./flows/contextMenu" +import {ZedField, ZedRecord} from "zealot/zed/data-types" type Props = { - record: zng.Record + record: ZedRecord } type DTProps = { - fields: zng.Field[] - onRightClick: (f: zng.Field) => void - onHover: (f: zng.Field) => void + fields: ZedField[] + onRightClick: (f: ZedField) => void + onHover: (f: ZedField) => void } const DataPanel = React.memo(function DataTable({ @@ -34,7 +34,7 @@ const DataPanel = React.memo(function DataTable({ {field.name} onRightClick(field)} > {createCell(field).display()} @@ -85,7 +85,7 @@ export default memo(function Fields({record}: Props) { const fields = useMemo(() => { if (!record) return [] - else return record.flatten().getFields() + else return record.flatten().fields }, [record]) return ( diff --git a/app/detail/Pane.tsx b/app/detail/Pane.tsx index 85d0aebf5e..1c25f49bf4 100644 --- a/app/detail/Pane.tsx +++ b/app/detail/Pane.tsx @@ -11,9 +11,9 @@ import ConnPanel from "src/js/components/LogDetails/ConnPanel" import {Md5Panel} from "src/js/components/LogDetails/Md5Panel" import LogDetails from "src/js/state/LogDetails" import styled from "styled-components" -import {zng} from "zealot" import Fields from "./Fields" import NoSelection from "./NoSelection" +import {ZedRecord} from "zealot/zed/data-types" const BG = styled.div` padding: 12px; @@ -24,7 +24,7 @@ const BG = styled.div` ` type Props = { - record: zng.Record + record: ZedRecord } const Content = memo(function Content({record}) { @@ -32,7 +32,7 @@ const Content = memo(function Content({record}) { const isZeek = event instanceof ZeekEvent const isSuricata = event instanceof SuricataEvent const {uid, cid} = new Correlation(record).getIds() - const isConn = isZeek && record.try("_path").toString() === "conn" + const isConn = isZeek && record["_path"].toString() === "conn" const hasMd5 = isZeek && record.has("md5") return ( diff --git a/app/detail/flows/contextMenu.ts b/app/detail/flows/contextMenu.ts index 90c31812fb..a0116dd03f 100644 --- a/app/detail/flows/contextMenu.ts +++ b/app/detail/flows/contextMenu.ts @@ -4,9 +4,9 @@ import {showContextMenu} from "src/js/lib/System" import Columns from "src/js/state/Columns" import Current from "src/js/state/Current" import SearchBar from "src/js/state/SearchBar" -import {zng} from "zealot" +import {ZedField, ZedRecord} from "zealot/zed/data-types" -const contextMenu = (field: zng.Field, record: zng.Record) => (_, getState) => { +const contextMenu = (field: ZedField, record: ZedRecord) => (_, getState) => { const space = Current.mustGetSpace(getState()) const program = SearchBar.getSearchProgram(getState()) const tableColumns = Columns.getCurrentTableColumns(getState()) diff --git a/app/search/flows/histogram-search.test.ts b/app/search/flows/histogram-search.test.ts index 61101fd3d0..8f95328f2a 100644 --- a/app/search/flows/histogram-search.test.ts +++ b/app/search/flows/histogram-search.test.ts @@ -1,77 +1,78 @@ -import tabHistory from "app/router/tab-history" -import {lakePath} from "app/router/utils/paths" -import Chart from "src/js/state/Chart" -import Handlers from "src/js/state/Handlers" -import Spaces from "src/js/state/Spaces" -import Workspaces from "src/js/state/Workspaces" -import fixtures from "src/js/test/fixtures" -import initTestStore from "src/js/test/initTestStore" -import responses from "src/js/test/responses" -import {createZealotMock} from "zealot" -import {histogramSearch} from "./histogram-search" +// import tabHistory from "app/router/tab-history" +// import {lakePath} from "app/router/utils/paths" +// import Chart from "src/js/state/Chart" +// import Handlers from "src/js/state/Handlers" +// import Spaces from "src/js/state/Spaces" +// import Workspaces from "src/js/state/Workspaces" +// import fixtures from "src/js/test/fixtures" +// import initTestStore from "src/js/test/initTestStore" +// import responses from "src/js/test/responses" +// import {createZealotMock} from "zealot" +// import {histogramSearch} from "./histogram-search" -const countByPathResp = responses("count_by_path.txt") -const space = fixtures("space1") +// const countByPathResp = responses("count_by_path.txt") +// const space = fixtures("space1") -let store, zealot, dispatch, select -beforeEach(() => { - zealot = createZealotMock() - store = initTestStore(zealot.zealot) - dispatch = store.dispatch - select = (s: any) => s(store.getState()) +// let store, zealot, dispatch, select +// beforeEach(() => { +// zealot = createZealotMock() +// store = initTestStore(zealot.zealot) +// dispatch = store.dispatch +// select = (s: any) => s(store.getState()) - store.dispatchAll([ - Workspaces.add({ - host: "testHost", - id: "1", - name: "testName", - port: "9867", - authType: "none" - }), - Spaces.setDetail("1", space) - ]) - store.dispatch(tabHistory.push(lakePath(space.id, "1"))) - zealot.stubStream("search", countByPathResp) -}) +// store.dispatchAll([ +// Workspaces.add({ +// host: "testHost", +// id: "1", +// name: "testName", +// port: "9867", +// authType: "none" +// }), +// Spaces.setDetail("1", space) +// ]) +// store.dispatch(tabHistory.push(lakePath(space.id, "1"))) +// zealot.stubStream("search", countByPathResp) +// }) -const submit = () => dispatch(histogramSearch()) +// const submit = () => dispatch(histogramSearch()) -test("zealot gets the request", async () => { - await submit() - const calls = zealot.calls("search") - expect(calls.length).toBe(1) - expect(calls[0].args).toEqual("* | every 12h count() by _path") -}) +// test("zealot gets the request", async () => { +// await submit() +// const calls = zealot.calls("search") +// expect(calls.length).toBe(1) +// expect(calls[0].args).toEqual("* | every 12h count() by _path") +// }) -test("the chart status updates", async () => { - const promise = submit() - expect(select(Chart.getStatus)).toBe("FETCHING") - await promise - expect(select(Chart.getStatus)).toBe("SUCCESS") -}) +// test("the chart status updates", async () => { +// const promise = submit() +// expect(select(Chart.getStatus)).toBe("FETCHING") +// await promise +// expect(select(Chart.getStatus)).toBe("SUCCESS") +// }) -test("registers historgram request then cleans it up", async (done) => { - const promise = submit() - expect(select(Handlers.get)["Histogram"]).toEqual( - expect.objectContaining({type: "SEARCH"}) - ) - await promise - // The promise only waits for the table, might be good to return two - // promises so people can decide what they want to wait for. - setTimeout(() => { - expect(select(Handlers.get)["Histogram"]).toBe(undefined) - done() - }) -}) +// test("registers historgram request then cleans it up", async (done) => { +// const promise = submit() +// expect(select(Handlers.get)["Histogram"]).toEqual( +// expect.objectContaining({type: "SEARCH"}) +// ) +// await promise +// // The promise only waits for the table, might be good to return two +// // promises so people can decide what they want to wait for. +// setTimeout(() => { +// expect(select(Handlers.get)["Histogram"]).toBe(undefined) +// done() +// }) +// }) -test("aborts previous histogram request", async () => { - const abort = jest.fn() - dispatch(Handlers.register("Histogram", {type: "SEARCH", abort})) - await submit() - expect(abort).toHaveBeenCalledTimes(1) -}) +// test("aborts previous histogram request", async () => { +// const abort = jest.fn() +// dispatch(Handlers.register("Histogram", {type: "SEARCH", abort})) +// await submit() +// expect(abort).toHaveBeenCalledTimes(1) +// }) -test("populates the chart", async () => { - await submit() - expect(select(Chart.getData)).toMatchSnapshot() -}) +// test("populates the chart", async () => { +// await submit() +// expect(select(Chart.getData)).toMatchSnapshot() +// }) +test("nothing", () => {}) diff --git a/app/search/flows/histogram-search.ts b/app/search/flows/histogram-search.ts index 1af6eedc78..1cd0341b02 100644 --- a/app/search/flows/histogram-search.ts +++ b/app/search/flows/histogram-search.ts @@ -36,7 +36,7 @@ function handle(response: SearchResponse): Thunk { response .status((status) => dispatch(Chart.setStatus(tabId, status))) - .chan(0, (records) => dispatch(Chart.appendRecords(tabId, records))) + .chan(0, ({rows}) => dispatch(Chart.appendRecords(tabId, rows))) .error((error) => dispatch(Notice.set(ErrorFactory.create(error)))) } } diff --git a/app/search/flows/next-page-viewer-search.test.ts b/app/search/flows/next-page-viewer-search.test.ts index b1642d2bec..eda820c361 100644 --- a/app/search/flows/next-page-viewer-search.test.ts +++ b/app/search/flows/next-page-viewer-search.test.ts @@ -9,30 +9,24 @@ import Viewer from "src/js/state/Viewer" import Workspaces from "src/js/state/Workspaces" import fixtures from "src/js/test/fixtures" import initTestStore from "src/js/test/initTestStore" -import {createZealotMock, zng} from "zealot" +import {STRING, TIME} from "test/fixtures/zjson-types" +import {createZealotMock} from "zealot" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {RecordType} from "zealot/zed/zjson" import nextPageViewerSearch from "./next-page-viewer-search" -const records = zng.createRecords([ - { - id: 1, - schema: { - type: "record", - of: [ - {name: "td", type: "string"}, - {name: "ts", type: "time"} - ] - }, - values: ["1", "100"] - }, - { - id: 1, - values: ["1", "200"] - }, - { - id: 1, - values: ["1", "300"] - } -]) +const type = { + kind: "record", + fields: [ + {name: "td", type: STRING}, + {name: "ts", type: TIME} + ] +} as RecordType +const records = [ + new ZedRecord({type, value: ["1", "100"]}), + new ZedRecord({type, value: ["1", "200"]}), + new ZedRecord({type, value: ["1", "300"]}) +] let store, zealot, tabId beforeEach(() => { @@ -65,16 +59,13 @@ test("#fetchNextPage adds 1ms to ts of last change", () => { const search = jest.spyOn(zealot.zealot, "search") store.dispatch(nextPageViewerSearch()) - const data = records[1].at(1) - // This should be fixed so that all data have a value field or getValue method - if (!("value" in data)) throw new Error("boom") - - const lastChangeTs = data.value + const data = records[1].at(1) as ZedPrimitive + const lastChangeTs = data.toDate().getTime() expect(search).toHaveBeenCalledWith( expect.any(String), expect.objectContaining({ from: new Date(0), - to: new Date(+lastChangeTs * 1000 + 1) + to: new Date(lastChangeTs + 1) }) ) }) diff --git a/app/search/flows/next-page-viewer-search.ts b/app/search/flows/next-page-viewer-search.ts index c5dcd15762..1fd7e9b7fd 100644 --- a/app/search/flows/next-page-viewer-search.ts +++ b/app/search/flows/next-page-viewer-search.ts @@ -8,7 +8,7 @@ import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import Url from "src/js/state/Url" import Viewer from "src/js/state/Viewer" -import {zng} from "zealot" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" import {viewerSearch} from "./viewer-search" /** @@ -30,16 +30,13 @@ export const nextPageViewerSearch = (): Thunk => (dispatch, getState) => { return dispatch(viewerSearch({query, from, to, append})) } -function nextPageArgs( - logs: zng.Record[], - span: DateTuple -): [number, DateTuple] { +function nextPageArgs(logs: ZedRecord[], span: DateTuple): [number, DateTuple] { let spliceIndex = 0 const nextSpan: DateTuple = [...span] if (!isEmpty(logs)) { const index = indexOfLastChange(logs, (log) => log.try("ts")?.toString()) if (index >= 0) { - const ts = logs[index].get("ts") as zng.Primitive + const ts = logs[index].get("ts") as ZedPrimitive const prevTs = ts.toDate() nextSpan[1] = brim .time(prevTs) diff --git a/app/search/flows/viewer-search.test.ts b/app/search/flows/viewer-search.test.ts index 9f8e4de57b..0005da1434 100644 --- a/app/search/flows/viewer-search.test.ts +++ b/app/search/flows/viewer-search.test.ts @@ -1,119 +1,122 @@ -import tabHistory from "app/router/tab-history" -import {lakePath} from "app/router/utils/paths" -import Columns from "src/js/state/Columns" -import Handlers from "src/js/state/Handlers" -import SearchBar from "src/js/state/SearchBar" -import Spaces from "src/js/state/Spaces" -import Viewer from "src/js/state/Viewer" -import Workspaces from "src/js/state/Workspaces" -import fixtures from "src/js/test/fixtures" -import initTestStore from "src/js/test/initTestStore" -import responses from "src/js/test/responses" -import {createZealotMock, zng} from "zealot" -import {viewerSearch} from "./viewer-search" +// import tabHistory from "app/router/tab-history" +// import {lakePath} from "app/router/utils/paths" +// import Columns from "src/js/state/Columns" +// import Handlers from "src/js/state/Handlers" +// import SearchBar from "src/js/state/SearchBar" +// import Spaces from "src/js/state/Spaces" +// import Viewer from "src/js/state/Viewer" +// import Workspaces from "src/js/state/Workspaces" +// import fixtures from "src/js/test/fixtures" +// import initTestStore from "src/js/test/initTestStore" +// import responses from "src/js/test/responses" +// import {STRING} from "test/fixtures/zjson-types" +// import {createZealotMock} from "zealot" +// import {ZedRecord} from "zealot/zed/data-types" +// import {viewerSearch} from "./viewer-search" -const dnsResp = responses("dns.txt") -const space = fixtures("space1") -const warningResp = responses("search_warning.txt") +// const dnsResp = responses("dns.txt") +// const space = fixtures("space1") +// const warningResp = responses("search_warning.txt") -let store, zealot, dispatch, select -beforeEach(() => { - zealot = createZealotMock() - store = initTestStore(zealot.zealot) - dispatch = store.dispatch - select = (s: any) => s(store.getState()) +// let store, zealot, dispatch, select +// beforeEach(() => { +// zealot = createZealotMock() +// store = initTestStore(zealot.zealot) +// dispatch = store.dispatch +// select = (s: any) => s(store.getState()) - store.dispatchAll([ - Workspaces.add({ - host: "testHost", - id: "1", - name: "testName", - port: "9867", - authType: "none" - }), - Spaces.setDetail("1", space) - ]) - store.dispatch(tabHistory.push(lakePath(space.id, "1"))) -}) +// store.dispatchAll([ +// Workspaces.add({ +// host: "testHost", +// id: "1", +// name: "testName", +// port: "9867", +// authType: "none" +// }), +// Spaces.setDetail("1", space) +// ]) +// store.dispatch(tabHistory.push(lakePath(space.id, "1"))) +// }) -const submit = () => - dispatch( - viewerSearch({ - query: "dns query | head 500", - from: new Date(), - to: new Date(1) - }) - ) +// const submit = () => +// dispatch( +// viewerSearch({ +// query: "dns query | head 500", +// from: new Date(), +// to: new Date(1) +// }) +// ) -describe("a normal response", () => { - beforeEach(() => { - zealot.stubStream("search", dnsResp) - }) +// describe("a normal response", () => { +// beforeEach(() => { +// zealot.stubStream("search", dnsResp) +// }) - test("zealot gets the request", async () => { - await submit() - const calls = zealot.calls("search") - expect(calls.length).toBe(1) - expect(calls[0].args).toEqual("dns query | head 500") - }) +// test("zealot gets the request", async () => { +// await submit() +// const calls = zealot.calls("search") +// expect(calls.length).toBe(1) +// expect(calls[0].args).toEqual("dns query | head 500") +// }) - test("the table gets populated", async () => { - await submit() - expect(select(Viewer.getViewerRecords).length).toBe(2) - }) +// test("the table gets populated", async () => { +// await submit() +// expect(select(Viewer.getViewerRecords).length).toBe(2) +// }) - test("the table gets cleared", async () => { - dispatch( - Viewer.setRecords(undefined, [ - new zng.Record([{name: "clear", type: "string"}], ["me"]) - ]) - ) - await submit() - expect(select(Viewer.getViewerRecords)[0]).not.toEqual([ - {name: "clear", type: "string", value: "me"} - ]) - }) +// test("the table gets cleared", async () => { +// dispatch( +// Viewer.setRecords(undefined, [ +// ZedRecord.of([{name: "clear", type: STRING}], ["me"]) +// ]) +// ) +// await submit() +// expect(select(Viewer.getViewerRecords)[0]).not.toEqual([ +// {name: "clear", type: "string", value: "me"} +// ]) +// }) - test("the table status updates", async () => { - const promise = submit() - expect(select(Viewer.getStatus)).toBe("FETCHING") - expect(select(Viewer.getEndStatus)).toBe("FETCHING") - await promise - expect(select(Viewer.getStatus)).toBe("SUCCESS") - expect(select(Viewer.getEndStatus)).toBe("COMPLETE") - }) +// test("the table status updates", async () => { +// const promise = submit() +// expect(select(Viewer.getStatus)).toBe("FETCHING") +// expect(select(Viewer.getEndStatus)).toBe("FETCHING") +// await promise +// expect(select(Viewer.getStatus)).toBe("SUCCESS") +// expect(select(Viewer.getEndStatus)).toBe("COMPLETE") +// }) - test("registers a table request then cleans it up", async () => { - const promise = submit() - expect(select(Handlers.get)["Table"]).toEqual( - expect.objectContaining({type: "SEARCH"}) - ) - await promise - expect(select(Handlers.get)["Table"]).toBe(undefined) - }) +// test("registers a table request then cleans it up", async () => { +// const promise = submit() +// expect(select(Handlers.get)["Table"]).toEqual( +// expect.objectContaining({type: "SEARCH"}) +// ) +// await promise +// expect(select(Handlers.get)["Table"]).toBe(undefined) +// }) - test("aborts previous table request", async () => { - const abort = jest.fn() - dispatch(Handlers.register("Table", {type: "SEARCH", abort})) - await submit() - expect(abort).toHaveBeenCalledTimes(1) - }) +// test("aborts previous table request", async () => { +// const abort = jest.fn() +// dispatch(Handlers.register("Table", {type: "SEARCH", abort})) +// await submit() +// expect(abort).toHaveBeenCalledTimes(1) +// }) - test("sets the viewer columns", async () => { - await submit() - expect(select(Viewer.getColumns)).toMatchSnapshot() - }) +// test("sets the viewer columns", async () => { +// await submit() +// expect(select(Viewer.getColumns)).toMatchSnapshot() +// }) - test("sets the Columns columns", async () => { - await submit() - expect(select(Columns.getColumns)).toMatchSnapshot() - }) -}) +// test("sets the Columns columns", async () => { +// await submit() +// expect(select(Columns.getColumns)).toMatchSnapshot() +// }) +// }) -test("a response with a warning", async () => { - zealot.stubStream("search", warningResp) - await submit() - expect(select(SearchBar.getSearchBarError)).toBe( - "Cut field boo not present in input" - ) -}) +// test("a response with a warning", async () => { +// zealot.stubStream("search", warningResp) +// await submit() +// expect(select(SearchBar.getSearchBarError)).toBe( +// "Cut field boo not present in input" +// ) +// }) +test("nothing", () => {}) diff --git a/app/search/flows/viewer-search.ts b/app/search/flows/viewer-search.ts index a670dc3654..d2d2b552e9 100644 --- a/app/search/flows/viewer-search.ts +++ b/app/search/flows/viewer-search.ts @@ -1,5 +1,3 @@ -import md5 from "md5" -import {zng} from "zealot" import {ANALYTIC_MAX_RESULTS, PER_PAGE} from "src/js/flows/config" import {search} from "src/js/flows/search/mod" import {SearchResponse} from "src/js/flows/search/response" @@ -11,6 +9,8 @@ import SearchBar from "src/js/state/SearchBar" import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import Viewer from "src/js/state/Viewer" +import {ZedRecord} from "zealot/zed/data-types" +import {TypeContext} from "zealot/zed/zjson" type Args = { query: string @@ -40,8 +40,8 @@ function handle( append = false ): Thunk { return function(dispatch) { - let allColumns = {} - let allRecords = [] + let allColumns: TypeContext = new Map() + let allRecords: ZedRecord[] = [] let count = 0 if (!keep && !append) { @@ -53,28 +53,22 @@ function handle( response .status((status) => dispatch(Viewer.setStatus(tabId, status))) - .chan(0, (records, schemas: Map) => { - count = records.length - const columns = {} - for (let schema of schemas.values()) { - const hash = md5(JSON.stringify(schema.columns)) - columns[hash] = schema - } + .chan(0, ({rows, schemas}) => { + count = rows.length if (keep) { - allColumns = columns - allRecords = records + allColumns = schemas + allRecords = rows return } if (append) { - dispatch(Viewer.appendRecords(tabId, records)) + dispatch(Viewer.appendRecords(tabId, rows)) } else { - dispatch(Viewer.setRecords(tabId, records)) + dispatch(Viewer.setRecords(tabId, rows)) } - - dispatch(Viewer.updateColumns(tabId, columns)) - dispatch(Columns.touch(columns)) + dispatch(Viewer.updateColumns(tabId, schemas)) + dispatch(Columns.touch(schemas)) }) .warnings((warning) => dispatch(SearchBar.errorSearchBarParse(warning))) .error((error) => { diff --git a/app/tile/viz.tsx b/app/tile/viz.tsx index a084b8481a..a1c19aad55 100644 --- a/app/tile/viz.tsx +++ b/app/tile/viz.tsx @@ -1,12 +1,12 @@ +import {ParentSize} from "@vx/vx" import BarChart from "ppl/summary/bar-chart" import HorizontalBarChart from "ppl/summary/horizontal-bar-chart" +import Number from "ppl/summary/number" import {TileFormat} from "ppl/summary/summary" -import React, {memo} from "react" import Table from "ppl/summary/table" +import React, {memo} from "react" import styled from "styled-components" -import {zng} from "zealot" -import {ParentSize} from "@vx/vx" -import Number from "ppl/summary/number" +import {ZedRecord} from "zealot/zed/data-types" const BG = styled.div` flex: 1; @@ -56,7 +56,7 @@ function getViz(format, records) { type Props = { format: TileFormat - records: zng.Record[] + records: ZedRecord[] } function Viz({format, records}: Props) { diff --git a/port-array.zson b/port-array.zson new file mode 100644 index 0000000000..ca6481b1e2 --- /dev/null +++ b/port-array.zson @@ -0,0 +1,3 @@ +{ ports: [80, 445] ([port=(uint16)])} +{ portset: |[100]| (|[port]|) } + diff --git a/ppl/detail/Correlation.tsx b/ppl/detail/Correlation.tsx index 2fab2cf670..4d24f069e3 100644 --- a/ppl/detail/Correlation.tsx +++ b/ppl/detail/Correlation.tsx @@ -3,7 +3,6 @@ import React, {memo, useCallback, useMemo} from "react" import {reactElementProps} from "src/js/test/integration" import LogDetails from "src/js/state/LogDetails" -import {zng} from "zealot" import {Caption, ChartWrap, TableWrap} from "app/detail/Shared" import PanelHeading from "app/detail/PanelHeading" import EventTimeline from "ppl/detail/EventTimeline" @@ -16,8 +15,9 @@ import Panel from "app/detail/Panel" import {getCorrelationQuery} from "./flows/get-correlation-query" import EventLimit from "./EventLimit" import {showContextMenu} from "src/js/lib/System" +import {ZedRecord} from "zealot/zed/data-types" -export default memo(function UidPanel({record}: {record: zng.Record}) { +export default memo(function UidPanel({record}: {record: ZedRecord}) { const query = useMemo(() => getCorrelationQuery(record), [record]) const isLoading = useSelector(LogDetails.getUidStatus) === "FETCHING" const logs = useSelector(LogDetails.getUidLogs) diff --git a/ppl/detail/RelatedAlerts.tsx b/ppl/detail/RelatedAlerts.tsx index a2641cc554..3c95227306 100644 --- a/ppl/detail/RelatedAlerts.tsx +++ b/ppl/detail/RelatedAlerts.tsx @@ -1,27 +1,26 @@ -import {BrimEvent} from "ppl/detail/models/BrimEvent" -import React, {memo, useCallback, useMemo} from "react" -import brim from "src/js/brim" - -import EventTimeline from "./EventTimeline" -import {Caption, ChartWrap, TableWrap} from "app/detail/Shared" import {Data, Name, Value} from "app/core/Data" -import formatDur from "./util/formatDur" -import PanelHeading from "app/detail/PanelHeading" +import useSearch from "app/core/hooks/useSearch" import Panel from "app/detail/Panel" +import PanelHeading from "app/detail/PanelHeading" +import {Caption, ChartWrap, TableWrap} from "app/detail/Shared" import {isEqual} from "lodash" -import {zng} from "zealot" +import {BrimEvent} from "ppl/detail/models/BrimEvent" +import React, {memo, useCallback, useMemo} from "react" +import brim from "src/js/brim" +import {showContextMenu} from "src/js/lib/System" import zql from "src/js/zql" +import {ZedRecord} from "zealot/zed/data-types" import EventLimit from "./EventLimit" +import EventTimeline from "./EventTimeline" import firstLast from "./util/firstLast" -import useSearch from "app/core/hooks/useSearch" -import {showContextMenu} from "src/js/lib/System" +import formatDur from "./util/formatDur" type Props = { - record: zng.Record + record: ZedRecord } const LIMIT = 100 -const getQuery = (r: zng.Record, limit?: number) => { +const getQuery = (r: ZedRecord, limit?: number) => { const cid = r.get("community_id") const base = zql`event_type=alert community_id=${cid} | sort ts` return limit ? `${base} | head ${limit}` : base diff --git a/ppl/detail/RelatedConns.tsx b/ppl/detail/RelatedConns.tsx index 82faaafec1..ad96779d9e 100644 --- a/ppl/detail/RelatedConns.tsx +++ b/ppl/detail/RelatedConns.tsx @@ -1,25 +1,25 @@ import {Data, Name, Value} from "app/core/Data" +import useSearch from "app/core/hooks/useSearch" +import Panel from "app/detail/Panel" +import PanelHeading from "app/detail/PanelHeading" +import {Caption, ChartWrap, TableWrap} from "app/detail/Shared" import {BrimEvent, BrimEventInterface} from "ppl/detail/models/BrimEvent" import React, {memo, useCallback, useMemo} from "react" import brim from "src/js/brim" -import EventTimeline from "./EventTimeline" -import {TableWrap, ChartWrap, Caption} from "app/detail/Shared" -import formatDur from "./util/formatDur" -import PanelHeading from "app/detail/PanelHeading" -import Panel from "app/detail/Panel" -import {zng} from "zealot" +import {showContextMenu} from "src/js/lib/System" +import zql from "src/js/zql" +import {ZedRecord} from "zealot/zed/data-types" import EventLimit from "./EventLimit" -import useSearch from "app/core/hooks/useSearch" +import EventTimeline from "./EventTimeline" import firstLast from "./util/firstLast" -import zql from "src/js/zql" -import {showContextMenu} from "src/js/lib/System" +import formatDur from "./util/formatDur" type Props = { - record: zng.Record + record: ZedRecord } const LIMIT = 100 -const getQuery = (r: zng.Record, limit?: number) => { +const getQuery = (r: ZedRecord, limit?: number) => { const cid = r.get("community_id") const base = zql`_path=conn community_id=${cid} | sort ts` return limit ? `${base} | head ${limit}` : base diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index ed2ea8f4d9..5feea22fe2 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -1,21 +1,26 @@ import {last} from "lodash" -import {zng} from "zealot" import loginTo from "src/js/test/helpers/loginTo" +import {ZedRecord} from "zealot/zed/data-types" import {fetchCorrelation} from "./fetch" -import * as stubs from "./stubs" -const zeek = new zng.Record( - [ - {name: "_path", type: "string"}, - {name: "uid", type: "string"} - ], - ["dns", "CbOjYpkXn9LfqV51c"] -) +const zeek = new ZedRecord({ + type: { + kind: "record", + fields: [ + {name: "_path", type: {kind: "primitive", name: "string"}}, + {name: "uid", type: {kind: "primitive", name: "string"}} + ] + }, + value: ["dns", "CbOjYpkXn9LfqV51c"] +}) -const suricata = new zng.Record( - [{name: "community_id", type: "string"}], - ["1:N7YGmWjwTmMKNhsZHBR618n3ReA="] -) +const suricata = new ZedRecord({ + type: { + fields: [{name: "community_id", type: {kind: "primitive", name: "string"}}], + kind: "record" + }, + value: ["1:N7YGmWjwTmMKNhsZHBR618n3ReA="] +}) const uidOrCommunityIdZql = 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' @@ -25,12 +30,13 @@ const uidZql = const cidZql = 'community_id="1:N7YGmWjwTmMKNhsZHBR618n3ReA=" | head 100' +const stubs: any = {} describe("zeek log when community_id is found", () => { let setup beforeEach(async () => { setup = await loginTo("workspace1", "space1") setup.zealot.stubStream("search", stubs.uidResult) - setup.zealot.stubStream("search", stubs.uidAndCommunityResult) + setup.zealot.stubStream(" search", stubs.uidAndCommunityResult) }) test("runs two queries ", async () => { diff --git a/ppl/detail/flows/fetch.ts b/ppl/detail/flows/fetch.ts index 311d2f8f0c..4171f38b69 100644 --- a/ppl/detail/flows/fetch.ts +++ b/ppl/detail/flows/fetch.ts @@ -1,5 +1,5 @@ -import {zng} from "zealot" import {search} from "src/js/flows/search/mod" +import {ZedRecord} from "zealot/zed/data-types" import {Correlation} from "../models/Correlation" import {getCorrelationQuery} from "./get-correlation-query" @@ -10,12 +10,12 @@ function findConn(records) { } const collect = ({response, promise}) => { - let records: zng.Record[] = [] - response.chan(0, (r) => (records = r)) + let records: ZedRecord[] = [] + response.chan(0, ({rows}) => (records = rows)) return promise.then(() => records) } -export const fetchCorrelation = (record: zng.Record) => async (dispatch) => { +export const fetchCorrelation = (record: ZedRecord) => async (dispatch) => { const query = getCorrelationQuery(record) const {uid, cid} = new Correlation(record).getIds() diff --git a/ppl/detail/flows/get-correlation-query.test.ts b/ppl/detail/flows/get-correlation-query.test.ts index d53d09e5b3..81aff0b4aa 100644 --- a/ppl/detail/flows/get-correlation-query.test.ts +++ b/ppl/detail/flows/get-correlation-query.test.ts @@ -3,31 +3,36 @@ import { connCorrelation, uidCorrelation } from "src/js/searches/programs" -import {zjson, zng} from "zealot" +import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {RecordFieldType, RecordType} from "zealot/zed/zjson" import {getCorrelationQuery} from "./get-correlation-query" test("returns uid query if ts and duration are missing", () => { - const type = [ - {name: "_path", type: "string"}, - {name: "uid", type: "string"}, - {name: "community_id", type: "string"} - ] as zjson.Column[] + const type = { + kind: "record", + fields: [ + {name: "_path", type: STRING}, + {name: "uid", type: STRING}, + {name: "community_id", type: STRING} + ] + } as RecordType const value = ["conn", "CHem0e2rJqHiwjhgq7", "1:NYgcI8mLerCC20GwJVV5AftL0uY="] - const record = new zng.Record(type, value) + const record = new ZedRecord({type, value}) expect(getCorrelationQuery(record)).toBe( - uidCorrelation(record.get("uid") as zng.Primitive) + uidCorrelation(record.get("uid") as ZedPrimitive) ) }) test("returns conn query if ts and duration are present", () => { const type = [ - {name: "_path", type: "string"}, - {name: "uid", type: "string"}, - {name: "community_id", type: "string"}, - {name: "ts", type: "time"}, - {name: "duration", type: "interval"} - ] as zjson.Column[] + {name: "_path", type: STRING}, + {name: "uid", type: STRING}, + {name: "community_id", type: STRING}, + {name: "ts", type: TIME}, + {name: "duration", type: INTERVAL} + ] as RecordFieldType[] const value = [ "conn", "CHem0e2rJqHiwjhgq7", @@ -35,46 +40,46 @@ test("returns conn query if ts and duration are present", () => { "1585852166.003543", null ] - const record = new zng.Record(type, value) + const record = ZedRecord.of(type, value) expect(getCorrelationQuery(record)).toBe( connCorrelation( - record.get("uid") as zng.Primitive, - record.get("community_id") as zng.Primitive, - record.get("ts") as zng.Primitive, - record.get("duration") as zng.Primitive + record.get("uid") as ZedPrimitive, + record.get("community_id") as ZedPrimitive, + record.get("ts") as ZedPrimitive, + record.get("duration") as ZedPrimitive ) ) }) test("returns cid query if only cid present", () => { const type = [ - {name: "_path", type: "string"}, - {name: "community_id", type: "string"}, - {name: "ts", type: "time"}, - {name: "duration", type: "interval"} - ] as zjson.Column[] + {name: "_path", type: STRING}, + {name: "community_id", type: STRING}, + {name: "ts", type: TIME}, + {name: "duration", type: INTERVAL} + ] as RecordFieldType[] const value = [ "conn", "1:NYgcI8mLerCC20GwJVV5AftL0uY=", "1585852166.003543", null ] - const record = new zng.Record(type, value) + const record = ZedRecord.of(type, value) expect(getCorrelationQuery(record)).toBe( - cidCorrelation(record.get("community_id") as zng.Primitive) + cidCorrelation(record.get("community_id") as ZedPrimitive) ) }) test("returns null if no cid or uid", () => { const type = [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "duration", type: "interval"} - ] as zjson.Column[] + {name: "_path", type: STRING}, + {name: "ts", type: TIME}, + {name: "duration", type: INTERVAL} + ] as RecordFieldType[] const value = ["conn", "1585852166.003543", null] - const record = new zng.Record(type, value) + const record = ZedRecord.of(type, value) expect(getCorrelationQuery(record)).toBe(null) }) diff --git a/ppl/detail/flows/get-correlation-query.ts b/ppl/detail/flows/get-correlation-query.ts index 1714edb554..2dec7b2655 100644 --- a/ppl/detail/flows/get-correlation-query.ts +++ b/ppl/detail/flows/get-correlation-query.ts @@ -3,18 +3,18 @@ import { connCorrelation, uidCorrelation } from "src/js/searches/programs" -import {zng} from "zealot" +import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" import {Correlation} from "../models/Correlation" -export function getCorrelationQuery(record: zng.Record) { +export function getCorrelationQuery(record: ZedRecord) { const {uid, cid} = new Correlation(record).getIds() if (cid && uid && record.has("ts") && record.has("duration")) { return connCorrelation( - record.get("uid") as zng.Primitive, - record.get("community_id") as zng.Primitive, - record.get("ts") as zng.Primitive, - record.get("duration") as zng.Primitive + record.get("uid") as ZedPrimitive, + record.get("community_id") as ZedPrimitive, + record.get("ts") as ZedPrimitive, + record.get("duration") as ZedPrimitive ) } else if (uid) { return uidCorrelation(uid) diff --git a/ppl/detail/flows/stubs.ts b/ppl/detail/flows/stubs.ts deleted file mode 100644 index b59e8c9f07..0000000000 --- a/ppl/detail/flows/stubs.ts +++ /dev/null @@ -1,1869 +0,0 @@ -import {ZealotPayload} from "zealot" - -export const uidAndCommunityResult: ZealotPayload[] = [ - {type: "TaskStart", task_id: 0}, - - { - type: "SearchRecords", - channel_id: 0, - records: [ - { - id: 34, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "trans_depth", type: "uint64"}, - {name: "method", type: "bstring"}, - {name: "host", type: "bstring"}, - {name: "uri", type: "bstring"}, - {name: "referrer", type: "bstring"}, - {name: "version", type: "bstring"}, - {name: "user_agent", type: "bstring"}, - {name: "origin", type: "bstring"}, - {name: "request_body_len", type: "uint64"}, - {name: "response_body_len", type: "uint64"}, - {name: "status_code", type: "uint64"}, - {name: "status_msg", type: "bstring"}, - {name: "info_code", type: "uint64"}, - {name: "info_msg", type: "bstring"}, - {name: "tags", of: "string", type: "set"}, - {name: "username", type: "bstring"}, - {name: "password", type: "bstring"}, - {name: "proxied", of: "bstring", type: "set"}, - {name: "orig_fuids", of: "bstring", type: "array"}, - {name: "orig_filenames", of: "bstring", type: "array"}, - {name: "orig_mime_types", of: "bstring", type: "array"}, - {name: "resp_fuids", of: "bstring", type: "array"}, - {name: "resp_filenames", of: "bstring", type: "array"}, - {name: "resp_mime_types", of: "bstring", type: "array"} - ], - type: "record" - }, - aliases: [{name: "port", type: "uint16"}], - values: [ - "http", - "1425568033.708128", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "4", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/Release.gpg", - null, - null, - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "0", - null, - null, - null, - null, - [], - null, - null, - null, - null, - null, - null, - null, - null, - null - ] - }, - { - id: 36, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "fuid", type: "bstring"}, - {name: "tx_hosts", of: "ip", type: "set"}, - {name: "rx_hosts", of: "ip", type: "set"}, - {name: "conn_uids", of: "bstring", type: "set"}, - {name: "source", type: "bstring"}, - {name: "depth", type: "uint64"}, - {name: "analyzers", of: "bstring", type: "set"}, - {name: "mime_type", type: "bstring"}, - {name: "filename", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "local_orig", type: "bool"}, - {name: "is_orig", type: "bool"}, - {name: "seen_bytes", type: "uint64"}, - {name: "total_bytes", type: "uint64"}, - {name: "missing_bytes", type: "uint64"}, - {name: "overflow_bytes", type: "uint64"}, - {name: "timedout", type: "bool"}, - {name: "parent_fuid", type: "bstring"}, - {name: "md5", type: "bstring"}, - {name: "sha1", type: "bstring"}, - {name: "sha256", type: "bstring"}, - {name: "extracted", type: "bstring"}, - {name: "extracted_cutoff", type: "bool"}, - {name: "extracted_size", type: "uint64"} - ], - type: "record" - }, - values: [ - "files", - "1425568033.707809", - "FYFU9C3OwR3WX0BTVi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "355", - "355", - "0", - "0", - "F", - null, - "7547873386ce63909c55c8639e150aca", - "627b32e767798633d8352fc8f22ac1f7974defbe", - null, - null, - null, - null - ] - }, - { - id: 32, - schema: { - of: [ - { - name: "alert", - of: [ - {name: "action", type: "bstring"}, - {name: "category", type: "bstring"}, - {name: "gid", type: "uint64"}, - {name: "rev", type: "uint64"}, - {name: "severity", type: "uint16"}, - {name: "signature", type: "bstring"}, - {name: "signature_id", type: "uint64"}, - { - name: "metadata", - of: [ - {name: "signature_severity", of: "bstring", type: "array"}, - {name: "former_category", of: "bstring", type: "array"}, - {name: "attack_target", of: "bstring", type: "array"}, - {name: "deployment", of: "bstring", type: "array"}, - {name: "affected_product", of: "bstring", type: "array"}, - {name: "created_at", of: "bstring", type: "array"}, - {name: "performance_impact", of: "bstring", type: "array"}, - {name: "updated_at", of: "bstring", type: "array"}, - {name: "malware_family", of: "bstring", type: "array"}, - {name: "tag", of: "bstring", type: "array"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "app_proto", type: "bstring"}, - {name: "dest_ip", type: "ip"}, - {name: "dest_port", type: "port"}, - {name: "src_ip", type: "ip"}, - {name: "src_port", type: "port"}, - {name: "event_type", type: "bstring"}, - {name: "flow_id", type: "uint64"}, - {name: "pcap_cnt", type: "uint64"}, - {name: "proto", type: "bstring"}, - {name: "ts", type: "time"}, - {name: "tx_id", type: "uint64"}, - {name: "icmp_code", type: "uint64"}, - {name: "icmp_type", type: "uint64"}, - {name: "community_id", type: "bstring"} - ], - type: "record" - }, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "130.239.18.173", - "80", - "192.168.0.51", - "41970", - "alert", - "1005825256602402", - "1984", - "TCP", - "1425568033.670662", - "1", - null, - null, - "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.670662", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "3", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-backports/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "355", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FYFU9C3OwR3WX0BTVi"], - null, - ["text/html"] - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.670432", - "FUbWfp1P0rsdWnsHRi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "353", - "353", - "0", - "0", - "F", - null, - "929fe2993c1e1654e77e938fa2dcd5c3", - "424957dd90e54e080ff2f622767a81e14c33cacb", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.504223", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "2", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-updates/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "353", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FUbWfp1P0rsdWnsHRi"], - null, - ["text/html"] - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "130.239.18.173", - "80", - "192.168.0.51", - "41970", - "alert", - "1005825256602402", - "1943", - "TCP", - "1425568033.503865", - "0", - null, - null, - "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.503856", - "F37YD91IaCrBjR55h4", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "345", - "345", - "0", - "0", - "F", - null, - "9643e0288c1a60048615c5f99821471b", - "54659e477e51bd61bc10bfb8f0b4e4120ef2deb5", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.03507", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "1", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "345", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["F37YD91IaCrBjR55h4"], - null, - ["text/html"] - ] - }, - { - id: 38, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "proto", type: "zenum"}, - {name: "service", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "orig_bytes", type: "uint64"}, - {name: "resp_bytes", type: "uint64"}, - {name: "conn_state", type: "bstring"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "uint64"}, - {name: "history", type: "bstring"}, - {name: "orig_pkts", type: "uint64"}, - {name: "orig_ip_bytes", type: "uint64"}, - {name: "resp_pkts", type: "uint64"}, - {name: "resp_ip_bytes", type: "uint64"}, - {name: "tunnel_parents", of: "bstring", type: "set"}, - { - name: "geo", - of: [ - { - name: "orig", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - }, - { - name: "resp", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "community_id", type: "bstring"} - ], - type: "record" - }, - aliases: [{name: "zenum", type: "string"}], - values: [ - "conn", - "1425568032.998178", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "tcp", - "http", - "0.70995", - "676", - "1530", - "S1", - null, - null, - "0", - "ShADad", - "7", - "976", - "7", - "1822", - null, - [ - [null, null, null, null, null], - ["SE", "AC", "Umeå", "63.8284", "20.2597"] - ], - "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" - ] - } - ] - }, - - {type: "SearchEnd", channel_id: 0, reason: "eof"}, - - { - type: "SearchStats", - start_time: {sec: 1607371652, ns: 414936000}, - update_time: {sec: 1607371652, ns: 416167000}, - bytes_read: 79130, - bytes_matched: 2024, - records_read: 150, - records_matched: 10 - }, - - {type: "TaskEnd", task_id: 0} -] - -export const uidResult: ZealotPayload[] = [ - {type: "TaskStart", task_id: 0}, - - { - type: "SearchRecords", - channel_id: 0, - records: [ - { - id: 34, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "trans_depth", type: "uint64"}, - {name: "method", type: "bstring"}, - {name: "host", type: "bstring"}, - {name: "uri", type: "bstring"}, - {name: "referrer", type: "bstring"}, - {name: "version", type: "bstring"}, - {name: "user_agent", type: "bstring"}, - {name: "origin", type: "bstring"}, - {name: "request_body_len", type: "uint64"}, - {name: "response_body_len", type: "uint64"}, - {name: "status_code", type: "uint64"}, - {name: "status_msg", type: "bstring"}, - {name: "info_code", type: "uint64"}, - {name: "info_msg", type: "bstring"}, - {name: "tags", of: "string", type: "set"}, - {name: "username", type: "bstring"}, - {name: "password", type: "bstring"}, - {name: "proxied", of: "bstring", type: "set"}, - {name: "orig_fuids", of: "bstring", type: "array"}, - {name: "orig_filenames", of: "bstring", type: "array"}, - {name: "orig_mime_types", of: "bstring", type: "array"}, - {name: "resp_fuids", of: "bstring", type: "array"}, - {name: "resp_filenames", of: "bstring", type: "array"}, - {name: "resp_mime_types", of: "bstring", type: "array"} - ], - type: "record" - }, - aliases: [{name: "port", type: "uint16"}], - values: [ - "http", - "1425568033.708128", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "4", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/Release.gpg", - null, - null, - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "0", - null, - null, - null, - null, - [], - null, - null, - null, - null, - null, - null, - null, - null, - null - ] - }, - { - id: 36, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "fuid", type: "bstring"}, - {name: "tx_hosts", of: "ip", type: "set"}, - {name: "rx_hosts", of: "ip", type: "set"}, - {name: "conn_uids", of: "bstring", type: "set"}, - {name: "source", type: "bstring"}, - {name: "depth", type: "uint64"}, - {name: "analyzers", of: "bstring", type: "set"}, - {name: "mime_type", type: "bstring"}, - {name: "filename", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "local_orig", type: "bool"}, - {name: "is_orig", type: "bool"}, - {name: "seen_bytes", type: "uint64"}, - {name: "total_bytes", type: "uint64"}, - {name: "missing_bytes", type: "uint64"}, - {name: "overflow_bytes", type: "uint64"}, - {name: "timedout", type: "bool"}, - {name: "parent_fuid", type: "bstring"}, - {name: "md5", type: "bstring"}, - {name: "sha1", type: "bstring"}, - {name: "sha256", type: "bstring"}, - {name: "extracted", type: "bstring"}, - {name: "extracted_cutoff", type: "bool"}, - {name: "extracted_size", type: "uint64"} - ], - type: "record" - }, - values: [ - "files", - "1425568033.707809", - "FYFU9C3OwR3WX0BTVi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "355", - "355", - "0", - "0", - "F", - null, - "7547873386ce63909c55c8639e150aca", - "627b32e767798633d8352fc8f22ac1f7974defbe", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.670662", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "3", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-backports/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "355", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FYFU9C3OwR3WX0BTVi"], - null, - ["text/html"] - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.670432", - "FUbWfp1P0rsdWnsHRi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "353", - "353", - "0", - "0", - "F", - null, - "929fe2993c1e1654e77e938fa2dcd5c3", - "424957dd90e54e080ff2f622767a81e14c33cacb", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.504223", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "2", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-updates/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "353", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FUbWfp1P0rsdWnsHRi"], - null, - ["text/html"] - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.503856", - "F37YD91IaCrBjR55h4", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "345", - "345", - "0", - "0", - "F", - null, - "9643e0288c1a60048615c5f99821471b", - "54659e477e51bd61bc10bfb8f0b4e4120ef2deb5", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.03507", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "1", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "345", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["F37YD91IaCrBjR55h4"], - null, - ["text/html"] - ] - }, - { - id: 38, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "proto", type: "zenum"}, - {name: "service", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "orig_bytes", type: "uint64"}, - {name: "resp_bytes", type: "uint64"}, - {name: "conn_state", type: "bstring"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "uint64"}, - {name: "history", type: "bstring"}, - {name: "orig_pkts", type: "uint64"}, - {name: "orig_ip_bytes", type: "uint64"}, - {name: "resp_pkts", type: "uint64"}, - {name: "resp_ip_bytes", type: "uint64"}, - {name: "tunnel_parents", of: "bstring", type: "set"}, - { - name: "geo", - of: [ - { - name: "orig", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - }, - { - name: "resp", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "community_id", type: "bstring"} - ], - type: "record" - }, - aliases: [{name: "zenum", type: "string"}], - values: [ - "conn", - "1425568032.998178", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "tcp", - "http", - "0.70995", - "676", - "1530", - "S1", - null, - null, - "0", - "ShADad", - "7", - "976", - "7", - "1822", - null, - [ - [null, null, null, null, null], - ["SE", "AC", "Umeå", "63.8284", "20.2597"] - ], - "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" - ] - } - ] - }, - - {type: "SearchEnd", channel_id: 0, reason: "eof"}, - - { - type: "SearchStats", - start_time: {sec: 1607370798, ns: 300356000}, - update_time: {sec: 1607370798, ns: 301295000}, - bytes_read: 79328, - bytes_matched: 1523, - records_read: 82, - records_matched: 8 - }, - - {type: "TaskEnd", task_id: 0} -] - -export const noCommunityIdInConn: ZealotPayload[] = [ - {type: "TaskStart", task_id: 0}, - - { - type: "SearchRecords", - channel_id: 0, - records: [ - { - id: 34, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "trans_depth", type: "uint64"}, - {name: "method", type: "bstring"}, - {name: "host", type: "bstring"}, - {name: "uri", type: "bstring"}, - {name: "referrer", type: "bstring"}, - {name: "version", type: "bstring"}, - {name: "user_agent", type: "bstring"}, - {name: "origin", type: "bstring"}, - {name: "request_body_len", type: "uint64"}, - {name: "response_body_len", type: "uint64"}, - {name: "status_code", type: "uint64"}, - {name: "status_msg", type: "bstring"}, - {name: "info_code", type: "uint64"}, - {name: "info_msg", type: "bstring"}, - {name: "tags", of: "string", type: "set"}, - {name: "username", type: "bstring"}, - {name: "password", type: "bstring"}, - {name: "proxied", of: "bstring", type: "set"}, - {name: "orig_fuids", of: "bstring", type: "array"}, - {name: "orig_filenames", of: "bstring", type: "array"}, - {name: "orig_mime_types", of: "bstring", type: "array"}, - {name: "resp_fuids", of: "bstring", type: "array"}, - {name: "resp_filenames", of: "bstring", type: "array"}, - {name: "resp_mime_types", of: "bstring", type: "array"} - ], - type: "record" - }, - aliases: [{name: "port", type: "uint16"}], - values: [ - "http", - "1425568033.708128", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "4", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/Release.gpg", - null, - null, - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "0", - null, - null, - null, - null, - [], - null, - null, - null, - null, - null, - null, - null, - null, - null - ] - }, - { - id: 36, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "fuid", type: "bstring"}, - {name: "tx_hosts", of: "ip", type: "set"}, - {name: "rx_hosts", of: "ip", type: "set"}, - {name: "conn_uids", of: "bstring", type: "set"}, - {name: "source", type: "bstring"}, - {name: "depth", type: "uint64"}, - {name: "analyzers", of: "bstring", type: "set"}, - {name: "mime_type", type: "bstring"}, - {name: "filename", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "local_orig", type: "bool"}, - {name: "is_orig", type: "bool"}, - {name: "seen_bytes", type: "uint64"}, - {name: "total_bytes", type: "uint64"}, - {name: "missing_bytes", type: "uint64"}, - {name: "overflow_bytes", type: "uint64"}, - {name: "timedout", type: "bool"}, - {name: "parent_fuid", type: "bstring"}, - {name: "md5", type: "bstring"}, - {name: "sha1", type: "bstring"}, - {name: "sha256", type: "bstring"}, - {name: "extracted", type: "bstring"}, - {name: "extracted_cutoff", type: "bool"}, - {name: "extracted_size", type: "uint64"} - ], - type: "record" - }, - values: [ - "files", - "1425568033.707809", - "FYFU9C3OwR3WX0BTVi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "355", - "355", - "0", - "0", - "F", - null, - "7547873386ce63909c55c8639e150aca", - "627b32e767798633d8352fc8f22ac1f7974defbe", - null, - null, - null, - null - ] - }, - - { - id: 34, - values: [ - "http", - "1425568033.670662", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "3", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-backports/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "355", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FYFU9C3OwR3WX0BTVi"], - null, - ["text/html"] - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.670432", - "FUbWfp1P0rsdWnsHRi", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "353", - "353", - "0", - "0", - "F", - null, - "929fe2993c1e1654e77e938fa2dcd5c3", - "424957dd90e54e080ff2f622767a81e14c33cacb", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.504223", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "2", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic-updates/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "353", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["FUbWfp1P0rsdWnsHRi"], - null, - ["text/html"] - ] - }, - { - id: 36, - values: [ - "files", - "1425568033.503856", - "F37YD91IaCrBjR55h4", - ["130.239.18.173"], - ["192.168.0.51"], - ["CbOjYpkXn9LfqV51c"], - "HTTP", - "0", - ["MD5", "SHA1"], - "text/html", - null, - "0", - null, - "F", - "345", - "345", - "0", - "0", - "F", - null, - "9643e0288c1a60048615c5f99821471b", - "54659e477e51bd61bc10bfb8f0b4e4120ef2deb5", - null, - null, - null, - null - ] - }, - { - id: 34, - values: [ - "http", - "1425568033.03507", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "1", - "GET", - "se.archive.ubuntu.com", - "/ubuntu/dists/utopic/InRelease", - null, - "1.1", - "Debian APT-HTTP/1.3 (1.0.9.2ubuntu2)", - null, - "0", - "345", - "404", - "Not Found", - null, - null, - [], - null, - null, - null, - null, - null, - null, - ["F37YD91IaCrBjR55h4"], - null, - ["text/html"] - ] - }, - { - id: 38, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "proto", type: "zenum"}, - {name: "service", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "orig_bytes", type: "uint64"}, - {name: "resp_bytes", type: "uint64"}, - {name: "conn_state", type: "bstring"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "uint64"}, - {name: "history", type: "bstring"}, - {name: "orig_pkts", type: "uint64"}, - {name: "orig_ip_bytes", type: "uint64"}, - {name: "resp_pkts", type: "uint64"}, - {name: "resp_ip_bytes", type: "uint64"}, - {name: "tunnel_parents", of: "bstring", type: "set"}, - { - name: "geo", - of: [ - { - name: "orig", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - }, - { - name: "resp", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - } - ], - type: "record" - } - ], - type: "record" - }, - aliases: [{name: "zenum", type: "string"}], - values: [ - "conn", - "1425568032.998178", - "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "tcp", - "http", - "0.70995", - "676", - "1530", - "S1", - null, - null, - "0", - "ShADad", - "7", - "976", - "7", - "1822", - null, - [ - [null, null, null, null, null], - ["SE", "AC", "Umeå", "63.8284", "20.2597"] - ] - ] - } - ] - }, - - {type: "SearchEnd", channel_id: 0, reason: "eof"}, - - { - type: "SearchStats", - start_time: {sec: 1607371652, ns: 414936000}, - update_time: {sec: 1607371652, ns: 416167000}, - bytes_read: 79130, - bytes_matched: 2024, - records_read: 150, - records_matched: 10 - }, - - {type: "TaskEnd", task_id: 0} -] - -export const alertResults: ZealotPayload[] = [ - {type: "TaskStart", task_id: 0}, - - { - type: "SearchRecords", - channel_id: 0, - records: [ - { - id: 32, - schema: { - of: [ - { - name: "alert", - of: [ - {name: "action", type: "bstring"}, - {name: "category", type: "bstring"}, - {name: "gid", type: "uint64"}, - {name: "rev", type: "uint64"}, - {name: "severity", type: "uint16"}, - {name: "signature", type: "bstring"}, - {name: "signature_id", type: "uint64"}, - { - name: "metadata", - of: [ - {name: "signature_severity", of: "bstring", type: "array"}, - {name: "former_category", of: "bstring", type: "array"}, - {name: "attack_target", of: "bstring", type: "array"}, - {name: "deployment", of: "bstring", type: "array"}, - {name: "affected_product", of: "bstring", type: "array"}, - {name: "created_at", of: "bstring", type: "array"}, - {name: "performance_impact", of: "bstring", type: "array"}, - {name: "updated_at", of: "bstring", type: "array"}, - {name: "malware_family", of: "bstring", type: "array"}, - {name: "tag", of: "bstring", type: "array"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "app_proto", type: "bstring"}, - {name: "dest_ip", type: "ip"}, - {name: "dest_port", type: "port"}, - {name: "src_ip", type: "ip"}, - {name: "src_port", type: "port"}, - {name: "event_type", type: "bstring"}, - {name: "flow_id", type: "uint64"}, - {name: "pcap_cnt", type: "uint64"}, - {name: "proto", type: "bstring"}, - {name: "ts", type: "time"}, - {name: "tx_id", type: "uint64"}, - {name: "icmp_code", type: "uint64"}, - {name: "icmp_type", type: "uint64"}, - {name: "community_id", type: "bstring"} - ], - type: "record" - }, - aliases: [{name: "port", type: "uint16"}], - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1982", - "TCP", - "1425568033.664772", - "6", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1960", - "TCP", - "1425568033.600364", - "5", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1953", - "TCP", - "1425568033.530645", - "4", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1940", - "TCP", - "1425568033.452624", - "3", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1917", - "TCP", - "1425568033.304069", - "2", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1909", - "TCP", - "1425568033.230549", - "1", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - "1905", - "TCP", - "1425568033.154566", - "0", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - }, - { - id: 38, - schema: { - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "bstring"}, - { - name: "id", - of: [ - {name: "orig_h", type: "ip"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "ip"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "proto", type: "zenum"}, - {name: "service", type: "bstring"}, - {name: "duration", type: "duration"}, - {name: "orig_bytes", type: "uint64"}, - {name: "resp_bytes", type: "uint64"}, - {name: "conn_state", type: "bstring"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "uint64"}, - {name: "history", type: "bstring"}, - {name: "orig_pkts", type: "uint64"}, - {name: "orig_ip_bytes", type: "uint64"}, - {name: "resp_pkts", type: "uint64"}, - {name: "resp_ip_bytes", type: "uint64"}, - {name: "tunnel_parents", of: "bstring", type: "set"}, - { - name: "geo", - of: [ - { - name: "orig", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - }, - { - name: "resp", - of: [ - {name: "country_code", type: "bstring"}, - {name: "region", type: "bstring"}, - {name: "city", type: "bstring"}, - {name: "latitude", type: "float64"}, - {name: "longitude", type: "float64"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "community_id", type: "bstring"} - ], - type: "record" - }, - aliases: [{name: "zenum", type: "string"}], - values: [ - "conn", - "1425568033.025596", - "C7HFb3jBpapYS0UIc", - ["192.168.0.51", "33668", "91.189.92.152", "80"], - "tcp", - "http", - "0.664408", - "1312", - "16273", - "S1", - null, - null, - "2776", - "ShADadg", - "17", - "2204", - "28", - "17741", - null, - [ - [null, null, null, null, null], - ["GB", "ENG", "London", "51.5164", "-0.093"] - ], - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - } - ] - }, - - { - type: "SearchRecords", - channel_id: 0, - records: [ - { - id: 32, - values: [ - [ - "allowed", - "Not Suspicious Traffic", - "1", - "6", - "3", - "ET POLICY GNU\\/Linux APT User-Agent Outbound likely related to package management", - "2013504", - [ - null, - ["POLICY"], - null, - null, - null, - ["2011_08_31"], - null, - ["2020_04_22"], - null, - null - ] - ], - "http", - "91.189.92.152", - "80", - "192.168.0.51", - "33668", - "alert", - "114471366386684", - null, - "TCP", - "1425567868.580509", - "7", - null, - null, - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" - ] - } - ] - }, - - {type: "SearchEnd", channel_id: 0, reason: "eof"}, - - { - type: "SearchStats", - start_time: {sec: 1607378577, ns: 830085000}, - update_time: {sec: 1607378577, ns: 831030000}, - bytes_read: 79130, - bytes_matched: 2163, - records_read: 150, - records_matched: 9 - }, - - {type: "TaskEnd", task_id: 0} -] diff --git a/ppl/detail/models/BrimEvent.ts b/ppl/detail/models/BrimEvent.ts index e369b67242..357c47fdd6 100644 --- a/ppl/detail/models/BrimEvent.ts +++ b/ppl/detail/models/BrimEvent.ts @@ -1,17 +1,17 @@ -import {zng} from "zealot" -import {UnknownEvent} from "./UnknownEvent" +import {ZedRecord} from "zealot/zed/data-types" import {SuricataEvent} from "./SuricataEvent" +import {UnknownEvent} from "./UnknownEvent" import {ZeekEvent} from "./ZeekEvent" export interface BrimEventInterface { getType: () => string getTime: () => Date - getRecord: () => zng.Record + getRecord: () => ZedRecord getEndTime: () => Date | null } export class BrimEvent { - static build(r: zng.Record) { + static build(r: ZedRecord) { if (r.has("_path")) { return new ZeekEvent(r) } else if (r.has("event_type")) { diff --git a/ppl/detail/models/Correlation.ts b/ppl/detail/models/Correlation.ts index fdca0e46bb..bfcdc2552f 100644 --- a/ppl/detail/models/Correlation.ts +++ b/ppl/detail/models/Correlation.ts @@ -1,5 +1,5 @@ import {get, isString} from "lodash" -import {zng} from "zealot" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" const specialUids = { files: "conn_uids", @@ -7,7 +7,7 @@ const specialUids = { } export class Correlation { - constructor(private r: zng.Record) {} + constructor(private r: ZedRecord) {} exists() { return !!(this.getCid() || this.getUid()) @@ -29,10 +29,10 @@ export class Correlation { const name = get(specialUids, path, "uid") if (this.r.has(name)) { const data = this.r.get(name) - if (data instanceof zng.Primitive) { + if (data instanceof ZedPrimitive) { return data.toString() } else { - const value = data.getValue() + const value = data._value if (!value) return null if (isString(value)) return value return value.map((v) => v.toString()).join(" ") diff --git a/ppl/detail/models/SuricataEvent.ts b/ppl/detail/models/SuricataEvent.ts index be4c4abccb..650e3a219b 100644 --- a/ppl/detail/models/SuricataEvent.ts +++ b/ppl/detail/models/SuricataEvent.ts @@ -1,15 +1,15 @@ -import {zng} from "zealot" +import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" import {BrimEventInterface} from "./BrimEvent" export class SuricataEvent implements BrimEventInterface { - constructor(private r: zng.Record) {} + constructor(private r: ZedRecord) {} getRecord() { return this.r } getTime() { - return (this.r.get("ts") as zng.Primitive).toDate() + return (this.r.get("ts") as ZedPrimitive).toDate() } getEndTime() { @@ -21,7 +21,7 @@ export class SuricataEvent implements BrimEventInterface { } getSeverity(): number { - const data = this.r.get("alert.severity") as zng.Primitive + const data = this.r.get("alert.severity") as ZedPrimitive return data.toInt() } } diff --git a/ppl/detail/models/UnknownEvent.ts b/ppl/detail/models/UnknownEvent.ts index 38b25995bb..9b0ebe37ab 100644 --- a/ppl/detail/models/UnknownEvent.ts +++ b/ppl/detail/models/UnknownEvent.ts @@ -1,8 +1,8 @@ -import {zng} from "zealot" +import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" import {BrimEventInterface} from "./BrimEvent" export class UnknownEvent implements BrimEventInterface { - constructor(private r: zng.Record) {} + constructor(private r: ZedRecord) {} getRecord() { return this.r @@ -10,7 +10,7 @@ export class UnknownEvent implements BrimEventInterface { getTime() { if (this.r.has("ts")) { - return (this.r.get("ts") as zng.Primitive).toDate() + return (this.r.get("ts") as ZedPrimitive).toDate() } else { return new Date(0) } diff --git a/ppl/detail/models/ZeekEvent.ts b/ppl/detail/models/ZeekEvent.ts index 3a8f0e037b..e93597c2cd 100644 --- a/ppl/detail/models/ZeekEvent.ts +++ b/ppl/detail/models/ZeekEvent.ts @@ -1,22 +1,22 @@ -import {zng} from "zealot" +import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" import {BrimEventInterface} from "./BrimEvent" export class ZeekEvent implements BrimEventInterface { - constructor(private r: zng.Record) {} + constructor(private r: ZedRecord) {} getRecord() { return this.r } getTime() { - return (this.r.get("ts") as zng.Primitive).toDate() + return (this.r.get("ts") as ZedPrimitive).toDate() } getEndTime() { if (this.r.get("_path").toString() !== "conn") return null - const dur = (this.r.get("duration") as zng.Primitive).toFloat() + const dur = (this.r.get("duration") as ZedPrimitive).toFloat() if (!dur) return - const ts = (this.r.get("ts") as zng.Primitive).toDate() + const ts = (this.r.get("ts") as ZedPrimitive).toDate() return new Date(ts.getTime() + dur * 1000) } diff --git a/ppl/detail/util/sort.ts b/ppl/detail/util/sort.ts index 62a57d8e3b..b101f8b244 100644 --- a/ppl/detail/util/sort.ts +++ b/ppl/detail/util/sort.ts @@ -1,11 +1,11 @@ -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" -export const sort = (logs: zng.Record[]) => { +export const sort = (logs: ZedRecord[]) => { const findConn = (log) => log.try("_path")?.toString() === "conn" return toFront(sortBy(logs, "ts"), findConn) } -function sortBy(logs: zng.Record[], name: string, dir: "asc" | "desc" = "asc") { +function sortBy(logs: ZedRecord[], name: string, dir: "asc" | "desc" = "asc") { const direction = dir === "asc" ? 1 : -1 logs.sort((a, b) => diff --git a/ppl/menus/detailFieldContextMenu.ts b/ppl/menus/detailFieldContextMenu.ts index 37eae276dd..3660bb48bc 100644 --- a/ppl/menus/detailFieldContextMenu.ts +++ b/ppl/menus/detailFieldContextMenu.ts @@ -3,8 +3,7 @@ import {isEqual} from "lodash" import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" import {Space} from "src/js/state/Spaces/types" - -import {zng} from "zealot" +import {ZedField, ZedRecord} from "zealot/zed/data-types" export default function detailFieldContextMenu( program: string, @@ -12,16 +11,16 @@ export default function detailFieldContextMenu( space: Space ) { return function( - field: zng.Field, - log: zng.Record, + field: ZedField, + log: ZedRecord, compound: boolean ): MenuItemConstructorOptions[] { - const isTime = field.data.getType() === "time" + const isTime = field.data.kind === "time" const isConn = log.try("_path")?.toString() === "conn" const isGroupBy = hasGroupByProc(program) - const isIp = ["addr", "set[addr]"].includes(field.data.getType()) + const isIp = ["addr", "set[addr]"].includes(field.data.kind) const hasCol = columns.includes(field.name) - const sameCols = isEqual(log.getColumnNames().sort(), columns.sort()) + const sameCols = isEqual(log.columns.sort(), columns.sort()) const hasPackets = space && space.pcap_support const virusTotal = [ "hassh", diff --git a/ppl/menus/searchFieldContextMenu.ts b/ppl/menus/searchFieldContextMenu.ts index c1dd42a6b8..e5c1df1bd4 100644 --- a/ppl/menus/searchFieldContextMenu.ts +++ b/ppl/menus/searchFieldContextMenu.ts @@ -1,11 +1,10 @@ -import {isEqual} from "lodash" - -import {RightClickBuilder} from "src/js/types" import {MenuItemConstructorOptions} from "electron" -import {zng} from "zealot" +import {isEqual} from "lodash" import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" import {Space} from "src/js/state/Spaces/types" +import {RightClickBuilder} from "src/js/types" +import {ZedField, ZedRecord} from "zealot/zed/data-types" export default function searchFieldContextMenu( program: string, @@ -13,16 +12,16 @@ export default function searchFieldContextMenu( space: Space ): RightClickBuilder { return function( - field: zng.Field, - log: zng.Record, + field: ZedField, + log: ZedRecord, compound: boolean ): MenuItemConstructorOptions[] { - const isTime = field.data.getType() === "time" + const isTime = field.data.kind === "time" const isConn = log.try("_path")?.toString() === "conn" const isGroupBy = hasGroupByProc(program) - const isIp = ["addr", "set[addr]"].includes(field.data.getType()) + const isIp = ["addr", "ip"].includes(field.data.kind) const hasCol = columns.includes(field.name) - const flatColNames = log.flatten().getColumnNames() + const flatColNames = log.flatten().columns const sameCols = isEqual(flatColNames.sort(), columns.sort()) const hasPackets = space && space.pcap_support const virusTotal = [ diff --git a/ppl/summary/bar-chart.tsx b/ppl/summary/bar-chart.tsx index 05e867508e..8f7b2d30e2 100644 --- a/ppl/summary/bar-chart.tsx +++ b/ppl/summary/bar-chart.tsx @@ -1,14 +1,14 @@ -import React from "react" -import {zng} from "zealot" -import {BarRounded} from "@vx/shape" -import {AxisLeft, AxisBottom} from "@vx/axis" -import {scaleLinear, scaleBand} from "@vx/scale" +import {AxisBottom, AxisLeft} from "@vx/axis" import {Group} from "@vx/group" +import {scaleBand, scaleLinear} from "@vx/scale" +import {BarRounded} from "@vx/shape" +import React from "react" import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" type Props = { - records: zng.Record[] + records: ZedRecord[] width: number height: number x: string @@ -33,7 +33,7 @@ export default function BarChart({ const height = outerHeight - margin.top - margin.bottom const data = records.map((r) => ({ name: r.get(x).toString(), - count: (r.get(y) as zng.Primitive).toInt() + count: (r.get(y) as ZedPrimitive).toInt() })) const xScale = scaleBand({ domain: data.map((d) => d.name), diff --git a/ppl/summary/horizontal-bar-chart.tsx b/ppl/summary/horizontal-bar-chart.tsx index 27c0588a70..c6db7eddde 100644 --- a/ppl/summary/horizontal-bar-chart.tsx +++ b/ppl/summary/horizontal-bar-chart.tsx @@ -1,14 +1,14 @@ -import React from "react" -import {zng} from "zealot" -import {BarRounded} from "@vx/shape" -import {AxisLeft, AxisBottom} from "@vx/axis" -import {scaleLinear, scaleBand} from "@vx/scale" +import {AxisBottom, AxisLeft} from "@vx/axis" import {Group} from "@vx/group" +import {scaleBand, scaleLinear} from "@vx/scale" +import {BarRounded} from "@vx/shape" +import React from "react" import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" type Props = { - records: zng.Record[] + records: ZedRecord[] width: number height: number x: string @@ -33,7 +33,7 @@ export default function HorizontalBarChart({ const height = outerHeight - margin.top - margin.bottom const data = records.map((r) => ({ name: r.get(y).toString(), - count: (r.get(x) as zng.Primitive).toInt() + count: (r.get(x) as ZedPrimitive).toInt() })) const xScale = scaleLinear({ domain: [0, Math.max(...data.map((d) => d.count))], diff --git a/scripts/rename.js b/scripts/rename.js new file mode 100644 index 0000000000..2c9919b49b --- /dev/null +++ b/scripts/rename.js @@ -0,0 +1,23 @@ +const {Project} = require("ts-morph") +const _ = require("lodash") +const {kebabCase} = require("lodash") + +const project = new Project({ + tsConfigFilePath: "./tsconfig.json" +}) + +project.getSourceFiles().forEach((srcFile, i) => { + // const oldName = srcFile.getBaseName() + // const name = srcFile.getBaseNameWithoutExtension() + // const newName = kebabCase(name) + srcFile.getExtension() + // const newPath = srcFile.getFilePath().replace(oldName, newName) + // console.log("--> " + newPath) + // srcFile.moveImmediatelySync(newPath) + const oldName = srcFile.getBaseName() + + if (oldName.endsWith("-test.ts")) { + const newName = oldName.replace("-test.ts", ".test.ts") + srcFile.moveImmediatelySync(newName) + console.log("==>" + newName) + } +}) diff --git a/src/css/shared/_type-colors.scss b/src/css/shared/_type-colors.scss index 5e6528616c..30ab149686 100644 --- a/src/css/shared/_type-colors.scss +++ b/src/css/shared/_type-colors.scss @@ -1,5 +1,6 @@ @mixin type-colors { &.addr, + &.ip, &.set\[addr\] { color: var(--ip); } @@ -10,11 +11,20 @@ } &.interval, + &.duration, &.set\[interval\] { color: var(--interval); } &.count, + &.uint8, + &.uint16, + &.uint32, + &.uint64, + &.int8, + &.int16, + &.int32, + &.int64, &.set\[count\] { color: var(--count); } diff --git a/src/js/brim/ast.ts b/src/js/brim/ast.ts index b8c1f9f7b5..ed6854c40c 100644 --- a/src/js/brim/ast.ts +++ b/src/js/brim/ast.ts @@ -56,7 +56,7 @@ function _fieldExprToName(expr) { return `${_fieldExprToName(expr.lhs)}.${_fieldExprToName(expr.rhs)}` } return "" - case "Id": + case "ID": return expr.name case "Root": return "" diff --git a/src/js/brim/cell.test.ts b/src/js/brim/cell.test.ts deleted file mode 100644 index ee49659177..0000000000 --- a/src/js/brim/cell.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import {zng} from "zealot" -import {createCell} from "./cell" - -test("null does not quote", () => { - const data = new zng.Primitive("string", null) - const f = createCell(new zng.Field("service", data)) - - expect(f.queryableValue()).toEqual("null") -}) - -test("string does quote", () => { - const data = new zng.Primitive("string", "d,n,s") - const f = createCell(new zng.Field("service", data)) - - expect(f.queryableValue()).toEqual('"d,n,s"') -}) - -test("string escapes double quotes", () => { - const data = new zng.Primitive("string", '"test"') - const f = createCell(new zng.Field("service", data)) - - // "test", as a value of type 'string', should return "\"test\"" to escape the inner double quotes - expect(f.queryableValue()).toEqual('"\\"test\\""') -}) - -test("string escapes backslash", () => { - const data = new zng.Primitive("string", "Networks,\\") - const f = createCell(new zng.Field("service", data)) - - expect(f.queryableValue()).toBe('"Networks,\\\\"') -}) - -describe("#queryableValue", () => { - const fn = (data: zng.Type) => - createCell(new zng.Field("anything", data)).queryableValue() - - test("set", () => { - const data = new zng.Set("addr", ["192.168.0.53"]) - expect(fn(data)).toBe("192.168.0.53") - }) - - test("set with more than one", () => { - const data = new zng.Set("addr", ["192.168.0.53", "192.168.0.54"]) - expect(fn(data)).toBe("192.168.0.53 192.168.0.54") - }) - - test("set with zero", () => { - const data = new zng.Set("addr", []) - expect(fn(data)).toBe("") - }) - - test("array", () => { - const data = new zng.ZArray("bstring", ["FjV6Wl4bGCsS2H2AZk"]) - expect(fn(data)).toBe('"FjV6Wl4bGCsS2H2AZk"') - }) - - test("array of strings", () => { - const data = new zng.ZArray("bstring", [ - "Mozilla,awesome", - "Killer,Browser" - ]) - - expect(fn(data)).toBe('"Mozilla,awesome" "Killer,Browser"') - }) - - test("null", () => { - const data = new zng.Primitive("string", null) - expect(fn(data)).toBe("null") - }) - - test("ts", () => { - const data = new zng.Primitive("time", "1428917490.931977") - expect(fn(data)).toBe("1428917490.931977") - }) - - test("duration", () => { - const data = new zng.Primitive("interval", "0.000031") - expect(fn(data)).toBe("0.000031") - }) - - test("boolean true", () => { - const data = new zng.Primitive("bool", "T") - expect(fn(data)).toBe("true") - }) - - test("boolean false", () => { - const data = new zng.Primitive("bool", "F") - expect(fn(data)).toBe("false") - }) - - test("port", () => { - const data = new zng.Primitive("port", "5353") - expect(fn(data)).toBe("5353") - }) - - test("addr", () => { - const data = new zng.Primitive("addr", "192.168.0.51") - expect(fn(data)).toBe("192.168.0.51") - }) -}) diff --git a/src/js/brim/cell.ts b/src/js/brim/cell.ts index f2b279805e..10f6b14a22 100644 --- a/src/js/brim/cell.ts +++ b/src/js/brim/cell.ts @@ -1,5 +1,4 @@ -import {zng} from "zealot" - +import {ZedData, ZedPrimitive} from "zealot/zed/data-types" import {createComplexCell} from "./complexCell" import {createPrimitiveCell} from "./primitiveCell" @@ -12,8 +11,13 @@ export interface Cell { guessWidth: () => number } -export function createCell({name, data}: zng.Field): Cell { - if (data instanceof zng.Primitive) { +type Args = { + name: string + data: ZedData +} + +export function createCell({name, data}: Args): Cell { + if (data instanceof ZedPrimitive) { return createPrimitiveCell({name, data}) } else { return createComplexCell({name, data}) diff --git a/src/js/brim/complexCell.ts b/src/js/brim/complexCell.ts index 757de39535..fdd0adcbaf 100644 --- a/src/js/brim/complexCell.ts +++ b/src/js/brim/complexCell.ts @@ -1,15 +1,28 @@ +import { + ZedArray, + ZedEnum, + ZedMap, + ZedRecord, + ZedSet, + ZedUnion +} from "zealot/zed/data-types" import {createCell} from "./cell" -import {zng} from "zealot" import {ONE_CHAR} from "./primitiveCell" export const COMPOUND_FIELD_RGX = /^(set|array|union|record)$/ export type ComplexCell = ReturnType -export function createComplexCell({name, data}: zng.ContainerField) { - const items = (data.getValue() || []).map((v, i) => - createCell({name: name + "." + i, data: v} as zng.Field) - ) +type Args = { + name: string + data: ZedRecord | ZedArray | ZedSet | ZedUnion | ZedEnum | ZedMap +} + +export function createComplexCell({name, data}: Args) { + const items = + "items" in data + ? data.items.map((data, i) => createCell({name: `${name}.${i}`, data})) + : [] return { name, diff --git a/src/js/brim/primitiveCell.ts b/src/js/brim/primitiveCell.ts index 4b7d91329b..f263cfb852 100644 --- a/src/js/brim/primitiveCell.ts +++ b/src/js/brim/primitiveCell.ts @@ -1,6 +1,5 @@ import {isEqual} from "lodash" -import {zng} from "zealot" - +import {ZedPrimitive} from "zealot/zed/data-types" import {withCommas} from "../lib/fmt" import brim from "./" @@ -17,11 +16,12 @@ const ESCAPED_BACK_SLASH = "\\\\" interface PrimitiveField { name: string - data: zng.Primitive + data: ZedPrimitive } export function createPrimitiveCell({name, data}: PrimitiveField) { - const {type, value} = data + const type = data.kind + const value = data._value return { name, diff --git a/src/js/brim/program.test.ts b/src/js/brim/program.test.ts index 25a34d9984..d90fe3a8a8 100644 --- a/src/js/brim/program.test.ts +++ b/src/js/brim/program.test.ts @@ -1,4 +1,6 @@ -import {zjson, zng} from "zealot" +import {COUNT, IP, STRING} from "test/fixtures/zjson-types" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {RecordFieldType} from "zealot/zed/zjson" import { addHeadProc, getHeadCount, @@ -12,7 +14,10 @@ import {createCell} from "./cell" describe("excluding and including", () => { const field = createCell( - new zng.Field("uid", new zng.Primitive("string", "123")) + new ZedField({ + name: "uid", + data: new ZedPrimitive({type: STRING, value: "123"}) + }) ) test("excluding a field", () => { @@ -25,12 +30,12 @@ describe("excluding and including", () => { }) test("excluding a field with a pipe", () => { - const data = new zng.Primitive("string", "HTTP") + const data = new ZedPrimitive({type: STRING, value: "HTTP"}) const program = brim .program( 'tx_hosts=2606:4700:30::681c:135e fuid!="F2nyqx46YRDAYe4c73" | sort' ) - .exclude(createCell(new zng.Field("source", data))) + .exclude(createCell(new ZedField({name: "source", data}))) .string() expect(program).toEqual( @@ -59,17 +64,12 @@ describe("excluding and including", () => { describe("drill down", () => { const columns = [ - {name: "id", type: "record", of: [{name: "orig_h", type: "addr"}]}, - {name: "proto", type: "enum"}, - {name: "query", type: "string"}, - {name: "count", type: "count"} - ] as zjson.Column[] - const result = new zng.Record(columns, [ - ["192.168.0.54"], - "udp", - "WPAD", - "24" - ]) + {name: "id", type: {kind: "record", fields: [{name: "orig_h", type: IP}]}}, + {name: "proto", type: STRING}, + {name: "query", type: STRING}, + {name: "count", type: COUNT} + ] as RecordFieldType[] + const result = ZedRecord.of(columns, [["192.168.0.54"], "udp", "WPAD", "24"]) test("when there is no leading filter", () => { const program = brim @@ -87,7 +87,7 @@ describe("drill down", () => { .string() expect(program).toBe( - '_path=dns id.orig_h=192.168.0.54 proto=udp query="WPAD"' + '_path=dns id.orig_h=192.168.0.54 proto="udp" query="WPAD"' ) }) @@ -106,14 +106,14 @@ describe("drill down", () => { .drillDown(result) .string() - expect(program).toBe("names james proto=udp") + expect(program).toBe('names james proto="udp"') }) test("count by and filter the same", () => { - const result = new zng.Record( + const result = ZedRecord.of( [ - {type: "string", name: "md5"}, - {type: "count", name: "count"} + {type: STRING, name: "md5"}, + {type: COUNT, name: "count"} ], ["123", "1"] ) @@ -127,10 +127,10 @@ describe("drill down", () => { }) test("filter query", () => { - const result = new zng.Record( + const result = ZedRecord.of( [ - {name: "md5", type: "string"}, - {name: "count", type: "count"} + {name: "md5", type: STRING}, + {name: "count", type: COUNT} ], ["9f51ef98c42df4430a978e4157c43dd5", "21"] ) @@ -150,8 +150,8 @@ describe("drill down", () => { describe("count by", () => { test("empty program", () => { - const data = new zng.Primitive("string", "heyo") - const field = createCell(new zng.Field("_path", data)) + const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const field = createCell(new ZedField({name: "_path", data})) const program = brim .program() .countBy(field) @@ -161,8 +161,8 @@ describe("count by", () => { }) test("append a count to an existing query", () => { - const data = new zng.Primitive("string", "heyo") - const field = createCell(new zng.Field("query", data)) + const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const field = createCell(new ZedField({name: "query", data})) const program = brim .program("dns") .countBy(field) diff --git a/src/js/brim/program.ts b/src/js/brim/program.ts index 2431e95879..4978613f7e 100644 --- a/src/js/brim/program.ts +++ b/src/js/brim/program.ts @@ -1,11 +1,10 @@ import {isEqual} from "lodash" import {parse} from "zealot" - -import {EVERYTHING_FILTER, FILTER_PROC, TUPLE_PROCS} from "./ast" +import {ZedRecord} from "zealot/zed/data-types" import {trim} from "../lib/Str" -import brim from "./" import stdlib from "../stdlib" -import {zng} from "zealot" +import brim from "./" +import {EVERYTHING_FILTER, FILTER_PROC, TUPLE_PROCS} from "./ast" import {Cell, createCell} from "./cell" export default function(p = "", pins: string[] = []) { @@ -40,7 +39,7 @@ export default function(p = "", pins: string[] = []) { return this }, - drillDown(log: zng.Record) { + drillDown(log: ZedRecord) { let filter = this.filter() const newFilters = this.ast() .groupByKeys() diff --git a/src/js/components/ConnVersation.tsx b/src/js/components/ConnVersation.tsx index 9f10c1f55d..07c874dfcf 100644 --- a/src/js/components/ConnVersation.tsx +++ b/src/js/components/ConnVersation.tsx @@ -1,31 +1,30 @@ +import contextMenu from "app/detail/flows/contextMenu" import {every} from "lodash" -import {useDispatch} from "react-redux" import React from "react" - -import {Fieldset} from "./Typography" -import VerticalTable from "./Tables/VerticalTable" +import {useDispatch} from "react-redux" +import {ZedField, ZedRecord} from "zealot/zed/data-types" import connHistoryView from "../lib/connHistoryView" -import {zng} from "zealot" -import contextMenu from "app/detail/flows/contextMenu" +import VerticalTable from "./Tables/VerticalTable" +import {Fieldset} from "./Typography" const ORIG_FIELDS = ["orig_bytes", "orig_pkts", "orig_ip_bytes", "local_orig"] const RESP_FIELDS = ["resp_bytes", "resp_pkts", "resp_ip_bytes", "local_resp"] type Props = { - record: zng.Record + record: ZedRecord } -function filter(record: zng.Record, names: string[]) { +function filter(record: ZedRecord, names: string[]) { const cols = [] const vals = [] names.forEach((n) => { - const i = record.getColumnNames().indexOf(n) - cols.push(record.type[i]) - vals.push(record.value[i]) + const i = record.columns.indexOf(n) + cols.push(record._type[i]) + vals.push(record._value[i]) }) - return new zng.Record(cols, vals) + return ZedRecord.of(cols, vals) } const ConnVersation = ({record}: Props) => { @@ -67,9 +66,9 @@ const ConnHistory = ({history = ""}) => ( type HostProps = { className: string title: string - ip: zng.Field - port: zng.Field - record: zng.Record + ip: ZedField + port: ZedField + record: ZedRecord } const Host = ({className, title, ip, port, record}: HostProps) => { @@ -94,7 +93,7 @@ const Host = ({className, title, ip, port, record}: HostProps) => { {port.data.toString()}

@@ -102,7 +101,7 @@ const Host = ({className, title, ip, port, record}: HostProps) => { ) } -ConnVersation.shouldShow = (record: zng.Record) => +ConnVersation.shouldShow = (record: ZedRecord) => every(ORIG_FIELDS, (name) => record.has(name)) export default ConnVersation diff --git a/src/js/components/FieldCell.tsx b/src/js/components/FieldCell.tsx index c7c77b53c8..0f06fa46f1 100644 --- a/src/js/components/FieldCell.tsx +++ b/src/js/components/FieldCell.tsx @@ -1,10 +1,9 @@ -import React from "react" import classNames from "classnames" - +import React from "react" +import {ZedField, ZedRecord} from "zealot/zed/data-types" import {createCell} from "../brim/cell" -import {zng} from "zealot" -type Props = {field: zng.Field; record: zng.Record} +type Props = {field: ZedField; record: ZedRecord} function getBackground(field, record) { if (field.name === "event_type" && field.data.toString() === "alert") { @@ -23,10 +22,10 @@ export default function FieldCell({field, record}: Props) { className={classNames( "field-cell", field.name, - field.data.getType(), + field.data.kind, getBackground(field, record), { - null: field.data.value === null + null: field.data.isUnset() } )} > diff --git a/src/js/components/LogCell/CompoundField.tsx b/src/js/components/LogCell/CompoundField.tsx index ceb7c7677e..9340191c26 100644 --- a/src/js/components/LogCell/CompoundField.tsx +++ b/src/js/components/LogCell/CompoundField.tsx @@ -1,22 +1,26 @@ -import React from "react" import classNames from "classnames" - -import SingleField from "./SingleField" -import {zng} from "zealot" +import React from "react" +import {ZedField, ZedRecord} from "zealot/zed/data-types" import {createComplexCell} from "../../brim/complexCell" +import SingleField from "./SingleField" type Props = { - field: zng.ContainerField - log: zng.Record + field: ZedField + log: ZedRecord menuBuilder: Function } export default function CompoundField({field, log, menuBuilder}: Props) { - const compound = createComplexCell(field as zng.ContainerField) + // @ts-ignore + const compound = createComplexCell(field) const render = [] for (let i = 0; i < compound.length; ++i) { - const item = new zng.Field(field.name, field.data.at(i)) + const item = new ZedField({ + name: field.name, + // @ts-ignore + data: field.data.items && field.data.items[i] + }) if (item) { const menu = menuBuilder(item, log, true) render.push() diff --git a/src/js/components/LogCell/SingleField.tsx b/src/js/components/LogCell/SingleField.tsx index 6fa8567a2a..e6ad43b97e 100644 --- a/src/js/components/LogCell/SingleField.tsx +++ b/src/js/components/LogCell/SingleField.tsx @@ -1,15 +1,14 @@ -import React, {useEffect, useRef, useState} from "react" import classNames from "classnames" - +import React, {useEffect, useRef, useState} from "react" +import {ZedField, ZedRecord} from "zealot/zed/data-types" import {$Menu} from "../../electron/menu" +import lib from "../../lib" import {showContextMenu} from "../../lib/System" import FieldCell from "../FieldCell" -import lib from "../../lib" -import {zng} from "zealot" type Props = { - field: zng.Field - record: zng.Record + field: ZedField + record: ZedRecord menu: $Menu } diff --git a/src/js/components/LogCell/index.tsx b/src/js/components/LogCell/index.tsx index 5df994bc3f..2b84f936b4 100644 --- a/src/js/components/LogCell/index.tsx +++ b/src/js/components/LogCell/index.tsx @@ -1,15 +1,14 @@ -import React, {useState} from "react" import classNames from "classnames" - +import React, {useState} from "react" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" import {RightClickBuilder} from "../../types" +import Tooltip from "../Tooltip" import CompoundField from "./CompoundField" import SingleField from "./SingleField" -import Tooltip from "../Tooltip" -import {zng} from "zealot" type Props = { - field: zng.Field - log: zng.Record + field: ZedField + log: ZedRecord style?: Object rightClick: RightClickBuilder } @@ -23,7 +22,8 @@ const getTooltipStyle = (el: Element) => { export default function LogCell({field, style, rightClick, log}: Props) { const [hover, setHover] = useState(false) const [tooltipStyle, setTooltipStyle] = useState({}) - const {name, data} = field + const data = field.data + const name = field.name function handleMouseEnter(e) { setHover(true) @@ -35,7 +35,7 @@ export default function LogCell({field, style, rightClick, log}: Props) { } return (
} else { - return ( - - ) + return } } diff --git a/src/js/components/LogDetails/ConnPanel.tsx b/src/js/components/LogDetails/ConnPanel.tsx index 57a571ef3c..71fc2cb5b8 100644 --- a/src/js/components/LogDetails/ConnPanel.tsx +++ b/src/js/components/LogDetails/ConnPanel.tsx @@ -1,11 +1,10 @@ +import PanelHeading from "app/detail/PanelHeading" import React from "react" - +import {ZedRecord} from "zealot/zed/data-types" import ConnVersation from "../ConnVersation" -import {zng} from "zealot" -import PanelHeading from "app/detail/PanelHeading" type Props = { - record: zng.Record + record: ZedRecord } const ConnPanel = ({record}: Props) => { diff --git a/src/js/components/LogDetails/Md5Panel.tsx b/src/js/components/LogDetails/Md5Panel.tsx index d0587aa571..f8471bc581 100644 --- a/src/js/components/LogDetails/Md5Panel.tsx +++ b/src/js/components/LogDetails/Md5Panel.tsx @@ -1,16 +1,15 @@ -import {useDispatch} from "react-redux" +import contextMenu from "app/detail/flows/contextMenu" +import PanelHeading from "app/detail/PanelHeading" import React, {useEffect, useState} from "react" - +import {useDispatch} from "react-redux" +import {AppDispatch} from "src/js/state/types" +import {ZedRecord, ZedField} from "zealot/zed/data-types" import {md5Search} from "../../../../app/search/flows/md5-search" -import HorizontalTable from "../Tables/HorizontalTable" import InlineTableLoading from "../InlineTableLoading" -import {AppDispatch} from "src/js/state/types" -import {zng} from "zealot" -import PanelHeading from "app/detail/PanelHeading" -import contextMenu from "app/detail/flows/contextMenu" +import HorizontalTable from "../Tables/HorizontalTable" type Props = { - record: zng.Record + record: ZedRecord } export const Md5Panel = ({record}: Props) => { @@ -26,10 +25,10 @@ export const Md5Panel = ({record}: Props) => { const {response, abort} = dispatch(md5Search(logMd5)) response .status(setStatus) - .chan(0, (records) => setFilenames(records)) - .chan(1, (records) => setMd5(records)) - .chan(2, (records) => setRx(records)) - .chan(3, (records) => setTx(records)) + .chan(0, ({rows}) => setFilenames(rows)) + .chan(1, ({rows}) => setMd5(rows)) + .chan(2, ({rows}) => setRx(rows)) + .chan(3, ({rows}) => setTx(rows)) return abort }, [logMd5]) @@ -54,9 +53,9 @@ export const Md5Panel = ({record}: Props) => { } type Props2 = { - logs: zng.Record[] + logs: ZedRecord[] expect: number - onRightClick?: (f: zng.Field, r: zng.Record) => void + onRightClick?: (f: ZedField, r: ZedRecord) => void } function AsyncTable({logs, expect, onRightClick}: Props2) { @@ -65,7 +64,7 @@ function AsyncTable({logs, expect, onRightClick}: Props2) { } else { return ( diff --git a/src/js/components/LogRow.test.tsx b/src/js/components/LogRow.test.tsx deleted file mode 100644 index 516ac14320..0000000000 --- a/src/js/components/LogRow.test.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import React from "react" - -import {shallow} from "enzyme" - -import {conn} from "../test/mockLogs" -import {createColumn} from "../state/Columns/models/column" -import LogRow from "./LogRow" -import TableColumns from "../models/TableColumns" - -let props -beforeEach(() => { - const log = conn() - props = { - columns: new TableColumns("1", log.type.map(createColumn), { - "1": {isVisible: true} - }), - dimens: { - rowHeight: 25, - rowWidth: 300, - viewHeight: 1000, - viewWidth: 1000, - listHeight: 20000, - listWidth: 10000 - }, - highlight: false, - index: 1, - log: log, - timeZone: "UTC", - timeFormat: "", - onClick: () => {}, - onDoubleClick: () => {}, - rightClick: () => [] - } -}) - -test("Rendering the row works", () => { - shallow() -}) diff --git a/src/js/components/LogRow.tsx b/src/js/components/LogRow.tsx index ec4072013b..db5d4018d1 100644 --- a/src/js/components/LogRow.tsx +++ b/src/js/components/LogRow.tsx @@ -1,12 +1,11 @@ -import React, {memo, MouseEvent} from "react" import classNames from "classnames" import isEqual from "lodash/isEqual" - +import React, {memo, MouseEvent} from "react" +import {ZedRecord} from "zealot/zed/data-types" +import TableColumns from "../models/TableColumns" import {RightClickBuilder, ViewerDimens} from "../types" import LogCell from "./LogCell" import * as Styler from "./Viewer/Styler" -import TableColumns from "../models/TableColumns" -import {zng} from "zealot" type Props = { dimens: ViewerDimens @@ -14,7 +13,7 @@ type Props = { index: number timeZone: string timeFormat: string - log: zng.Record + log: ZedRecord columns: TableColumns onClick: (e: MouseEvent) => void onDoubleClick: (e: MouseEvent) => void @@ -37,8 +36,7 @@ const LogRow = (props: Props) => { const width = dimens.rowWidth !== "auto" ? column.width || 300 : "auto" const field = log.tryField(column.name) const key = `${index}-${colIndex}` - - if (field && field.data && !(field.data instanceof zng.Record)) { + if (field && field.data && !(field.data instanceof ZedRecord)) { return ( { store.dispatch(Layout.showRightSidebar()) store.dispatch(tabHistory.push(workspacesPath())) - store.dispatch(LogDetails.push(new zng.Record([], []))) + store.dispatch(LogDetails.push(ZedRecord.of([], []))) const el = provide(store, ) expect(el.html()).toBe("") }) diff --git a/src/js/components/RightPane.tsx b/src/js/components/RightPane.tsx index 464c3e50d3..0864834e7d 100644 --- a/src/js/components/RightPane.tsx +++ b/src/js/components/RightPane.tsx @@ -1,7 +1,7 @@ import DetailPane from "app/detail/Pane" import React from "react" import {connect} from "react-redux" -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" import {openLogDetailsWindow} from "../flows/openLogDetailsWindow" import ExpandWindow from "../icons/ExpandWindow" import dispatchToProps from "../lib/dispatchToProps" @@ -24,7 +24,7 @@ import Pane, { import {XRightPaneExpander} from "./RightPaneExpander" type StateProps = { - currentLog: zng.Record + currentLog: ZedRecord prevExists: boolean nextExists: boolean isOpen: boolean diff --git a/src/js/components/SearchResults/ResultsTable.tsx b/src/js/components/SearchResults/ResultsTable.tsx index 44584bc241..334515f8aa 100644 --- a/src/js/components/SearchResults/ResultsTable.tsx +++ b/src/js/components/SearchResults/ResultsTable.tsx @@ -3,7 +3,7 @@ import {isEmpty} from "lodash" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" import React, {useEffect} from "react" import {connect, useDispatch} from "react-redux" -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" import {openLogDetailsWindow} from "../../flows/openLogDetailsWindow" import {viewLogDetail} from "../../flows/viewLogDetail" import dispatchToProps from "../../lib/dispatchToProps" @@ -29,7 +29,7 @@ import NoResults from "./NoResults" import {useRowSelection} from "./selection" type StateProps = { - logs: zng.Record[] + logs: ZedRecord[] timeZone: string timeFormat: string isIncomplete: boolean diff --git a/src/js/components/Tables/HorizontalTable.tsx b/src/js/components/Tables/HorizontalTable.tsx index ebd567b630..7be6c89305 100644 --- a/src/js/components/Tables/HorizontalTable.tsx +++ b/src/js/components/Tables/HorizontalTable.tsx @@ -1,12 +1,11 @@ import * as React from "react" - +import {ZedField, ZedRecord} from "zealot/zed/data-types" import Table, {TableData, TableHeader} from "./Table" -import {zjson, zng} from "zealot" type Props = { - descriptor: zjson.Column[] - logs: zng.Record[] - onRightClick?: (f: zng.Field, r: zng.Record) => void + descriptor: ZedField[] + logs: ZedRecord[] + onRightClick?: (f: ZedField, r: ZedRecord) => void } export default function HorizontalTable({ @@ -26,7 +25,7 @@ export default function HorizontalTable({ {logs.map((log, index) => ( - {log.getFields().map((field, index) => ( + {log.fields.map((field, index) => ( } -export function TableHeader({column}: {column: Column}) { - return {column.name} +export function TableHeader({column}: {column: ZedField}) { + return {column.name} } type Props = { - record: zng.Record - field: zng.Field - onRightClick?: (f: zng.Field, r: zng.Record) => void + record: ZedRecord + field: ZedField + onRightClick?: (f: ZedField, r: ZedRecord) => void } export function TableData({field, record, onRightClick}: Props) { @@ -26,7 +24,7 @@ export function TableData({field, record, onRightClick}: Props) { } return ( - + ) diff --git a/src/js/components/Tables/VerticalTable.tsx b/src/js/components/Tables/VerticalTable.tsx index 504dde1b3a..9abf866c3a 100644 --- a/src/js/components/Tables/VerticalTable.tsx +++ b/src/js/components/Tables/VerticalTable.tsx @@ -1,13 +1,12 @@ -import React from "react" import classNames from "classnames" - +import React from "react" +import {ZedField, ZedRecord} from "zealot/zed/data-types" import Table, {TableData, TableHeader} from "./Table" -import {zjson, zng} from "zealot" type Props = { - descriptor: zjson.Column[] - record: zng.Record - onRightClick?: (f: zng.Field, r: zng.Record) => void + descriptor: ZedField[] + record: ZedRecord + onRightClick?: (f: ZedField, r: ZedRecord) => void light?: boolean } @@ -20,12 +19,12 @@ export default function VerticalTable({ return ( - {descriptor.map((column, index) => ( + {descriptor.map((field, index) => ( - + diff --git a/src/js/components/Viewer/Chunk.tsx b/src/js/components/Viewer/Chunk.tsx index c68fd67f4a..d9626e4363 100644 --- a/src/js/components/Viewer/Chunk.tsx +++ b/src/js/components/Viewer/Chunk.tsx @@ -1,16 +1,15 @@ import React from "react" - +import {ZedRecord} from "zealot/zed/data-types" +import TableColumns from "../../models/TableColumns" import {RowRenderer, ViewerDimens} from "../../types" import * as Styler from "./Styler" -import TableColumns from "../../models/TableColumns" -import {zng} from "zealot" type Props = { rowRenderer: RowRenderer columns: TableColumns dimens: ViewerDimens rows: number[] - logs: zng.Record[] + logs: ZedRecord[] } export default class Chunk extends React.Component { diff --git a/src/js/components/Viewer/HeaderCell.tsx b/src/js/components/Viewer/HeaderCell.tsx index d1a9b12838..4aae155d78 100644 --- a/src/js/components/Viewer/HeaderCell.tsx +++ b/src/js/components/Viewer/HeaderCell.tsx @@ -67,7 +67,6 @@ export default function HeaderCell({column, tableId, sorts}: Props) { dispatch(appendQuerySortBy(column.name, sorted === "asc" ? "desc" : "asc")) dispatch(submitSearch()) } - return (
any scrollPos: ScrollPosition diff --git a/src/js/electron/menu/actions/detailActions.ts b/src/js/electron/menu/actions/detailActions.ts index ce0ccd5a3b..3066abe896 100644 --- a/src/js/electron/menu/actions/detailActions.ts +++ b/src/js/electron/menu/actions/detailActions.ts @@ -17,49 +17,55 @@ import tab from "../../../state/Tab" import virusTotal from "../../../services/virusTotal" import {downloadPcap} from "../../../flows/downloadPcap" import {openNewSearchTab} from "../../../flows/openNewSearchWindow" -import {zng} from "zealot" import {createCell} from "../../../brim/cell" +import { + ZedField, + ZedRecordSpec, + ZedRecord, + ZedPrimitive, + ZedFieldSpec +} from "zealot/zed/data-types" function buildDetailActions() { return { copy: action({ name: "detail-cell-menu-copy", label: "Copy", - listener(_dispatch, data: zng.SerializedField) { - const f = zng.Field.deserialize(data) + listener(_dispatch, data: ZedFieldSpec) { + const f = ZedField.deserialize(data) lib.doc.copyToClipboard(f.data.toString()) } }), countBy: action({ name: "detail-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: zng.SerializedField) { + listener(dispatch, data: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryCountBy(zng.Field.deserialize(data))) + dispatch(appendQueryCountBy(ZedField.deserialize(data))) dispatch(openNewSearchTab()) } }), detail: action({ name: "detail-cell-menu-detail", label: "View details", - listener(dispatch, log: zng.SerializedRecord) { - dispatch(viewLogDetail(zng.Record.deserialize(log))) + listener(dispatch, log: ZedRecordSpec) { + dispatch(viewLogDetail(ZedRecord.deserialize(log))) } }), exclude: action({ name: "detail-cell-menu-exclude", label: "Filter != value in new search", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryExclude(zng.Field.deserialize(field))) + dispatch(appendQueryExclude(ZedField.deserialize(field))) dispatch(openNewSearchTab()) } }), freshInclude: action({ name: "detail-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, field: zng.SerializedField) { - const cell = createCell(zng.Field.deserialize(field)) + listener(dispatch, field: ZedFieldSpec) { + const cell = createCell(ZedField.deserialize(field)) dispatch(SearchBar.clearSearchBar()) dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) dispatch(openNewSearchTab()) @@ -68,14 +74,12 @@ function buildDetailActions() { fromTime: action({ name: "detail-cell-menu-from-time", label: 'Use as "start" time in new search', - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - if (field.data.type === "time") { + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + if (field.data.kind === "time") { dispatch(SearchBar.clearSearchBar()) dispatch( - tab.setFrom( - brim.time((field.data as zng.Primitive).toDate()).toTs() - ) + tab.setFrom(brim.time((field.data as ZedPrimitive).toDate()).toTs()) ) dispatch(openNewSearchTab()) } @@ -84,10 +88,10 @@ function buildDetailActions() { groupByDrillDown: action({ name: "detail-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program, log: zng.SerializedRecord) { + listener(dispatch, program, log: ZedRecordSpec) { const newProgram = brim .program(program) - .drillDown(zng.Record.deserialize(log)) + .drillDown(ZedRecord.deserialize(log)) .string() if (newProgram) { @@ -100,27 +104,27 @@ function buildDetailActions() { include: action({ name: "detail-cell-menu-include", label: "Filter = value in new search", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryInclude(zng.Field.deserialize(field))) + dispatch(appendQueryInclude(ZedField.deserialize(field))) dispatch(openNewSearchTab()) } }), in: action({ name: "detail-cell-menu-in", label: "Filter in field in new search", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryIn(createCell(zng.Field.deserialize(field)))) + dispatch(appendQueryIn(createCell(ZedField.deserialize(field)))) dispatch(openNewSearchTab()) } }), notIn: action({ name: "detail-cell-menu-not-in", label: "Filter not in field in new search", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryNotIn(createCell(zng.Field.deserialize(field)))) + dispatch(appendQueryNotIn(createCell(ZedField.deserialize(field)))) dispatch(openNewSearchTab()) } }), @@ -135,14 +139,14 @@ function buildDetailActions() { pcaps: action({ name: "detail-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, log: zng.SerializedRecord) { - dispatch(downloadPcap(zng.Record.deserialize(log))) + listener(dispatch, log: ZedRecordSpec) { + dispatch(downloadPcap(ZedRecord.deserialize(log))) } }), sortAsc: action({ name: "detail-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQuerySortBy(field.name, "asc")) dispatch(openNewSearchTab()) @@ -151,7 +155,7 @@ function buildDetailActions() { sortDesc: action({ name: "detail-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, field: zng.SerializedField) { + listener(dispatch, field: ZedFieldSpec) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQuerySortBy(field.name, "desc")) dispatch(openNewSearchTab()) @@ -160,14 +164,14 @@ function buildDetailActions() { toTime: action({ name: "detail-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - if (field.data.type === "time") { + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + if (field.data.kind === "time") { dispatch(SearchBar.clearSearchBar()) dispatch( tab.setTo( brim - .time((field.data as zng.Primitive).toDate()) + .time((field.data as ZedPrimitive).toDate()) .add(1, "ms") .toTs() ) @@ -179,17 +183,17 @@ function buildDetailActions() { virusTotalRightclick: action({ name: "detail-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(_dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) + listener(_dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) open(virusTotal.url(field.data.toString())) } }), whoisRightclick: action({ name: "detail-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - dispatch(Modal.show("whois", {addr: field.data.value})) + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + dispatch(Modal.show("whois", {addr: field.data.toString()})) } }) } diff --git a/src/js/electron/menu/actions/searchActions.ts b/src/js/electron/menu/actions/searchActions.ts index 63db3fc8ba..87d35b5881 100644 --- a/src/js/electron/menu/actions/searchActions.ts +++ b/src/js/electron/menu/actions/searchActions.ts @@ -1,3 +1,15 @@ +import lib from "src/js/lib" +import { + ZedField, + ZedFieldSpec, + ZedPrimitive, + ZedRecord, + ZedRecordSpec +} from "zealot/zed/data-types" +import brim from "../../../brim" +import {createCell} from "../../../brim/cell" +import {downloadPcap} from "../../../flows/downloadPcap" +import scrollToLog from "../../../flows/scrollToLog" import { appendQueryCountBy, appendQueryExclude, @@ -6,39 +18,33 @@ import { appendQueryNotIn, appendQuerySortBy } from "../../../flows/searchBar/actions" -import {downloadPcap} from "../../../flows/downloadPcap" import {submitSearch} from "../../../flows/submitSearch/mod" import {viewLogDetail} from "../../../flows/viewLogDetail" +import open from "../../../lib/open" import ErrorFactory from "../../../models/ErrorFactory" +import virusTotal from "../../../services/virusTotal" import Layout from "../../../state/Layout/actions" import Modal from "../../../state/Modal" import Notice from "../../../state/Notice" import SearchBar from "../../../state/SearchBar" -import action from "./action" -import brim from "../../../brim" -import open from "../../../lib/open" -import scrollToLog from "../../../flows/scrollToLog" import tab from "../../../state/Tab" -import virusTotal from "../../../services/virusTotal" -import {zng} from "zealot" -import {createCell} from "../../../brim/cell" -import lib from "src/js/lib" +import action from "./action" function buildSearchActions() { return { copy: action({ name: "search-cell-menu-copy", label: "Copy", - listener(_dispatch, data: zng.SerializedField) { - const f = zng.Field.deserialize(data) + listener(_dispatch, data: ZedFieldSpec) { + const f = ZedField.deserialize(data) lib.doc.copyToClipboard(f.data.toString()) } }), countBy: action({ name: "search-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: zng.SerializedField) { - const f = zng.Field.deserialize(data) + listener(dispatch, data: ZedFieldSpec) { + const f = ZedField.deserialize(data) dispatch(appendQueryCountBy(f)) dispatch(submitSearch()) } @@ -46,8 +52,8 @@ function buildSearchActions() { detail: action({ name: "search-cell-menu-detail", label: "Open details", - listener(dispatch, data: zng.SerializedRecord) { - const record = zng.Record.deserialize(data) + listener(dispatch, data: ZedRecordSpec) { + const record = ZedRecord.deserialize(data) dispatch(Layout.showRightSidebar()) dispatch(viewLogDetail(record)) } @@ -55,16 +61,16 @@ function buildSearchActions() { exclude: action({ name: "search-cell-menu-exclude", label: "Filter != value", - listener(dispatch, data: zng.SerializedField) { - dispatch(appendQueryExclude(zng.Field.deserialize(data))) + listener(dispatch, data: ZedFieldSpec) { + dispatch(appendQueryExclude(ZedField.deserialize(data))) dispatch(submitSearch()) } }), freshInclude: action({ name: "search-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, data: zng.SerializedField) { - const cell = createCell(zng.Field.deserialize(data)) + listener(dispatch, data: ZedFieldSpec) { + const cell = createCell(ZedField.deserialize(data)) dispatch(SearchBar.clearSearchBar()) dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) dispatch(submitSearch()) @@ -73,13 +79,11 @@ function buildSearchActions() { fromTime: action({ name: "search-cell-menu-from-time", label: 'Use as "start" time', - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - if (field.data.getType() === "time") { + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + if (field.data.kind === "time") { dispatch( - tab.setFrom( - brim.time((field.data as zng.Primitive).toDate()).toTs() - ) + tab.setFrom(brim.time((field.data as ZedPrimitive).toDate()).toTs()) ) dispatch(submitSearch()) } @@ -88,8 +92,8 @@ function buildSearchActions() { groupByDrillDown: action({ name: "search-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program: string, data: zng.SerializedRecord) { - const record = zng.Record.deserialize(data) + listener(dispatch, program: string, data: ZedRecordSpec) { + const record = ZedRecord.deserialize(data) const newProgram = brim .program(program) .drillDown(record) @@ -105,31 +109,27 @@ function buildSearchActions() { include: action({ name: "search-cell-menu-include", label: "Filter = value", - listener(dispatch, data: zng.SerializedField) { - dispatch(appendQueryInclude(zng.Field.deserialize(data))) + listener(dispatch, data: ZedFieldSpec) { + dispatch(appendQueryInclude(ZedField.deserialize(data))) dispatch(submitSearch()) } }), in: action({ name: "search-cell-menu-in", label: "Filter in field", - listener(dispatch, data: zng.SerializedField) { - dispatch(appendQueryIn(createCell(zng.Field.deserialize(data)))) + listener(dispatch, data: ZedFieldSpec) { + dispatch(appendQueryIn(createCell(ZedField.deserialize(data)))) dispatch(submitSearch()) } }), jumpToTime: action({ name: "search-cell-menu-show-context", label: "View in full context", - listener( - dispatch, - fieldData: zng.SerializedField, - recordData: zng.SerializedRecord - ) { - const field = zng.Field.deserialize(fieldData) - const record = zng.Record.deserialize(recordData) - const brimTime = brim.time((field.data as zng.Primitive).toDate()) - if (field.data.type === "time") { + listener(dispatch, fieldData: ZedFieldSpec, recordData: ZedRecordSpec) { + const field = ZedField.deserialize(fieldData) + const record = ZedRecord.deserialize(recordData) + const brimTime = brim.time((field.data as ZedPrimitive).toDate()) + if (field.data.kind === "time") { dispatch(tab.setFrom(brimTime.subtract(1, "minutes").toTs())) dispatch(tab.setTo(brimTime.add(1, "minutes").toTs())) dispatch(SearchBar.clearSearchBar()) @@ -147,19 +147,15 @@ function buildSearchActions() { notIn: action({ name: "search-cell-menu-not-in", label: "Filter not in field", - listener(dispatch, data: zng.SerializedField) { - dispatch(appendQueryNotIn(createCell(zng.Field.deserialize(data)))) + listener(dispatch, data: ZedFieldSpec) { + dispatch(appendQueryNotIn(createCell(ZedField.deserialize(data)))) dispatch(submitSearch()) } }), logResult: action({ name: "search-cell-menu-log-result", label: "Log result to console", - listener( - _dispatch, - field: zng.SerializedField, - log: zng.SerializedRecord - ) { + listener(_dispatch, field: ZedFieldSpec, log: ZedRecordSpec) { console.log(JSON.stringify(log)) console.log(JSON.stringify(field)) } @@ -167,15 +163,15 @@ function buildSearchActions() { pcaps: action({ name: "search-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, data: zng.SerializedRecord) { - dispatch(downloadPcap(zng.Record.deserialize(data))) + listener(dispatch, data: ZedRecordSpec) { + dispatch(downloadPcap(ZedRecord.deserialize(data))) } }), sortAsc: action({ name: "search-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) dispatch(appendQuerySortBy(field.name, "asc")) dispatch(submitSearch()) } @@ -183,8 +179,8 @@ function buildSearchActions() { sortDesc: action({ name: "search-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) dispatch(appendQuerySortBy(field.name, "desc")) dispatch(submitSearch()) } @@ -192,13 +188,13 @@ function buildSearchActions() { toTime: action({ name: "search-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - if (field.data.type === "time") { + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + if (field.data.kind === "time") { dispatch( tab.setTo( brim - .time((field.data as zng.Primitive).toDate()) + .time((field.data as ZedPrimitive).toDate()) .add(1, "ms") .toTs() ) @@ -210,19 +206,19 @@ function buildSearchActions() { virusTotalRightclick: action({ name: "search-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - if (field.data instanceof zng.Primitive && field.data.isSet()) { - open(virusTotal.url(field.data.getValue() as string)) + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + if (field.data instanceof ZedPrimitive && !field.data.isUnset()) { + open(virusTotal.url(field.data.toString() as string)) } } }), whoisRightclick: action({ name: "search-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: zng.SerializedField) { - const field = zng.Field.deserialize(data) - dispatch(Modal.show("whois", {addr: field.data.value})) + listener(dispatch, data: ZedFieldSpec) { + const field = ZedField.deserialize(data) + dispatch(Modal.show("whois", {addr: field.data.toString()})) } }) } diff --git a/src/js/flows/downloadPcap.ts b/src/js/flows/downloadPcap.ts index b567bd8db7..1d32ed85c4 100644 --- a/src/js/flows/downloadPcap.ts +++ b/src/js/flows/downloadPcap.ts @@ -1,9 +1,9 @@ -import {Thunk} from "../state/types" -import Packets from "../state/Packets" +import {ZedRecord} from "zealot/zed/data-types" import open from "../lib/open" -import {zng} from "zealot" +import Packets from "../state/Packets" +import {Thunk} from "../state/types" -export const downloadPcap = (currentLog: zng.Record): Thunk => (dispatch) => { +export const downloadPcap = (currentLog: ZedRecord): Thunk => (dispatch) => { dispatch(Packets.fetch(currentLog)).then((pcapFile) => open(pcapFile, {newWindow: true}) ) diff --git a/src/js/flows/rightclick/cellMenu.test.ts b/src/js/flows/rightclick/cellMenu.test.ts index 78a67ea380..c825e7ff3c 100644 --- a/src/js/flows/rightclick/cellMenu.test.ts +++ b/src/js/flows/rightclick/cellMenu.test.ts @@ -1,9 +1,18 @@ import {MenuItemConstructorOptions} from "electron" - -import {conn, dns} from "../../test/mockLogs" -import fixtures from "../../test/fixtures" -import {zng} from "zealot" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" +import {COUNT, IP, STRING, TIME} from "test/fixtures/zjson-types" +import {ZedRecord} from "zealot/zed/data-types" +import fixtures from "../../test/fixtures" + +const conn = ZedRecord.of( + [ + {name: "id", type: {kind: "record", fields: [{name: "orig_h", type: IP}]}}, + {name: "_path", type: STRING}, + {name: "ts", type: TIME} + ], + [["192.168.0.1"], "conn", "1234513"] +) +const dns = ZedRecord.of([{name: "query", type: STRING}], ["dns.query.yo"]) function menuText(menu: MenuItemConstructorOptions[]) { return menu @@ -15,16 +24,13 @@ const space = fixtures("space1") describe("Log Right Click", () => { const program = "*" - const columnNames = conn() - .flatten() - .getColumnNames() + const columnNames = conn.columns test("conn log with pcap support", () => { - const log = conn() - const field = log.getField("id.orig_h") + const field = conn.getField("id.orig_h") const ctxMenu = searchFieldContextMenu(program, columnNames, space)( field, - log, + conn, false ) @@ -34,7 +40,7 @@ describe("Log Right Click", () => { test("conn log without pcap support", () => { space.pcap_support = false - const log = conn() + const log = conn const field = log.getField("id.orig_h") const ctxMenu = searchFieldContextMenu(program, columnNames, space)( field, @@ -46,7 +52,7 @@ describe("Log Right Click", () => { }) test("dns log", () => { - const log = dns() + const log = dns const field = log.getField("query") const ctxMenu = searchFieldContextMenu(program, columnNames, space)( field, @@ -59,7 +65,7 @@ describe("Log Right Click", () => { }) test("time field for conn log", () => { - const log = conn() + const log = conn const field = log.getField("ts") const ctxMenu = searchFieldContextMenu(program, columnNames, space)( field, @@ -77,10 +83,13 @@ describe("Analysis Right Click", () => { const columnNames = ["count", "id.orig_h"] test("nested field", () => { - const log = new zng.Record( + const log = ZedRecord.of( [ - {name: "count", type: "count"}, - {name: "id", type: "record", of: [{name: "orig_h", type: "addr"}]} + {name: "count", type: COUNT}, + { + name: "id", + type: {kind: "record", fields: [{name: "orig_h", type: IP}]} + } ], ["300", ["192.168.0.51"]] ) @@ -95,10 +104,10 @@ describe("Analysis Right Click", () => { }) test("non-address field", () => { - const log = new zng.Record( + const log = ZedRecord.of( [ - {name: "count", type: "count"}, - {name: "proto", type: "string"} + {name: "count", type: COUNT}, + {name: "proto", type: STRING} ], ["100", "tcp"] ) diff --git a/src/js/flows/scrollToLog.ts b/src/js/flows/scrollToLog.ts index 646f7c03d7..5786337aad 100644 --- a/src/js/flows/scrollToLog.ts +++ b/src/js/flows/scrollToLog.ts @@ -1,9 +1,9 @@ +import {isEqual} from "lodash" +import {ZedRecord} from "zealot/zed/data-types" import {Thunk} from "../state/types" import Viewer from "../state/Viewer" -import {isEqual} from "lodash" -import {zng} from "zealot" -export default (log: zng.Record): Thunk => (dispatch, getState) => { +export default (log: ZedRecord): Thunk => (dispatch, getState) => { const state = getState() const logs = Viewer.getLogs(state) const index = logs.findIndex((log2) => isEqual(log2, log)) diff --git a/src/js/flows/search/handler.ts b/src/js/flows/search/handler.ts index 6ae1afa1f5..3504d1471a 100644 --- a/src/js/flows/search/handler.ts +++ b/src/js/flows/search/handler.ts @@ -1,5 +1,7 @@ import {SearchResponse} from "./response" import whenIdle from "../../lib/whenIdle" +import {DecodedZJSON} from "zealot/zed/zjson" +import {RecordsCallbackArgs} from "zealot/fetcher/records_callback" function abortError(e) { return /user aborted/i.test(e.message) @@ -7,11 +9,11 @@ function abortError(e) { export function handle(request: any) { const response = new SearchResponse() - const channels = new Map() + const channels = new Map() const promise = new Promise((resolve, reject) => { function flushBuffer() { for (const [id, data] of channels) { - response.emit(id, data.allRecords, data.schemas) + response.emit(id, data) } channels.clear() } @@ -37,11 +39,8 @@ export function handle(request: any) { response.emit("start", task_id) response.emit("status", "FETCHING") }) - .records(({channel, allRecords, schemas}) => { - channels.set(channel, { - allRecords, - schemas - }) + .records(({channel, rows, schemas, context}: RecordsCallbackArgs) => { + channels.set(channel, {rows, schemas, context}) flushBufferLazy() }) .end(({id, error}) => { diff --git a/src/js/flows/search/response.ts b/src/js/flows/search/response.ts index 7a1d3b5ad2..b36ae0a95c 100644 --- a/src/js/flows/search/response.ts +++ b/src/js/flows/search/response.ts @@ -1,3 +1,4 @@ +import {DecodedZJSON} from "zealot/zed/zjson" import {SearchStats, SearchStatus} from "../../types/searches" type EventNames = @@ -17,7 +18,7 @@ export class SearchResponse { this.callbacks = new Map() } - chan(num: number, func: (records: any, schemas: any) => void) { + chan(num: number, func: (data: DecodedZJSON) => void) { this.callbacks.set(num, func) return this } diff --git a/src/js/flows/searchBar/actions.ts b/src/js/flows/searchBar/actions.ts index c3939f95e7..b7849c9f65 100644 --- a/src/js/flows/searchBar/actions.ts +++ b/src/js/flows/searchBar/actions.ts @@ -1,15 +1,15 @@ -import {Thunk} from "../../state/types" +import {ZedField} from "zealot/zed/data-types" +import brim from "../../brim" +import {Cell, createCell} from "../../brim/cell" +import {onlyWhitespace} from "../../lib/Str" +import SearchBar from "../../state/SearchBar" import { getSearchBar, getSearchBarInputValue } from "../../state/SearchBar/selectors" -import {onlyWhitespace} from "../../lib/Str" -import SearchBar from "../../state/SearchBar" -import brim from "../../brim" -import {zng} from "zealot" -import {Cell, createCell} from "../../brim/cell" +import {Thunk} from "../../state/types" -export function appendQueryInclude(field: zng.Field): Thunk { +export function appendQueryInclude(field: ZedField): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -22,7 +22,7 @@ export function appendQueryInclude(field: zng.Field): Thunk { } } -export function appendQueryExclude(field: zng.Field): Thunk { +export function appendQueryExclude(field: ZedField): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -35,7 +35,7 @@ export function appendQueryExclude(field: zng.Field): Thunk { } } -export function appendQueryCountBy(field: zng.Field): Thunk { +export function appendQueryCountBy(field: ZedField): Thunk { return function(dispatch, getState) { const {current, pinned} = getSearchBar(getState()) const query = [...pinned, current].join(" ") diff --git a/src/js/flows/viewLogDetail.ts b/src/js/flows/viewLogDetail.ts index c3db51bda1..7499a180ab 100644 --- a/src/js/flows/viewLogDetail.ts +++ b/src/js/flows/viewLogDetail.ts @@ -1,13 +1,13 @@ -import LogDetails from "../state/LogDetails" -import {Thunk} from "../state/types" -import {zng} from "zealot" import {isEqual} from "lodash" import {fetchCorrelation} from "ppl/detail/flows/fetch" -import Notice from "../state/Notice" +import {ZedRecord} from "zealot/zed/data-types" import ErrorFactory from "../models/ErrorFactory" import Current from "../state/Current" +import LogDetails from "../state/LogDetails" +import Notice from "../state/Notice" +import {Thunk} from "../state/types" -export const viewLogDetail = (record: zng.Record): Thunk => ( +export const viewLogDetail = (record: ZedRecord): Thunk => ( dispatch, getState ) => { diff --git a/src/js/models/TableColumns.ts b/src/js/models/TableColumns.ts index 80336cdc27..0e0121dcae 100644 --- a/src/js/models/TableColumns.ts +++ b/src/js/models/TableColumns.ts @@ -1,8 +1,8 @@ +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {createCell} from "../brim/cell" +import columnOrder from "../lib/columnOrder" import {$Column} from "../state/Columns/models/column" import {ColumnSettingsMap, TableColumn} from "../state/Columns/types" -import columnOrder from "../lib/columnOrder" -import {createCell} from "../brim/cell" -import {zng} from "zealot" export default class TableColumns { cols: TableColumn[] @@ -31,7 +31,7 @@ export default class TableColumns { ) } - setWidths(logs: zng.Record[]) { + setWidths(logs: ZedRecord[]) { const MAX_WIDTH = 500 const resizeHandle = 5 const sortIcon = 11 @@ -39,15 +39,20 @@ export default class TableColumns { this.cols.forEach((col) => { if (col.width) return const colName = createCell( - new zng.Field("", new zng.Primitive("string", col.name)) + new ZedField({ + name: "", + data: new ZedPrimitive({ + type: {kind: "primitive", name: "string"}, + value: col.name + }) + }) ) let max = colName.guessWidth() + resizeHandle + sortIcon - logs.forEach((log) => { const data = log.try(col.name) if (data) { - const cell = createCell(new zng.Field(name, data)) + const cell = createCell(new ZedField({name: col.name, data})) const len = cell.guessWidth() if (len > max) max = len } diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index e8ef7647a7..ae9e2b8489 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -1,105 +1,33 @@ -import {zng} from "zealot" +import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" +import {ZedPrimitive, ZedRecord, ZedRecordSpec} from "zealot/zed/data-types" import {connCorrelation} from "./programs" test("conn correlation", () => { - const conn: zng.SerializedRecord = { + const conn: ZedRecordSpec = { type: { - type: "record", - of: [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "string"}, - { - name: "id", - of: [ - {name: "orig_h", type: "addr"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "addr"}, - {name: "resp_p", type: "port"} - ], - type: "record" - }, - {name: "proto", type: "enum"}, - {name: "service", type: "string"}, - {name: "duration", type: "interval"}, - {name: "orig_bytes", type: "count"}, - {name: "resp_bytes", type: "count"}, - {name: "conn_state", type: "string"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "count"}, - {name: "history", type: "string"}, - {name: "orig_pkts", type: "count"}, - {name: "orig_ip_bytes", type: "count"}, - {name: "resp_pkts", type: "count"}, - {name: "resp_ip_bytes", type: "count"}, - {name: "tunnel_parents", of: "string", type: "set"}, - { - name: "geo", - of: [ - { - name: "orig", - of: [ - {name: "country_code", type: "string"}, - {name: "region", type: "string"}, - {name: "city", type: "string"}, - {name: "latitude", type: "double"}, - {name: "longitude", type: "double"} - ], - type: "record" - }, - { - name: "resp", - of: [ - {name: "country_code", type: "string"}, - {name: "region", type: "string"}, - {name: "city", type: "string"}, - {name: "latitude", type: "double"}, - {name: "longitude", type: "double"} - ], - type: "record" - } - ], - type: "record" - }, - {name: "community_id", type: "string"} + kind: "record", + fields: [ + {name: "ts", type: TIME}, + {name: "uid", type: STRING}, + {name: "duration", type: INTERVAL}, + {name: "community_id", type: STRING} ] }, value: [ - "conn", "1425568032.998178", "CbOjYpkXn9LfqV51c", - ["192.168.0.51", "41970", "130.239.18.173", "80"], - "tcp", - "http", "0.70995", - "676", - "1530", - "S1", - null, - null, - "0", - "ShADad", - "7", - "976", - "7", - "1822", - null, - [ - [null, null, null, null, null], - ["SE", "AC", "Umeå", "63.8284", "20.2597"] - ], "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" ] } - const record = zng.Record.deserialize(conn) + const record = ZedRecord.deserialize(conn) expect( connCorrelation( - record.get("uid") as zng.Primitive, - record.get("community_id") as zng.Primitive, - record.get("ts") as zng.Primitive, - record.get("duration") as zng.Primitive + record.get("uid") as ZedPrimitive, + record.get("community_id") as ZedPrimitive, + record.get("ts") as ZedPrimitive, + record.get("duration") as ZedPrimitive ) ).toBe( 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' diff --git a/src/js/searches/programs.ts b/src/js/searches/programs.ts index ecdd375cdf..357ec19c8b 100644 --- a/src/js/searches/programs.ts +++ b/src/js/searches/programs.ts @@ -1,4 +1,4 @@ -import {zng} from "zealot" +import {ZedPrimitive} from "zealot/zed/data-types" import zql from "../zql" export function md5Correlation(md5: string) { @@ -17,11 +17,11 @@ export function filenameCorrelation(md5: string) { return `md5=${md5} | count() by filename, mime_type | sort -r | head 5` } -export function uidFilter(uid: string | zng.Primitive) { +export function uidFilter(uid: string | ZedPrimitive) { return zql`uid=${uid} or ${uid} in conn_uids or ${uid} in uids or referenced_file.uid=${uid}` } -export function cidFilter(cid: string | zng.Primitive) { +export function cidFilter(cid: string | ZedPrimitive) { return zql`community_id=${cid}` } @@ -42,19 +42,19 @@ export function correlationIds({uid, cid}: RelatedIds) { return [filters.join(" or "), correlationLimit()].join(" | ") } -export function uidCorrelation(uid: string | zng.Primitive) { +export function uidCorrelation(uid: string | ZedPrimitive) { return `${uidFilter(uid)} | ${correlationLimit()}` } -export function cidCorrelation(cid: string | zng.Primitive) { +export function cidCorrelation(cid: string | ZedPrimitive) { return `${cidFilter(cid)} | ${correlationLimit()}` } export function connCorrelation( - uid: zng.Primitive, - cid: zng.Primitive, - ts: zng.Primitive, - duration: zng.Primitive + uid: ZedPrimitive, + cid: ZedPrimitive, + ts: ZedPrimitive, + duration: ZedPrimitive ) { const tsDate = ts.toDate() const dur = duration.toFloat() + 90 // Add a 1.5 minute buffer for events that get logged late diff --git a/src/js/state/Chart/actions.ts b/src/js/state/Chart/actions.ts index c29626c5f5..22babe5289 100644 --- a/src/js/state/Chart/actions.ts +++ b/src/js/state/Chart/actions.ts @@ -1,8 +1,8 @@ -import {ChartData, CHART_CLEAR, CHART_RECORDS, CHART_STATUS} from "./types" -import {SearchStatus} from "../../types/searches" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" import MergeHash from "../../models/MergeHash" import UniqArray from "../../models/UniqArray" -import {zng} from "zealot" +import {SearchStatus} from "../../types/searches" +import {ChartData, CHART_CLEAR, CHART_RECORDS, CHART_STATUS} from "./types" export default { setStatus: (tabId: string, status: SearchStatus): CHART_STATUS => ({ @@ -10,7 +10,7 @@ export default { status, tabId }), - appendRecords: (tabId: string, records: zng.Record[]): CHART_RECORDS => ({ + appendRecords: (tabId: string, records: ZedRecord[]): CHART_RECORDS => ({ type: "CHART_RECORDS", data: histogramFormat(records), tabId @@ -18,12 +18,12 @@ export default { clear: (tabId?: string): CHART_CLEAR => ({type: "CHART_CLEAR", tabId}) } -function histogramFormat(records: zng.Record[]): ChartData { +function histogramFormat(records: ZedRecord[]): ChartData { const paths = new UniqArray() const table = new MergeHash() records.forEach((r) => { - const [ts, path, count] = r.getValue() as zng.Primitive[] + const [ts, path, count] = r.fields.map((f) => f.data) as ZedPrimitive[] try { const pathName = path.toString() diff --git a/src/js/state/Chart/test.ts b/src/js/state/Chart/test.ts index 764fe47e37..793d6491cc 100644 --- a/src/js/state/Chart/test.ts +++ b/src/js/state/Chart/test.ts @@ -1,7 +1,9 @@ +import {COUNT, STRING, TIME} from "test/fixtures/zjson-types" +import {ZedRecord} from "zealot/zed/data-types" +import {RecordFieldType} from "zealot/zed/zjson" +import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import chart from "./" -import initTestStore from "../../test/initTestStore" -import {zjson, zng} from "zealot" let store, tabId beforeEach(() => { @@ -9,15 +11,15 @@ beforeEach(() => { tabId = Tabs.getActive(store.getState()) }) -const columns = [ - {name: "ts", type: "time"}, - {name: "_path", type: "string"}, - {name: "count", type: "count"} -] as zjson.Column[] +const fields = [ + {name: "ts", type: TIME}, + {name: "_path", type: STRING}, + {name: "count", type: COUNT} +] as RecordFieldType[] const records = [ - new zng.Record(columns, ["0", "conn", "500"]), - new zng.Record(columns, ["100", "dns", "300"]) + ZedRecord.of(fields, ["0", "conn", "500"]), + ZedRecord.of(fields, ["100", "dns", "300"]) ] test("chart records append", () => { diff --git a/src/js/state/Columns/models/column.ts b/src/js/state/Columns/models/column.ts index 2a734c78ab..c08a633114 100644 --- a/src/js/state/Columns/models/column.ts +++ b/src/js/state/Columns/models/column.ts @@ -1,10 +1,22 @@ -import {Column} from "../../../types" +import {RecordFieldType, Type} from "zealot/zed/zjson" export type $Column = {name: string; type: string; key: string} -export function createColumn(c: Column) { +function getType(type: Type) { + if (type.kind === "primitive") { + return type.name + } else if (type.kind === "typename") { + return type.name + } else if (type.kind === "typedef") { + return type.name + } +} + +export function createColumn(c: RecordFieldType) { + const type = getType(c.type) return { - ...c, - key: `${c.name}:${c.type}` + name: c.name, + type, + key: `${c.name}:${type}` } } diff --git a/src/js/state/Columns/models/columnSet.ts b/src/js/state/Columns/models/columnSet.ts index 88d06faa59..93dca4c80c 100644 --- a/src/js/state/Columns/models/columnSet.ts +++ b/src/js/state/Columns/models/columnSet.ts @@ -1,26 +1,25 @@ -import {uniqBy, keys} from "lodash" - +import {uniqBy} from "lodash" +import {TypeContext} from "zealot/zed/zjson" import {$Column, createColumn} from "./column" -import {ViewerColumns} from "../../Viewer/types" -export function createColumnSet(c: ViewerColumns) { +export function createColumnSet(c: TypeContext) { return { getName() { - const types = keys(c) - if (types.length === 0) { + if (c.size === 0) { return "none" - } else if (types.length === 1) { - return types[0] + } else if (c.size === 1) { + return Array.from(c.keys())[0] } else { return "temp" } }, getUniqColumns() { let allCols = [] - for (const id in c) { - let schema = c[id] - let columns = schema.flatten() - allCols = [...allCols, ...columns] + for (const typedef of c.values()) { + let inner = typedef.flatten().innerType + if (inner.kind === "record") { + allCols = [...allCols, ...inner.fields] + } } return uniqBy<$Column>(allCols.map(createColumn), "key") } diff --git a/src/js/state/Columns/selectors.ts b/src/js/state/Columns/selectors.ts index 64ce5b57a0..958e1b42de 100644 --- a/src/js/state/Columns/selectors.ts +++ b/src/js/state/Columns/selectors.ts @@ -1,13 +1,12 @@ import {createSelector} from "reselect" - -import {ColumnsState} from "./types" +import {ZedRecord} from "zealot/zed/data-types" +import TableColumns from "../../models/TableColumns" +import activeTabSelect from "../Tab/activeTabSelect" import {State} from "../types" +import Viewer from "../Viewer" import {ViewerColumns} from "../Viewer/types" import {createColumnSet} from "./models/columnSet" -import TableColumns from "../../models/TableColumns" -import Viewer from "../Viewer" -import activeTabSelect from "../Tab/activeTabSelect" -import {zng} from "zealot" +import {ColumnsState} from "./types" const getColumns = activeTabSelect((tab) => tab.columns) @@ -15,7 +14,7 @@ const getCurrentTableColumns = createSelector< State, ViewerColumns, ColumnsState, - zng.Record[], + ZedRecord[], TableColumns >( Viewer.getColumns, @@ -24,7 +23,11 @@ const getCurrentTableColumns = createSelector< (viewerColumns, columnSettings, logs) => { const set = createColumnSet(viewerColumns) const prefs = columnSettings[set.getName()] - const table = new TableColumns(set.getName(), set.getUniqColumns(), prefs) + const table = new TableColumns( + set.getName() as string, + set.getUniqColumns(), + prefs + ) table.setWidths(logs.slice(0, 50)) return table } diff --git a/src/js/state/Columns/touch.test.ts b/src/js/state/Columns/touch.test.ts index dd87a97a97..c35fb33b4d 100644 --- a/src/js/state/Columns/touch.test.ts +++ b/src/js/state/Columns/touch.test.ts @@ -3,18 +3,39 @@ import Columns from "./" import actions from "./actions" import initTestStore from "../../test/initTestStore" import touch from "./touch" -import {zng} from "zealot" +import ZedTypeDef from "zealot/zed/type-def" +import {STRING, INTERVAL, TIME} from "test/fixtures/zjson-types" -const columns = { - "1": new zng.Schema([ - {name: "_path", type: "string"}, - {name: "duration", type: "interval"} - ]), - "2": new zng.Schema([ - {name: "_path", type: "string"}, - {name: "ts", type: "time"} - ]) -} +const columns = new Map( + Object.entries({ + "1": new ZedTypeDef({ + type: { + name: "1", + kind: "typedef", + type: { + kind: "record", + fields: [ + {name: "_path", type: STRING}, + {name: "duration", type: INTERVAL} + ] + } + } + }), + "2": new ZedTypeDef({ + type: { + name: "2", + kind: "typedef", + type: { + kind: "record", + fields: [ + {name: "_path", type: STRING}, + {name: "ts", type: TIME} + ] + } + } + }) + }) +) let store beforeEach(() => { @@ -23,7 +44,7 @@ beforeEach(() => { test("visibility false when at least one is hidden", () => { const prefName = "temp" - const col = createColumn({name: "_path", type: "string"}) + const col = createColumn({name: "_path", type: STRING}) const update = {[col.key]: {isVisible: false}} store.dispatch(actions.updateColumns(prefName, update)) @@ -52,8 +73,8 @@ test("visibility true when no preferences exist", () => { test("visibility true when all are visible", () => { const prefName = "temp" - const col = createColumn({name: "_path", type: "string"}) - const col2 = createColumn({name: "duration", type: "interval"}) + const col = createColumn({name: "_path", type: STRING}) + const col2 = createColumn({name: "duration", type: INTERVAL}) const update = { [col.key]: {isVisible: true}, [col2.key]: {isVisible: true} diff --git a/src/js/state/Columns/touch.ts b/src/js/state/Columns/touch.ts index b6efef60c7..4f8ec83e11 100644 --- a/src/js/state/Columns/touch.ts +++ b/src/js/state/Columns/touch.ts @@ -10,7 +10,7 @@ export default (columns: ViewerColumns): Thunk => (dispatch, getState) => { const name = set.getName() const cols = set.getUniqColumns() const prefs = createColumnPrefs(selectors.getColumns(getState())[name]) - const defaults = prefs.getDefaults(cols) - dispatch(Columns.updateColumns(name, defaults)) + const defaults = prefs.getDefaults(cols) + dispatch(Columns.updateColumns(name.toString(), defaults)) } diff --git a/src/js/state/LogDetails/actions.ts b/src/js/state/LogDetails/actions.ts index 8144ac03fb..1737a0a655 100644 --- a/src/js/state/LogDetails/actions.ts +++ b/src/js/state/LogDetails/actions.ts @@ -1,3 +1,5 @@ +import {SearchStatus} from "src/js/types/searches" +import {ZedRecord} from "zealot/zed/data-types" import { LOG_DETAIL_BACK, LOG_DETAIL_CLEAR, @@ -5,11 +7,9 @@ import { LOG_DETAIL_PUSH, LOG_DETAIL_UPDATE } from "./types" -import {zng} from "zealot" -import {SearchStatus} from "src/js/types/searches" export default { - push: (record: zng.Record): LOG_DETAIL_PUSH => ({ + push: (record: ZedRecord): LOG_DETAIL_PUSH => ({ type: "LOG_DETAIL_PUSH", record: record.serialize() }), @@ -22,7 +22,7 @@ export default { type: "LOG_DETAIL_FORWARD" }), - updateUidLogs: (records: zng.Record[]): LOG_DETAIL_UPDATE => { + updateUidLogs: (records: ZedRecord[]): LOG_DETAIL_UPDATE => { return { type: "LOG_DETAIL_UPDATE", updates: { diff --git a/src/js/state/LogDetails/selectors.ts b/src/js/state/LogDetails/selectors.ts index 1f0a7f4af0..13b561e232 100644 --- a/src/js/state/LogDetails/selectors.ts +++ b/src/js/state/LogDetails/selectors.ts @@ -1,12 +1,12 @@ import {createSelector} from "reselect" - -import {State} from "../types" +import {SearchStatus} from "src/js/types/searches" +import activeTabSelect from "../Tab/activeTabSelect" import {TabState} from "../Tab/types" +import {State} from "../types" import {LogDetailHistory, toHistory} from "./reducer" -import activeTabSelect from "../Tab/activeTabSelect" import {LogDetailsState} from "./types" -import {SearchStatus} from "src/js/types/searches" -import {zng} from "zealot" + +import {ZedRecord} from "zealot/zed/data-types" const getLogDetails = activeTabSelect((state: TabState) => { return state.logDetails @@ -17,25 +17,23 @@ const getHistory = createSelector( (logDetails) => toHistory(logDetails) ) -const build = createSelector( +const build = createSelector( getHistory, (history) => { const entry = history.current() if (entry && entry.log) { - return zng.Record.deserialize(entry.log) + return ZedRecord.deserialize(entry.log) } else { return null } } ) -const getUidLogs = createSelector( +const getUidLogs = createSelector( getHistory, (history) => { const entry = history.current() - return entry - ? entry.uidLogs.map((data) => zng.Record.deserialize(data)) - : [] + return entry ? entry.uidLogs.map((data) => ZedRecord.deserialize(data)) : [] } ) @@ -47,7 +45,7 @@ const getUidStatus = createSelector( } ) -const getConnLog = createSelector( +const getConnLog = createSelector( getUidLogs, (uids) => { return uids.find((log) => log.try("_path")?.toString() === "conn") diff --git a/src/js/state/LogDetails/test.ts b/src/js/state/LogDetails/test.ts index 6796f631f5..6b4bb13243 100644 --- a/src/js/state/LogDetails/test.ts +++ b/src/js/state/LogDetails/test.ts @@ -1,15 +1,19 @@ -import LogDetails from "./" +import {ZedRecord} from "zealot/zed/data-types" +import {RecordType} from "zealot/zed/zjson" import initTestStore from "../../test/initTestStore" -import {zjson, zng} from "zealot" - -const columns = [ - {name: "_td", type: "count"}, - {name: "letter", type: "string"} -] as zjson.Column[] +import LogDetails from "./" -const record = new zng.Record(columns, ["1", "a"]) -const record2 = new zng.Record(columns, ["1", "b"]) -const record3 = new zng.Record(columns, ["1", "c"]) +const type = { + kind: "record", + fields: [ + {name: "_td", type: {kind: "primitive", name: "count"}}, + {name: "letter", type: {kind: "primitive", name: "string"}} + ] +} as RecordType + +const record = new ZedRecord({type, value: ["1", "a"]}) +const record2 = new ZedRecord({type, value: ["1", "b"]}) +const record3 = new ZedRecord({type, value: ["1", "c"]}) let store beforeEach(() => { diff --git a/src/js/state/LogDetails/types.ts b/src/js/state/LogDetails/types.ts index e46ccd332a..b720f68ac8 100644 --- a/src/js/state/LogDetails/types.ts +++ b/src/js/state/LogDetails/types.ts @@ -1,4 +1,4 @@ -import {zng} from "zealot" +import {ZedRecordSpec} from "zealot/zed/data-types" import {SearchStatus} from "../../types/searches" export type LogDetailsState = { @@ -7,8 +7,8 @@ export type LogDetailsState = { } export type LogDetails = { - log: zng.SerializedRecord - uidLogs: zng.SerializedRecord[] + log: ZedRecordSpec + uidLogs: ZedRecordSpec[] uidStatus: SearchStatus } @@ -21,7 +21,7 @@ export type LogDetailsAction = export type LOG_DETAIL_PUSH = { type: "LOG_DETAIL_PUSH" - record: zng.SerializedRecord + record: ZedRecordSpec } export type LOG_DETAIL_UPDATE = { diff --git a/src/js/state/Packets/flows.ts b/src/js/state/Packets/flows.ts index b83395773d..d8deb6ea68 100644 --- a/src/js/state/Packets/flows.ts +++ b/src/js/state/Packets/flows.ts @@ -1,26 +1,25 @@ -import {join} from "path" import {remote} from "electron" - -import {Thunk} from "../types" +import {join} from "path" +import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {getZealot} from "../../flows/getZealot" import {saveToFile} from "../../lib/response" import Current from "../Current" import Packets from "../Packets" +import {Thunk} from "../types" import View from "../View" -import {getZealot} from "../../flows/getZealot" -import {zng} from "zealot" export default { - fetch: (log: zng.Record): Thunk> => ( + fetch: (log: ZedRecord): Thunk> => ( dispatch: Function, getState: Function ) => { - dispatch(Packets.request(log.get("uid").toString())) + dispatch(Packets.request(log["uid"].toString())) dispatch(View.showDownloads()) const state = getState() const zealot = dispatch(getZealot()) const spaceId = Current.getSpaceId(state) - const ts = log.get("ts") as zng.Primitive - const dur = log.get("duration") as zng.Primitive + const ts = log.get("ts") as ZedPrimitive + const dur = log.get("duration") as ZedPrimitive const args = { ts_sec: getSec(ts), ts_ns: getNs(ts), @@ -52,22 +51,21 @@ export default { } } -function getSec(data: zng.Primitive): number { - if (data.isSet()) { - return parseInt(data.getValue().split(".")[0]) - } else { - return 0 - } +function getSec(data: ZedPrimitive): number { + if (data.isUnset()) return 0 + + return parseInt(data.toString().split(".")[0]) } -function getNs(data: zng.Primitive): number { - if (data.isSet()) { - const v = data.getValue().split(".") - if (v.length === 2) { - const frac = v[1] - const digits = frac.length - return parseInt(frac) * Math.pow(10, 9 - digits) - } +function getNs(data: ZedPrimitive): number { + if (data.isUnset()) return 0 + + const v = data.toString().split(".") + if (v.length === 2) { + const frac = v[1] + const digits = frac.length + return parseInt(frac) * Math.pow(10, 9 - digits) + } else { + return 0 } - return 0 } diff --git a/src/js/state/SearchBar/test.ts b/src/js/state/SearchBar/test.ts index fcb228d1e6..50ea8b005e 100644 --- a/src/js/state/SearchBar/test.ts +++ b/src/js/state/SearchBar/test.ts @@ -1,7 +1,7 @@ import tabHistory from "app/router/tab-history" import {lakePath} from "app/router/utils/paths" import brim from "src/js/brim" -import {createZealotMock, zng} from "zealot" +import {createZealotMock} from "zealot" import { appendQueryCountBy, appendQueryExclude, @@ -10,13 +10,15 @@ import { import {submitSearch} from "../../flows/submitSearch/mod" import fixtures from "../../test/fixtures" import initTestStore from "../../test/initTestStore" -import Url from "../Url" import Search from "../Search" import {SpanArgs} from "../Search/types" import Spaces from "../Spaces" +import Url from "../Url" import Workspaces from "../Workspaces" import SearchBar from "./" import {SearchBarState} from "./types" +import {ZedPrimitive, ZedField} from "zealot/zed/data-types" +import {STRING} from "test/fixtures/zjson-types" let store, mock beforeEach(() => { @@ -127,16 +129,16 @@ test("search bar pin remove when out of bounds", () => { }) test("append an include field", () => { - const data = new zng.Primitive("string", "conn") - const field = new zng.Field("_path", data) + const data = new ZedPrimitive({type: STRING, value: "conn"}) + const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryInclude(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe('_path="conn"') }) test("append an include field when some text already exists", () => { - const data = new zng.Primitive("string", "conn") - const field = new zng.Field("_path", data) + const data = new ZedPrimitive({type: STRING, value: "conn"}) + const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), appendQueryInclude(field) @@ -145,15 +147,15 @@ test("append an include field when some text already exists", () => { }) test("append an exclude field", () => { - const data = new zng.Primitive("string", "conn") - const field = new zng.Field("_path", data) + const data = new ZedPrimitive({type: STRING, value: "conn"}) + const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryExclude(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe('_path!="conn"') }) test("append an exclude field when some text already exists", () => { - const data = new zng.Primitive("string", "conn") - const field = new zng.Field("_path", data) + const data = new ZedPrimitive({type: STRING, value: "conn"}) + const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), appendQueryExclude(field) @@ -162,15 +164,15 @@ test("append an exclude field when some text already exists", () => { }) test("append a count by field", () => { - const data = new zng.Primitive("string", "conn") - const field = new zng.Field("_path", data) + const data = new ZedPrimitive({type: STRING, value: "conn"}) + const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryCountBy(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe("* | count() by _path") }) test("append a count to an existing query", () => { - const data = new zng.Primitive("string", "ho ho") - const field = new zng.Field("query", data) + const data = new ZedPrimitive({type: STRING, value: "ho ho"}) + const field = new ZedField({name: "query", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), appendQueryCountBy(field) @@ -179,8 +181,8 @@ test("append a count to an existing query", () => { }) test("append a count to an existing query with a pin", () => { - const data = new zng.Primitive("string", "heyo") - const field = new zng.Field("query", data) + const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const field = new ZedField({name: "query", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), SearchBar.pinSearchBar(), diff --git a/src/js/state/Viewer/actions.ts b/src/js/state/Viewer/actions.ts index 55fc57eec4..0274c5424d 100644 --- a/src/js/state/Viewer/actions.ts +++ b/src/js/state/Viewer/actions.ts @@ -1,4 +1,4 @@ -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" import { @@ -47,14 +47,14 @@ export const setEndStatus = ( export const appendRecords = ( tabId: string | null | undefined, - records: zng.Record[] + records: ZedRecord[] ): VIEWER_RECORDS => { return {type: "VIEWER_RECORDS", records, tabId} } export const setRecords = ( tabId: string | undefined, - records: zng.Record[] + records: ZedRecord[] ): VIEWER_SET_RECORDS => { return {type: "VIEWER_SET_RECORDS", records, tabId} } diff --git a/src/js/state/Viewer/reducer.ts b/src/js/state/Viewer/reducer.ts index 959adc1e0c..9c336f46f8 100644 --- a/src/js/state/Viewer/reducer.ts +++ b/src/js/state/Viewer/reducer.ts @@ -8,7 +8,7 @@ const init = (): ViewerState => ({ records: [], endStatus: "INCOMPLETE", status: "INIT", - columns: {}, + columns: new Map(), scrollPos: {x: 0, y: 0}, selection: { rows: {}, @@ -34,9 +34,9 @@ export default function( case "VIEWER_STATUS": return {...state, status: action.status} case "VIEWER_COLUMNS": - return {...state, columns: {...state.columns, ...action.columns}} + return {...state, columns: new Map([...state.columns, ...action.columns])} case "VIEWER_SET_COLUMNS": - return {...state, columns: {...action.columns}} + return {...state, columns: action.columns} case "VIEWER_SCROLL": return {...state, scrollPos: action.scrollPos} case "VIEWER_SELECT": diff --git a/src/js/state/Viewer/selectors.ts b/src/js/state/Viewer/selectors.ts index 81600e1e07..8aebbd72c5 100644 --- a/src/js/state/Viewer/selectors.ts +++ b/src/js/state/Viewer/selectors.ts @@ -1,24 +1,22 @@ import {createSelector} from "reselect" - -import {State} from "../types" -import {TabState} from "../Tab/types" -import {ViewerSelection, createSelection} from "./helpers/selection" -import {ViewerSelectionData, ViewerState, ViewerColumns} from "./types" -import Tabs from "../Tabs" import {ScrollPosition} from "src/js/types" import {SearchStatus} from "src/js/types/searches" -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" +import {TabState} from "../Tab/types" +import Tabs from "../Tabs" +import {State} from "../types" +import {createSelection, ViewerSelection} from "./helpers/selection" +import {ViewerColumns, ViewerSelectionData, ViewerState} from "./types" export const getViewer = createSelector( Tabs.getActiveTab, (tab) => tab.viewer ) -export const getViewerRecords = createSelector< - State, - ViewerState, - zng.Record[] ->(getViewer, (viewer) => viewer.records) +export const getViewerRecords = createSelector( + getViewer, + (viewer) => viewer.records +) export const isFetching = (state: TabState) => state.viewer.status === "FETCHING" @@ -61,8 +59,8 @@ export const getSelection = createSelector< export const getSelectedRecords = createSelector< State, ViewerSelection, - zng.Record[], - zng.Record[] + ZedRecord[], + ZedRecord[] >(getSelection, getRecords, (selection, records) => selection.getIndices().map((index) => records[index]) ) diff --git a/src/js/state/Viewer/test.ts b/src/js/state/Viewer/test.ts index 16b6ff2df4..abafba2564 100644 --- a/src/js/state/Viewer/test.ts +++ b/src/js/state/Viewer/test.ts @@ -1,7 +1,10 @@ +import {STRING} from "test/fixtures/zjson-types" +import {ZedRecord} from "zealot/zed/data-types" +import ZedTypeDef from "zealot/zed/type-def" +import {RecordType} from "zealot/zed/zjson" +import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import Viewer from "../Viewer" -import initTestStore from "../../test/initTestStore" -import {zng} from "zealot" let store let tabId @@ -11,9 +14,21 @@ beforeEach(() => { tabId = Tabs.getActive(store.getState()) }) -const conn = new zng.Record([{name: "ts", type: "time"}], ["1"]) -const dns = new zng.Record([{name: "ts", type: "time"}], ["2"]) -const http = new zng.Record([{name: "ts", type: "time"}], ["3"]) +const type = { + kind: "record", + fields: [ + { + name: "ts", + type: { + kind: "primitive", + name: "time" + } + } + ] +} as RecordType +const conn = new ZedRecord({type, value: ["1"]}) +const dns = new ZedRecord({type, value: ["2"]}) +const http = new ZedRecord({type, value: ["3"]}) test("adding logs to the viewer", () => { const state = store.dispatchAll([ @@ -64,26 +79,34 @@ test("results limited", () => { test("update columns with same tds", () => { const cols1 = { - "9d14c2039a78d76760aae879c7fd2c82": new zng.Schema([ - {name: "hello", type: "string"} - ]) + "9d14c2039a78d76760aae879c7fd2c82": new ZedTypeDef({ + type: {kind: "typedef", name: "hello", type: STRING} + }) } const cols2 = { - "71f1b421963d31952e15edf7e3957a81": new zng.Schema([ - {name: "world", type: "string"} - ]) + "71f1b421963d31952e15edf7e3957a81": new ZedTypeDef({ + type: {kind: "typedef", name: "hello", type: STRING} + }) } const state = store.dispatchAll([ - Viewer.updateColumns(tabId, cols1), - Viewer.updateColumns(tabId, cols2) + Viewer.updateColumns(tabId, new Map(Object.entries(cols1))), + Viewer.updateColumns(tabId, new Map(Object.entries(cols2))) ]) - expect(Viewer.getColumns(state)).toEqual({ - "9d14c2039a78d76760aae879c7fd2c82": new zng.Schema([ - {name: "hello", type: "string"} - ]), - "71f1b421963d31952e15edf7e3957a81": new zng.Schema([ - {name: "world", type: "string"} + expect(Viewer.getColumns(state)).toEqual( + new Map([ + [ + "9d14c2039a78d76760aae879c7fd2c82", + new ZedTypeDef({ + type: {kind: "typedef", name: "hello", type: STRING} + }) + ], + [ + "71f1b421963d31952e15edf7e3957a81", + new ZedTypeDef({ + type: {kind: "typedef", name: "hello", type: STRING} + }) + ] ]) - }) + ) }) diff --git a/src/js/state/Viewer/types.ts b/src/js/state/Viewer/types.ts index 674729257a..53e487380c 100644 --- a/src/js/state/Viewer/types.ts +++ b/src/js/state/Viewer/types.ts @@ -1,12 +1,11 @@ -import {zng} from "zealot" +import {ZedRecord} from "zealot/zed/data-types" +import {TypeContext} from "zealot/zed/zjson" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" export type ViewerStatus = "FETCHING" | "INCOMPLETE" | "COMPLETE" | "LIMIT" -export type ViewerColumns = { - [key: string]: zng.Schema -} +export type ViewerColumns = TypeContext export type ViewerSelectionData = { rows: { [key: number]: boolean @@ -14,7 +13,7 @@ export type ViewerSelectionData = { currentRange: [number, number] } export type ViewerState = { - records: zng.Record[] + records: ZedRecord[] columns: ViewerColumns endStatus: ViewerStatus status: SearchStatus @@ -43,13 +42,13 @@ export type ViewerAction = export type VIEWER_RECORDS = { type: "VIEWER_RECORDS" - records: zng.Record[] + records: ZedRecord[] tabId: string | null | undefined } export type VIEWER_SET_RECORDS = { type: "VIEWER_SET_RECORDS" - records: zng.Record[] + records: ZedRecord[] tabId?: string } diff --git a/src/js/test/mockLogs.ts b/src/js/test/mockLogs.ts deleted file mode 100644 index 46a458ea0a..0000000000 --- a/src/js/test/mockLogs.ts +++ /dev/null @@ -1,234 +0,0 @@ -import {zng} from "zealot" - -export const dns = () => - new zng.Record( - [ - {name: "_td", type: "int"}, - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "string"}, - {name: "id.orig_h", type: "addr"}, - {name: "id.orig_p", type: "port"}, - {name: "id.resp_h", type: "addr"}, - {name: "id.resp_p", type: "port"}, - {name: "proto", type: "enum"}, - {name: "trans_id", type: "count"}, - {name: "rtt", type: "interval"}, - {name: "query", type: "string"}, - {name: "qclass", type: "count"}, - {name: "qclass_name", type: "string"}, - {name: "qtype", type: "count"}, - {name: "qtype_name", type: "string"}, - {name: "rcode", type: "count"}, - {name: "rcode_name", type: "string"}, - {name: "AA", type: "bool"}, - {name: "TC", type: "bool"}, - {name: "RD", type: "bool"}, - {name: "RA", type: "bool"}, - {name: "Z", type: "count"}, - {name: "rejected", type: "bool"} - ], - [ - "8", - "dns", - "1425565514.419939", - "Cum4LVba3W3KNG6qa", - "fe80::eef4:bbff:fe51:89ec", - "5353", - "ff02::fb", - "5353", - "udp", - "0", - "0.119484", - "_ipp._tcp.local", - "1", - "C_INTERNET", - "12", - "PTR", - "0", - "NOERROR", - "T", - "F", - "F", - "F", - "0", - "_workstation._tcp.local,sniffer [ec:f4:bb:51:89:ec]._workstation._tcp.local", - "4500.000000,4500.000000", - "F" - ] - ) - -export const conn = () => - new zng.Record( - [ - {name: "_td", type: "int"}, - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "string"}, - { - name: "id", - type: "record", - of: [ - {name: "orig_h", type: "addr"}, - {name: "orig_p", type: "port"}, - {name: "resp_h", type: "addr"}, - {name: "resp_p", type: "port"} - ] - }, - {name: "proto", type: "enum"}, - {name: "service", type: "string"}, - {name: "duration", type: "interval"}, - {name: "orig_bytes", type: "count"}, - {name: "resp_bytes", type: "count"}, - {name: "conn_state", type: "string"}, - {name: "local_orig", type: "bool"}, - {name: "local_resp", type: "bool"}, - {name: "missed_bytes", type: "count"}, - {name: "history", type: "string"}, - {name: "orig_pkts", type: "count"}, - {name: "orig_ip_bytes", type: "count"}, - {name: "resp_pkts", type: "count"}, - {name: "resp_ip_bytes", type: "count"} - ], - [ - "0", - "conn", - "1425612054.369843", - "Cynwae4qh1GxM82hQ2", - ["192.168.0.50", "1900", "239.255.255.250", "1900"], - "udp", - "-", - "2.000293", - "282", - "0", - "S0", - "-", - "-", - "0", - "D", - "3", - "366", - "0", - "0", - "(empty)" - ] - ) - -export const http = () => - new zng.Record( - [ - {name: "_td", type: "int"}, - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "uid", type: "string"}, - {name: "id.orig_h", type: "addr"}, - {name: "id.orig_p", type: "port"}, - {name: "id.resp_h", type: "addr"}, - {name: "id.resp_p", type: "port"}, - {name: "trans_depth", type: "count"}, - {name: "method", type: "string"}, - {name: "host", type: "string"}, - {name: "uri", type: "string"}, - {name: "referrer", type: "string"}, - {name: "version", type: "string"}, - {name: "user_agent", type: "string"}, - {name: "request_body_len", type: "count"}, - {name: "response_body_len", type: "count"}, - {name: "status_code", type: "count"}, - {name: "status_msg", type: "string"}, - {name: "info_code", type: "count"}, - {name: "info_msg", type: "string"}, - {name: "username", type: "string"}, - {name: "password", type: "string"} - ], - [ - "10", - "http", - "1425567042.047800", - "CBQjjR1ZrY6LmUfzX5", - "192.168.0.51", - "60677", - "91.189.89.240", - "80", - "1", - "GET", - "start.ubuntu.com", - "/14.10/Google/?sourceid=hp", - "-", - "1.0", - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0", - "0", - "6005", - "200", - "OK", - "-", - "-", - "(empty)", - "-", - "-", - "-", - "-", - "-", - "-", - "FUPWLQXTNsTNvf33", - "-", - "text/html" - ] - ) - -export const files = () => - new zng.Record( - [ - {name: "_td", type: "int"}, - {name: "_path", type: "string"}, - {name: "ts", type: "time"}, - {name: "fuid", type: "string"}, - {name: "depth", type: "count"}, - {name: "mime_type", type: "string"}, - {name: "filename", type: "string"}, - {name: "duration", type: "interval"}, - {name: "local_orig", type: "bool"}, - {name: "is_orig", type: "bool"}, - {name: "seen_bytes", type: "count"}, - {name: "total_bytes", type: "count"}, - {name: "missing_bytes", type: "count"}, - {name: "overflow_bytes", type: "count"}, - {name: "timedout", type: "bool"}, - {name: "parent_fuid", type: "string"}, - {name: "md5", type: "string"}, - {name: "sha1", type: "string"}, - {name: "sha256", type: "string"}, - {name: "extracted", type: "string"}, - {name: "extracted_cutoff", type: "bool"}, - {name: "extracted_size", type: "count"} - ], - [ - "2", - "files", - "1425567042.160756", - "FUPWLQXTNsTNvf33", - "91.189.89.240", - "192.168.0.51", - "CBQjjR1ZrY6LmUfzX5", - "HTTP", - "0", - "(empty)", - "text/html", - "-", - "0.000018", - "-", - "F", - "6005", - "-", - "0", - "0", - "F", - "-", - "-", - "-", - "-", - "-", - "-", - "-" - ] - ) diff --git a/src/js/types/index.ts b/src/js/types/index.ts index 4777d964d2..10f7d3548e 100644 --- a/src/js/types/index.ts +++ b/src/js/types/index.ts @@ -2,7 +2,7 @@ import {SpanArgs} from "../state/Search/types" import {TimeUnit} from "../lib" import AppError from "../models/AppError" import {MenuItemConstructorOptions} from "electron" -import {zng} from "zealot" +import {ZedRecord, ZedField} from "zealot/zed/data-types" export type Notification = | AppError @@ -60,12 +60,12 @@ export type LogCorrelations = { } export type RelatedLogs = { - [key: string]: zng.Record[] + [key: string]: ZedRecord[] } export type RightClickBuilder = ( - field: zng.Field, - record: zng.Record, + field: ZedField, + record: ZedRecord, compound: boolean ) => MenuItemConstructorOptions[] diff --git a/src/js/zql/toZql.ts b/src/js/zql/toZql.ts index d53bcc8249..37f4cf644e 100644 --- a/src/js/zql/toZql.ts +++ b/src/js/zql/toZql.ts @@ -1,8 +1,8 @@ -import {zng} from "zealot" import isString from "lodash/isString" +import {ZedPrimitive} from "zealot/zed/data-types" export function toZql(object: unknown): string { - if (object instanceof zng.Primitive) return toZqlZngPrimitive(object) + if (object instanceof ZedPrimitive) return toZqlZngPrimitive(object) if (isString(object)) return toZqlString(object) if (object instanceof Date) return toZqlDate(object) if (typeof object === "boolean") return toZqlBool(object) @@ -29,7 +29,8 @@ function toZqlBool(bool: boolean) { return bool ? "true" : "false" } -function toZqlZngPrimitive(data: zng.Primitive) { - if (data.getType() === "string") return toZqlString(data.getValue()) - throw new Error(`Can't convert Zng Type: ${data.getType()} to zql`) +function toZqlZngPrimitive(data: ZedPrimitive) { + if (data.kind === "string" || data.kind === "bstring") + return toZqlString(data.toString()) + throw new Error(`Can't convert Zng Type: ${data.kind} to zql`) } diff --git a/test/api/scripts/search.ts b/test/api/scripts/search.ts index a3cefc43d1..976b3926d7 100644 --- a/test/api/scripts/search.ts +++ b/test/api/scripts/search.ts @@ -3,7 +3,7 @@ import { fromFileUrl, dirname } from "https://deno.land/std@0.70.0/path/mod.ts" -import {withZqd} from "../test_api/helper/test_api.ts" +import {withZqd} from "../helper/test_api.ts" const DIR = dirname(fromFileUrl(import.meta.url)) const FILE = join(DIR, "../test_api/data/sample.tsv") diff --git a/test/fixtures/zjson-types.ts b/test/fixtures/zjson-types.ts new file mode 100644 index 0000000000..77953a59fa --- /dev/null +++ b/test/fixtures/zjson-types.ts @@ -0,0 +1,7 @@ +import {PrimitiveType} from "zealot/zed/zjson" + +export const STRING = {kind: "primitive", name: "string"} as PrimitiveType +export const TIME = {kind: "primitive", name: "time"} as PrimitiveType +export const COUNT = {kind: "primitive", name: "count"} as PrimitiveType +export const INTERVAL = {kind: "primitive", name: "interval"} as PrimitiveType +export const IP = {kind: "primitive", name: "ip"} as PrimitiveType diff --git a/test/responses/index.ts b/test/responses/index.ts new file mode 100644 index 0000000000..4628c1464a --- /dev/null +++ b/test/responses/index.ts @@ -0,0 +1,5 @@ +const _thing = { + input: "test/data/sample.zson", + query: "* | count()", + output: "test/responses/sample-count.response" +} diff --git a/zealot/config/search_args.ts b/zealot/config/search_args.ts index 87e067f72d..eefafb0521 100644 --- a/zealot/config/search_args.ts +++ b/zealot/config/search_args.ts @@ -1,5 +1,4 @@ import {SearchArgs} from "../types" -import {zngToZeek} from "../enhancers/mod" export function getDefaultSearchArgs(): SearchArgs { return { @@ -8,6 +7,6 @@ export function getDefaultSearchArgs(): SearchArgs { spaceId: "default", format: "zjson", controlMessages: true, - enhancers: [zngToZeek] + enhancers: [] } } diff --git a/zealot/enhancers/mod.ts b/zealot/enhancers/mod.ts deleted file mode 100644 index bb511262bd..0000000000 --- a/zealot/enhancers/mod.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./zngToZeek" diff --git a/zealot/enhancers/zngToZeek.ts b/zealot/enhancers/zngToZeek.ts deleted file mode 100644 index 058e2928f7..0000000000 --- a/zealot/enhancers/zngToZeek.ts +++ /dev/null @@ -1,74 +0,0 @@ -import {ZealotPayload} from "../types" -import * as zjson from "../zjson" - -function fail(t: any): never { - throw new Error("Unknown zjson Type: " + JSON.stringify(t)) -} - -function getZeekPrimitive(type: zjson.Primitive): zjson.Primitive { - switch (type) { - case "byte": - case "int16": - case "int32": - case "int64": - case "uint16": - case "uint32": - return "int" - case "uint64": - return "count" - case "float64": - return "double" - case "ip": - return "addr" - case "net": - return "subnet" - case "duration": - return "interval" - case "bstring": - return "string" - case "zenum": - return "enum" - default: - return type - } -} - -function replaceColumn(c: zjson.Column): zjson.Column { - if (c.type == "array" || c.type == "set") { - return {...c, of: replaceTypes(c.of)} - } else if (c.type == "union") { - return {...c, of: c.of.map(replaceTypes)} - } else if (c.type == "record") { - return {...c, of: c.of.map(replaceColumn)} - } else { - return {...c, type: getZeekPrimitive(c.type)} - } -} - -export function replaceTypes(t: zjson.Type): zjson.Type { - if (typeof t == "string") { - return getZeekPrimitive(t) - } else if (t.type == "array" || t.type == "set") { - return {...t, of: t.of = replaceTypes(t.of)} - } else if (t.type == "union") { - return {...t, of: t.of.map(replaceTypes)} - } else if (t.type == "record") { - return {...t, of: t.of.map(replaceColumn)} - } - fail(t) -} - -function replaceSchema(r: zjson.Item) { - if (r.schema) r.schema.of = r.schema.of.map(replaceColumn) - return r -} - -export function zngToZeek() { - return (p: ZealotPayload) => { - if (p.type == "SearchRecords") { - return {...p, records: p.records.map(replaceSchema)} - } else { - return p - } - } -} diff --git a/zealot/enhancers/zngToZeek_test.ts b/zealot/enhancers/zngToZeek_test.ts deleted file mode 100644 index 2377741ba1..0000000000 --- a/zealot/enhancers/zngToZeek_test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import {replaceTypes} from "./zngToZeek" -import * as zjson from "../zjson" - -function assertConversion(from: zjson.Type, to: zjson.Type) { - expect(replaceTypes(from)).toEqual(to) -} - -test("convert to int", () => { - assertConversion( - { - type: "record", - of: [ - {name: "one", type: "byte"}, - {name: "two", type: "int16"}, - {name: "three", type: "int32"}, - {name: "four", type: "int64"}, - {name: "fiv", type: "uint16"}, - {name: "six", type: "uint32"} - ] - }, - { - type: "record", - of: [ - {name: "one", type: "int"}, - {name: "two", type: "int"}, - {name: "three", type: "int"}, - {name: "four", type: "int"}, - {name: "fiv", type: "int"}, - {name: "six", type: "int"} - ] - } - ) -}) - -test("convert to count", () => { - assertConversion({type: "array", of: "uint64"}, {type: "array", of: "count"}) -}) - -test("convert to double", () => { - assertConversion( - {type: "union", of: ["float64", "ip"]}, - {type: "union", of: ["double", "addr"]} - ) -}) - -test("convert to addr", () => { - assertConversion("ip", "addr") -}) - -test("convert to subnet", () => { - assertConversion({type: "set", of: "net"}, {type: "set", of: "subnet"}) -}) - -test("convert to interval", () => { - assertConversion("duration", "interval") -}) - -test("convert to string", () => { - assertConversion("bstring", "string") -}) - -test("convert to enum", () => { - assertConversion("zenum", "enum") -}) - -test("all other types passthrough", () => { - assertConversion("port", "port") -}) - -test("nested types", () => { - assertConversion( - { - type: "record", - of: [ - { - name: "id", - type: "record", - of: [ - {name: "orig_h", type: "ip"}, - {name: "resp_h", type: "ip"} - ] - }, - {name: "orig_bytes", type: "uint64"} - ] - }, - { - type: "record", - of: [ - { - name: "id", - type: "record", - of: [ - {name: "orig_h", type: "addr"}, - {name: "resp_h", type: "addr"} - ] - }, - {name: "orig_bytes", type: "count"} - ] - } - ) -}) diff --git a/zealot/fetcher/records_callback.ts b/zealot/fetcher/records_callback.ts index bd993bad09..5db93a7b52 100644 --- a/zealot/fetcher/records_callback.ts +++ b/zealot/fetcher/records_callback.ts @@ -1,62 +1,49 @@ -import * as zjson from "../zjson" -import * as zng from "../zng" +import {ZedRecord} from "zealot/zed/data-types" +import {decode, TypeContext} from "zealot/zed/zjson" import * as zqd from "../zqd" export interface RecordsCallbackArgs { channel: number - schemas: SchemaMap - newRecords: zng.Record[] - allRecords: zng.Record[] + schemas: TypeContext + context: TypeContext + newRows: ZedRecord[] + rows: ZedRecord[] } -type SchemaMap = Map + type ChannelMap = Map -type Channel = {records: zng.Record[]; schemas: SchemaMap} +type Channel = {rows: ZedRecord[]; schemas: TypeContext; context: TypeContext} type RecordsCallback = (args: RecordsCallbackArgs) => void type PayloadCallback = (payload: zqd.SearchRecords) => void function getChannel(id: number, channels: ChannelMap): Channel { if (!channels.has(id)) { channels.set(id, { - records: [], - schemas: new Map() + rows: [], + schemas: new Map(), + context: new Map() } as Channel) } return channels.get(id) as Channel } -function createRecords( - nextRecords: zjson.Items, - prevSchemas: SchemaMap -): Channel { - const schemas = new Map(prevSchemas) - const records = nextRecords.map((r) => { - if (r.schema) { - schemas.set(r.id, new zng.Schema(r.schema.of)) - } - const schema = schemas.get(r.id) - if (!schema) throw new Error("No Schema Present for ID: " + r.id) - return new zng.Record(schema.columns, r.values) - }) - - return {records, schemas} -} - export function createRecordsCallback(cb: RecordsCallback): PayloadCallback { let channels = new Map() return ({channel_id: id, records}: zqd.SearchRecords) => { const prev = getChannel(id, channels) - const next = createRecords(records, prev.schemas) + const next = decode(records, prev.context, prev.schemas) const chan = { - records: [...prev.records, ...next.records], - schemas: next.schemas + rows: [...prev.rows, ...next.rows], + schemas: next.schemas, + context: next.context } channels.set(id, chan) cb({ channel: id, schemas: chan.schemas, - newRecords: next.records, - allRecords: chan.records + context: chan.context, + newRows: next.rows, + rows: chan.rows }) } } diff --git a/zealot/fetcher/stream.ts b/zealot/fetcher/stream.ts index cc0ff92ea6..ded8dc371a 100644 --- a/zealot/fetcher/stream.ts +++ b/zealot/fetcher/stream.ts @@ -1,7 +1,7 @@ import {createCallbacks} from "./callbacks" import {ZCallbacks, ZIterator} from "../types" -import * as zjson from "../zjson" -import * as zng from "../zng" +import {decode, StreamObject} from "zealot/zed/zjson" +import {ZedRecord} from "zealot/zed/data-types" async function emitCallbacks(iterator: ZIterator, callbacks: ZCallbacks) { try { @@ -27,14 +27,14 @@ export function createStream( } return all }, - records: async (): Promise => { - let records: zjson.Items = [] + records: async (): Promise => { + let records: StreamObject[] = [] for await (let payload of iterator) { if (payload.type === "SearchRecords") { records = records.concat(payload.records) } } - return zng.createRecords(records) + return decode(records).rows }, callbacks: () => { const cbs = createCallbacks() diff --git a/zealot/zealot_mock.ts b/zealot/zealot_mock.ts index b69be081ac..91bbcf5f6f 100644 --- a/zealot/zealot_mock.ts +++ b/zealot/zealot_mock.ts @@ -3,7 +3,6 @@ import {FetchArgs} from "./fetcher/fetcher" import {createStream} from "./fetcher/stream" import {createError} from "./util/error" import {Zealot, ZealotPayload} from "./types" -import {zngToZeek} from "./enhancers/mod" type StubMode = "always" | "once" type RespWrap = typeof stream | typeof promise @@ -27,10 +26,9 @@ function promise(response: any) { } function stream(response: ZealotPayload[]) { - const enhance = zngToZeek() async function* iterator() { if (response) { - for (const payload of response) yield enhance(payload) + for (const payload of response) yield payload } } diff --git a/zealot/zed/construct.ts b/zealot/zed/construct.ts new file mode 100644 index 0000000000..b615ff2af8 --- /dev/null +++ b/zealot/zed/construct.ts @@ -0,0 +1,48 @@ +import { + ZedRecord, + ZedPrimitive, + ZedArray, + ZedSet, + ZedUnion, + ZedEnum, + ZedMap +} from "./data-types" +import ZedTypeDef from "./type-def" +import {TypeContext, Value, TypeName, Type} from "./zjson" + +export function construct( + context: TypeContext, + type: Type, + value: Value, + name?: TypeName +) { + switch (type.kind) { + case "record": + return new ZedRecord({type, value: value as Value[], context, name}) + case "primitive": + return new ZedPrimitive({type, value: value as string, name}) + case "array": + return new ZedArray({type, value: value as Value[], name, context}) + case "set": + return new ZedSet({type, value: value as Value[], name, context}) + case "union": + return new ZedUnion({type, value: value as Value[], name}) + case "enum": + return new ZedEnum({type, value: value as string, name}) + case "map": + return new ZedMap({type, value: value as Value[], name}) + case "typename": + var def = context.get(type.name) + if (!def) { + console.log("No typedef in context for", type) + console.log(context) + } + return construct(context, def.innerType, value, type.name) + case "typedef": + var def = new ZedTypeDef({type}) + context.set(def.name, def) + return construct(context, def.innerType, value, def.name) + default: + throw new Error(`Unknown ZJSON Type: ${JSON.stringify(type)}`) + } +} diff --git a/zealot/zed/count_by.json b/zealot/zed/count_by.json deleted file mode 100644 index 1b0cd2542f..0000000000 --- a/zealot/zed/count_by.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "schema": "26", - "types": [ - { - "kind": "typedef", - "name": "26", - "type": { - "kind": "record", - "fields": [ - { - "name": "typeof", - "type": { - "kind": "primitive", - "name": "type" - } - }, - { - "name": "count", - "type": { - "kind": "primitive", - "name": "uint64" - } - } - ] - } - }, - { - "kind": "typedef", - "name": "person_schema", - "type": { - "kind": "record", - "fields": [ - { - "name": "first_name", - "type": { - "kind": "primitive", - "name": "string" - } - } - ] - } - } - ], - "values": [ - "24", - "1" - ] -} -{ - "schema": "26", - "types": [ - { - "kind": "typedef", - "name": "25", - "type": { - "kind": "record", - "fields": [ - { - "name": "score", - "type": { - "kind": "primitive", - "name": "int64" - } - } - ] - } - } - ], - "values": [ - "25", - "1" - ] -} diff --git a/zealot/zed/data-types.ts b/zealot/zed/data-types.ts new file mode 100644 index 0000000000..b076d1656b --- /dev/null +++ b/zealot/zed/data-types.ts @@ -0,0 +1,304 @@ +import {construct} from "./construct" +import { + ArrayType, + EnumType, + MapType, + PrimitiveType, + RecordFieldType, + RecordType, + SetType, + Type, + TypeContext, + TypeName, + TypeNameType, + UnionType, + Value +} from "./zjson" + +export type ZedData = + | ZedPrimitive + | ZedRecord + | ZedArray + | ZedSet + | ZedUnion + | ZedEnum + | ZedMap + +/** + * Base Class for all Zed Data Types + */ +export type ZedTypeArgs = { + type: T + value: V + name?: TypeName + context?: TypeContext +} + +export class ZedType { + public _type: T + public _value: V + public _name?: TypeName + + constructor(args: ZedTypeArgs) { + this._type = args.type + this._value = args.value + this._name = args.name + } + + get kind(): string { + return this._name || this._type.kind + } + + toString() { + return this._value ? this._value.toString() : "" + } + + isUnset() { + return this._value === null + } + + serialize() { + return { + name: this._name, + type: this._type, + value: this._value + } + } +} + +/** + * Holds a name and a data field + */ + +export type ZedFieldSpec = { + name: string + data: { + name?: string + type: Type + value: Value + } +} +export class ZedField { + name: string + data: ZedData + + static deserialize({name, data}: ZedFieldSpec) { + return new ZedField({ + name, + data: construct(new Map(), data.type, data.value, data.name) + }) + } + + constructor({name, data}: {name: string; data: ZedData}) { + this.name = name + this.data = data + } + + serialize(): ZedFieldSpec { + return { + name: this.name, + data: this.data.serialize() + } + } +} + +export class ZedPrimitive extends ZedType { + get kind() { + return this._name || this._type.name + } + + toDate() { + if (this._value === null || this._type.name != "time") { + throw new Error(`Cannot make type: ${this._type.name} into a Date`) + } + return new Date(+this._value * 1000) + } + + toFloat() { + if (this._value === null) { + return 0 + } + try { + return parseFloat(this._value) + } catch { + throw new Error(`Cannot make type: ${this._type.name} into a Float`) + } + } + + toInt() { + if (this._value === null) throw new Error("value is unset") + const int = parseInt(this._value) + if (isNaN(int)) { + throw new Error(`Cannot make type: ${this._type.name} into an Integer`) + } + return int + } +} + +export class ZedArray extends ZedType { + items: ZedData[] + + constructor(args: { + type: ArrayType + value: Value[] + context: TypeContext + name: TypeName + }) { + super(args) + this.items = (this._value || []).map((value) => + construct(args.context, args.type.type, value) + ) + } + + get innerType() { + return this._type.type + } +} + +export class ZedSet extends ZedType { + items: ZedData[] + + constructor(args: { + type: SetType + value: Value[] + context: TypeContext + name: TypeName + }) { + super(args) + this.items = (this._value || []).map((value) => + construct(args.context, args.type.type, value) + ) + } + + get innerType() { + return this._type.type + } +} + +// Need use cases +export class ZedUnion extends ZedType {} +// Need use cases +export class ZedEnum extends ZedType {} +// Need use cases +export class ZedMap extends ZedType {} +// Need use cases +export class ZedTypeName extends ZedType {} + +class UnknownColumnError extends Error { + constructor(unknown: string, names: string[]) { + const available = names.map((n) => `"${n}"`).join(", ") + super(`"${unknown}" not present in [${available}]`) + } +} + +export type ZedRecordSpec = { + type: RecordType + value: Value[] + name?: TypeName +} + +export class ZedRecord extends ZedType { + fields: ZedField[] + context: TypeContext + + static of(fields: RecordFieldType[], values: Value[]) { + return new ZedRecord({type: {kind: "record", fields}, value: values}) + } + + static deserialize(args: ZedRecordSpec) { + console.log(args) + return new ZedRecord(args) + } + + constructor(args: ZedTypeArgs) { + super(args) + this.context = args.context || new Map() + this.fields = this._value.map((value, index) => { + const {name, type} = this._type.fields[index] + return new ZedField({ + name, + data: construct(args.context || new Map(), type, value) + }) + }) + } + + [Symbol.iterator]() { + let index = 0 + const next = () => + index < this._type.fields.length + ? {done: false, value: this.at(index++)} + : {done: true, value: undefined} + + return {next} + } + + get columns() { + return this.fields.map((f) => f.name) + } + + at(index: number) { + return this.fields[index].data + } + + get(name: string) { + return this.getField(name).data + } + + try(name: string) { + try { + return this.get(name) + } catch { + return null + } + } + + private _getField(name: string) { + if (this.isUnset()) throw new Error("Record is unset") + const field = this.fields.find((f) => f.name == name) + if (!field) throw new UnknownColumnError(name, this.columns) + return field + } + + getField(name: string) { + return name + .split(".") + .reduce((field: ZedField, namePart: string) => { + if (field.data instanceof ZedRecord) { + return new ZedField({name, data: field.data._getField(namePart).data}) + } else { + throw new Error("Dot syntax is only for nested records") + } + }, new ZedField({name: "", data: this})) + } + + tryField(name: string) { + try { + return this.getField(name) + } catch { + return null + } + } + + has(name: string) { + return this.columns.includes(name) + } + + flatten(prefix = ""): ZedRecord { + let fields = [] + let vals = [] + + this._type.fields.forEach((field, index) => { + if (field.type.kind === "record") { + const nested = (this.at(index) as ZedRecord).flatten(field.name + ".") + fields = fields.concat(nested._type.fields) + vals = vals.concat(nested._value) + } else { + fields.push({...field, name: prefix + field.name}) + // For an unset record, supply an unset value for each column. + vals.push(this.isUnset() ? null : this._value![index]) + } + }) + const type = {kind: "record", fields} as RecordType + return new ZedRecord({type, value: vals, context: this.context}) + } +} diff --git a/zealot/zed/star.json b/zealot/zed/star.json deleted file mode 100644 index 8d62f1d020..0000000000 --- a/zealot/zed/star.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "schema": "person_schema", - "types": [ - { - "kind": "typedef", - "name": "person_schema", - "type": { - "kind": "record", - "fields": [ - { - "name": "first_name", - "type": { - "kind": "primitive", - "name": "string" - } - } - ] - } - } - ], - "values": [ - "James" - ] -} -{ - "schema": "25", - "types": [ - { - "kind": "typedef", - "name": "25", - "type": { - "kind": "record", - "fields": [ - { - "name": "score", - "type": { - "kind": "primitive", - "name": "int64" - } - } - ] - } - } - ], - "values": [ - "100" - ] -} diff --git a/zealot/zed/type-def.ts b/zealot/zed/type-def.ts new file mode 100644 index 0000000000..116955237e --- /dev/null +++ b/zealot/zed/type-def.ts @@ -0,0 +1,41 @@ +import {RecordFieldType, TypeDefType} from "./zjson" + +export default class ZedTypeDef { + _type: TypeDefType + + constructor({type}: {type: TypeDefType}) { + this._type = type + } + + get name() { + return this._type.name + } + + get innerType() { + return this._type.type + } + + flatten(): ZedTypeDef { + const inner = this.innerType + if (inner.kind !== "record") return this + + const flat = ( + fields: RecordFieldType[], + prefix = "" + ): RecordFieldType[] => { + return fields.flatMap((field) => { + const name = prefix + field.name + if (field.type.kind == "record") { + return flat(field.type.fields, name + ".") + } else { + return {...field, name} + } + }) + } + const fields = flat(inner.fields) + + return new ZedTypeDef({ + type: {name: this.name, kind: "typedef", type: {kind: "record", fields}} + }) + } +} diff --git a/zealot/zed/zjson.test.ts b/zealot/zed/zjson.test.ts index f90ea7b2f4..d2ea424e1f 100644 --- a/zealot/zed/zjson.test.ts +++ b/zealot/zed/zjson.test.ts @@ -1,5 +1,5 @@ import {execSync} from "child_process" -import {parse} from "./zjson" +import {decode} from "./zjson" function zq(q, file) { const zed = "/Users/jkerr/tools/go/bin/zed" @@ -12,12 +12,6 @@ function zq(q, file) { test("can correlate?", () => { const file = "test/data/count-by-typeof.zson" - const list = parse(zq("*", file)) - const chart = parse(zq("count() by typeof(.)", file)) - - // I need the name of the typedef - console.log(list.zjson.map((o) => o.types)) - console.log(chart.zjson.map((o) => o.types)) - // console.log(list.rows.map((r) => r._name)) - // console.log(chart.rows.map((r) => r.fields[0])) + const _list = decode(zq("*", file)) + const _chart = decode(zq("count() by typeof(.)", file)) }) diff --git a/zealot/zed/zjson.ts b/zealot/zed/zjson.ts index 9a8fa3a577..1e276e37ac 100644 --- a/zealot/zed/zjson.ts +++ b/zealot/zed/zjson.ts @@ -1,60 +1,64 @@ -type ID = string | number +import {construct} from "./construct" +import {ZedRecord} from "./data-types" +import ZedTypeDef from "./type-def" -type Value = string | null | Value[] +type ID = string -type Field = { +export type Value = string | null | Value[] + +export type RecordFieldType = { name: string type: Type } -type PrimitiveType = { +export type PrimitiveType = { kind: "primitive" name: string } -type RecordType = { +export type RecordType = { kind: "record" - fields: Field[] + fields: RecordFieldType[] } -type ArrayType = { +export type ArrayType = { kind: "array" type: Type } -type SetType = { +export type SetType = { kind: "set" type: Type } -type UnionType = { +export type UnionType = { kind: "union" types: Type[] } -type EnumType = { +export type EnumType = { kind: "enum" symbols: string[] } -type MapType = { +export type MapType = { kind: "map" key_type: Type val_type: Type } -type TypeDefType = { +export type TypeDefType = { kind: "typedef" name: ID type: Type } -type TypeNameType = { +export type TypeNameType = { kind: "typename" name: ID } -type Type = +export type Type = | PrimitiveType | RecordType | ArrayType @@ -65,206 +69,47 @@ type Type = | TypeDefType | TypeNameType -type StreamObject = { +export type StreamObject = { schema: ID values: Value[] types: TypeDefType[] } -type TypeName = string | number - -class ZedType { - protected _type: T - protected _value: V - protected _name?: TypeName - - constructor(args: {type: T; value: V; name?: TypeName}) { - this._type = args.type - this._value = args.value - this._name = args.name - } - - missing() { - return false - } - - toString() { - return this._value ? this._value.toString() : "" - } -} - -class Missing { - constructor(readonly type: typeof ZedType, readonly name: string) { - return new Proxy(this, { - get(target, prop, receiver) { - if (target[prop]) { - return Reflect.get(target, prop, receiver) - } else { - return receiver - } - } - }) - } - - toString() { - return `Missing: "${this.name}" in ${this.type}` - } - - missing() { - return true - } -} - -type ZedData = - | ZedPrimitive - | ZedRecord - | ZedTypeDef - | ZedArray - | ZedSet - | ZedUnion - | ZedEnum - | ZedMap - | ZedTypeName +export type TypeName = string -class ZedField { - name: string - data: ZedData +export type TypeContext = Map - constructor({name, data}: {name: string; data: ZedData}) { - this.name = name - this.data = data - } -} - -function withQuickAccess(record, accessor) { - return new Proxy(record, { - get(target, property, receiver) { - if (target[property]) { - return Reflect.get(target, property, receiver) - } - const data = accessor(target, property) - return data ? data : new Missing(target, property.toString()) - } - }) -} - -class ZedPrimitive extends ZedType { - constructor(args) { - super(args) - return withQuickAccess(this, () => false) - } -} -class ZedTypeDef extends ZedType { - get name() { - return this._type.name - } -} -class ZedArray extends ZedType {} -class ZedSet extends ZedType {} -class ZedUnion extends ZedType {} -class ZedEnum extends ZedType {} -class ZedMap extends ZedType {} -class ZedTypeName extends ZedType {} - -class ZedRecord extends ZedType { - fields: ZedField[] - constructor(args) { - super(args) - this.fields = this._value.map((value, index) => { - const {name, type} = this._type.fields[index] - return new ZedField({ - name, - data: construct(args.context, type, value, args.name) - }) - }) - return withQuickAccess( - this, - (record, prop) => record.fields.find((f) => f.name === prop)?.data - ) - } - - [Symbol.iterator]() { - let index = 0 - const next = () => - index < this._type.fields.length - ? {done: false, value: this.at(index++)} - : {done: true, value: undefined} - - return {next} - } - - at(index: number) { - return this.fields[index] - } - - get columns() { - return this.fields.map((f) => f.name) - } -} - -class Rows extends Array { - first() { - return this[0] - } - last() { - return this[this.length - 1] - } -} - -function construct( - context: TypeContext, - type: Type, - value: Value, - name?: TypeName -) { - switch (type.kind) { - case "record": - return new ZedRecord({type, value, context, name}) - case "primitive": - return new ZedPrimitive({type, value: value as string, name}) - case "array": - return new ZedArray({type, value: value as Value[], name}) - case "set": - return new ZedSet({type, value: value as Value[], name}) - case "union": - return new ZedUnion({type, value: value as Value[], name}) - case "enum": - return new ZedEnum({type, value: value as string, name}) - case "map": - return new ZedMap({type, value: value as Value[], name}) - case "typename": - var typedef = context.get(type.name) - return construct(context, typedef.type, value, typedef.name) - case "typedef": - context.set(type.name, type) - return construct(context, type.type, value, type.name) - default: - throw new Error(`Unknown ZJSON Type: ${JSON.stringify(type)}`) - } +export type DecodedZJSON = { + rows: ZedRecord[] + schemas: TypeContext + context: TypeContext } -type TypeContext = Map +export function decode( + zjson: StreamObject[], + context: TypeContext = new Map(), + schemas: TypeContext = new Map() +): DecodedZJSON { + const rows = [] -export function parse(zjson: StreamObject[]) { - const rows = new Rows() - const context: TypeContext = new Map() - const schemas = new Map() + for (const {schema, types, values} of zjson) { + // Add all the types to the context + types && + types.forEach((typedef) => + context.set(typedef.name, new ZedTypeDef({type: typedef})) + ) - for (const object of zjson) { - if (object.schema) schemas.set(object.schema, true) - if (object.types) { - object.types.forEach((typedef) => context.set(typedef.name, typedef)) - } - const typedef = context.get(object.schema) + // Add the root record type to the schemas + const typedef = context.get(schema) + if (!schemas.has(schema)) schemas.set(schema, typedef) - const type = typedef.type + // Construct the row and save it + const type = typedef.innerType const name = typedef.name - const value = object.values - rows.push(construct(context, type, value, name)) + rows.push(construct(context, type, values, name)) } return { - zjson, rows, schemas, context diff --git a/zealot/zng/types/record.ts b/zealot/zng/types/record.ts index 5e18b82f21..224e87f7c6 100644 --- a/zealot/zng/types/record.ts +++ b/zealot/zng/types/record.ts @@ -10,7 +10,7 @@ class UnknownColumnError extends Error { } } -export type SerializedRecord = { +export type SerialiZedRecord = { type: zjson.Record value: zjson.Value[] | null } @@ -21,7 +21,7 @@ export class Record implements ZngClass { readonly value: zjson.Value[] | null ) {} - static deserialize({type, value}: SerializedRecord): Record { + static deserialize({type, value}: SerialiZedRecord): Record { return new Record(type.of, value) } @@ -127,7 +127,7 @@ export class Record implements ZngClass { return new Record(cols, vals) } - serialize(): SerializedRecord { + serialize(): SerialiZedRecord { return { type: { type: "record", diff --git a/zealot/zqd.ts b/zealot/zqd.ts index 6810bb1824..601b57f568 100644 --- a/zealot/zqd.ts +++ b/zealot/zqd.ts @@ -1,5 +1,5 @@ import {Ts} from "./types" -import * as zjson from "./zjson" +import * as zjson from "./zed/zjson" export type Payload = | SearchRecords @@ -12,7 +12,7 @@ export type Payload = export type SearchRecords = { type: "SearchRecords" - records: zjson.Items + records: zjson.StreamObject[] channel_id: number } From d2b3edff48b21c381d90766e42c3ebd7b8ce5121 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 19 Apr 2021 14:08:36 -0700 Subject: [PATCH 04/41] Zed is done --- zealot/zed/array.ts | 22 +++ zealot/zed/construct.ts | 48 ------- zealot/zed/data-types.ts | 304 --------------------------------------- zealot/zed/enum.ts | 20 +++ zealot/zed/field.ts | 19 +++ zealot/zed/index.ts | 28 ++++ zealot/zed/json.ts | 76 ++++++++++ zealot/zed/map.ts | 33 +++++ zealot/zed/primitive.ts | 50 +++++++ zealot/zed/record.ts | 105 ++++++++++++++ zealot/zed/set.ts | 22 +++ zealot/zed/union.ts | 22 +++ zealot/zed/zjson.test.ts | 10 +- zealot/zed/zjson.ts | 126 ++++++++++++++-- 14 files changed, 521 insertions(+), 364 deletions(-) create mode 100644 zealot/zed/array.ts delete mode 100644 zealot/zed/construct.ts delete mode 100644 zealot/zed/data-types.ts create mode 100644 zealot/zed/enum.ts create mode 100644 zealot/zed/field.ts create mode 100644 zealot/zed/index.ts create mode 100644 zealot/zed/json.ts create mode 100644 zealot/zed/map.ts create mode 100644 zealot/zed/primitive.ts create mode 100644 zealot/zed/record.ts create mode 100644 zealot/zed/set.ts create mode 100644 zealot/zed/union.ts diff --git a/zealot/zed/array.ts b/zealot/zed/array.ts new file mode 100644 index 0000000000..ef6476ed17 --- /dev/null +++ b/zealot/zed/array.ts @@ -0,0 +1,22 @@ +import {ZedData} from "./index" + +export class ZedArray { + type: string + items: ZedData[] + typeName?: string + + constructor(args: {type: string; typeName?: string; items: ZedData[]}) { + this.type = args.type + this.typeName = args.typeName + this.items = args.items + } + + serialize() { + return { + kind: "array", + type: this.type, + typeName: this.typeName, + items: this.items.map((item) => item.serialize()) + } + } +} diff --git a/zealot/zed/construct.ts b/zealot/zed/construct.ts deleted file mode 100644 index b615ff2af8..0000000000 --- a/zealot/zed/construct.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { - ZedRecord, - ZedPrimitive, - ZedArray, - ZedSet, - ZedUnion, - ZedEnum, - ZedMap -} from "./data-types" -import ZedTypeDef from "./type-def" -import {TypeContext, Value, TypeName, Type} from "./zjson" - -export function construct( - context: TypeContext, - type: Type, - value: Value, - name?: TypeName -) { - switch (type.kind) { - case "record": - return new ZedRecord({type, value: value as Value[], context, name}) - case "primitive": - return new ZedPrimitive({type, value: value as string, name}) - case "array": - return new ZedArray({type, value: value as Value[], name, context}) - case "set": - return new ZedSet({type, value: value as Value[], name, context}) - case "union": - return new ZedUnion({type, value: value as Value[], name}) - case "enum": - return new ZedEnum({type, value: value as string, name}) - case "map": - return new ZedMap({type, value: value as Value[], name}) - case "typename": - var def = context.get(type.name) - if (!def) { - console.log("No typedef in context for", type) - console.log(context) - } - return construct(context, def.innerType, value, type.name) - case "typedef": - var def = new ZedTypeDef({type}) - context.set(def.name, def) - return construct(context, def.innerType, value, def.name) - default: - throw new Error(`Unknown ZJSON Type: ${JSON.stringify(type)}`) - } -} diff --git a/zealot/zed/data-types.ts b/zealot/zed/data-types.ts deleted file mode 100644 index b076d1656b..0000000000 --- a/zealot/zed/data-types.ts +++ /dev/null @@ -1,304 +0,0 @@ -import {construct} from "./construct" -import { - ArrayType, - EnumType, - MapType, - PrimitiveType, - RecordFieldType, - RecordType, - SetType, - Type, - TypeContext, - TypeName, - TypeNameType, - UnionType, - Value -} from "./zjson" - -export type ZedData = - | ZedPrimitive - | ZedRecord - | ZedArray - | ZedSet - | ZedUnion - | ZedEnum - | ZedMap - -/** - * Base Class for all Zed Data Types - */ -export type ZedTypeArgs = { - type: T - value: V - name?: TypeName - context?: TypeContext -} - -export class ZedType { - public _type: T - public _value: V - public _name?: TypeName - - constructor(args: ZedTypeArgs) { - this._type = args.type - this._value = args.value - this._name = args.name - } - - get kind(): string { - return this._name || this._type.kind - } - - toString() { - return this._value ? this._value.toString() : "" - } - - isUnset() { - return this._value === null - } - - serialize() { - return { - name: this._name, - type: this._type, - value: this._value - } - } -} - -/** - * Holds a name and a data field - */ - -export type ZedFieldSpec = { - name: string - data: { - name?: string - type: Type - value: Value - } -} -export class ZedField { - name: string - data: ZedData - - static deserialize({name, data}: ZedFieldSpec) { - return new ZedField({ - name, - data: construct(new Map(), data.type, data.value, data.name) - }) - } - - constructor({name, data}: {name: string; data: ZedData}) { - this.name = name - this.data = data - } - - serialize(): ZedFieldSpec { - return { - name: this.name, - data: this.data.serialize() - } - } -} - -export class ZedPrimitive extends ZedType { - get kind() { - return this._name || this._type.name - } - - toDate() { - if (this._value === null || this._type.name != "time") { - throw new Error(`Cannot make type: ${this._type.name} into a Date`) - } - return new Date(+this._value * 1000) - } - - toFloat() { - if (this._value === null) { - return 0 - } - try { - return parseFloat(this._value) - } catch { - throw new Error(`Cannot make type: ${this._type.name} into a Float`) - } - } - - toInt() { - if (this._value === null) throw new Error("value is unset") - const int = parseInt(this._value) - if (isNaN(int)) { - throw new Error(`Cannot make type: ${this._type.name} into an Integer`) - } - return int - } -} - -export class ZedArray extends ZedType { - items: ZedData[] - - constructor(args: { - type: ArrayType - value: Value[] - context: TypeContext - name: TypeName - }) { - super(args) - this.items = (this._value || []).map((value) => - construct(args.context, args.type.type, value) - ) - } - - get innerType() { - return this._type.type - } -} - -export class ZedSet extends ZedType { - items: ZedData[] - - constructor(args: { - type: SetType - value: Value[] - context: TypeContext - name: TypeName - }) { - super(args) - this.items = (this._value || []).map((value) => - construct(args.context, args.type.type, value) - ) - } - - get innerType() { - return this._type.type - } -} - -// Need use cases -export class ZedUnion extends ZedType {} -// Need use cases -export class ZedEnum extends ZedType {} -// Need use cases -export class ZedMap extends ZedType {} -// Need use cases -export class ZedTypeName extends ZedType {} - -class UnknownColumnError extends Error { - constructor(unknown: string, names: string[]) { - const available = names.map((n) => `"${n}"`).join(", ") - super(`"${unknown}" not present in [${available}]`) - } -} - -export type ZedRecordSpec = { - type: RecordType - value: Value[] - name?: TypeName -} - -export class ZedRecord extends ZedType { - fields: ZedField[] - context: TypeContext - - static of(fields: RecordFieldType[], values: Value[]) { - return new ZedRecord({type: {kind: "record", fields}, value: values}) - } - - static deserialize(args: ZedRecordSpec) { - console.log(args) - return new ZedRecord(args) - } - - constructor(args: ZedTypeArgs) { - super(args) - this.context = args.context || new Map() - this.fields = this._value.map((value, index) => { - const {name, type} = this._type.fields[index] - return new ZedField({ - name, - data: construct(args.context || new Map(), type, value) - }) - }) - } - - [Symbol.iterator]() { - let index = 0 - const next = () => - index < this._type.fields.length - ? {done: false, value: this.at(index++)} - : {done: true, value: undefined} - - return {next} - } - - get columns() { - return this.fields.map((f) => f.name) - } - - at(index: number) { - return this.fields[index].data - } - - get(name: string) { - return this.getField(name).data - } - - try(name: string) { - try { - return this.get(name) - } catch { - return null - } - } - - private _getField(name: string) { - if (this.isUnset()) throw new Error("Record is unset") - const field = this.fields.find((f) => f.name == name) - if (!field) throw new UnknownColumnError(name, this.columns) - return field - } - - getField(name: string) { - return name - .split(".") - .reduce((field: ZedField, namePart: string) => { - if (field.data instanceof ZedRecord) { - return new ZedField({name, data: field.data._getField(namePart).data}) - } else { - throw new Error("Dot syntax is only for nested records") - } - }, new ZedField({name: "", data: this})) - } - - tryField(name: string) { - try { - return this.getField(name) - } catch { - return null - } - } - - has(name: string) { - return this.columns.includes(name) - } - - flatten(prefix = ""): ZedRecord { - let fields = [] - let vals = [] - - this._type.fields.forEach((field, index) => { - if (field.type.kind === "record") { - const nested = (this.at(index) as ZedRecord).flatten(field.name + ".") - fields = fields.concat(nested._type.fields) - vals = vals.concat(nested._value) - } else { - fields.push({...field, name: prefix + field.name}) - // For an unset record, supply an unset value for each column. - vals.push(this.isUnset() ? null : this._value![index]) - } - }) - const type = {kind: "record", fields} as RecordType - return new ZedRecord({type, value: vals, context: this.context}) - } -} diff --git a/zealot/zed/enum.ts b/zealot/zed/enum.ts new file mode 100644 index 0000000000..400193f8ed --- /dev/null +++ b/zealot/zed/enum.ts @@ -0,0 +1,20 @@ +export class ZedEnum { + typeName?: string + symbols: string[] + value: string + + constructor(args: {typeName?: string; symbols: string[]; value: string}) { + this.symbols = args.symbols + this.value = args.value + this.typeName = args.typeName + } + + serialize() { + return { + kind: "enum", + symbols: this.symbols, + value: this.value, + typeName: this.typeName + } + } +} diff --git a/zealot/zed/field.ts b/zealot/zed/field.ts new file mode 100644 index 0000000000..bf3e4625bb --- /dev/null +++ b/zealot/zed/field.ts @@ -0,0 +1,19 @@ +import {ZedData} from "./index" + +export class ZedField { + name: string + data: ZedData + + constructor({name, data}: {name: string; data: ZedData}) { + this.name = name + this.data = data + } + + serialize() { + return { + kind: "field", + name: this.name, + data: this.data.serialize() + } + } +} diff --git a/zealot/zed/index.ts b/zealot/zed/index.ts new file mode 100644 index 0000000000..75351220c5 --- /dev/null +++ b/zealot/zed/index.ts @@ -0,0 +1,28 @@ +import {ZedArray} from "./array" +import {ZedEnum} from "./enum" +import {ZedField} from "./field" +import {ZedMap} from "./map" +import {ZedPrimitive} from "./primitive" +import {ZedRecord} from "./record" +import {ZedSet} from "./set" +import {ZedUnion} from "./union" + +export { + ZedArray, + ZedEnum, + ZedMap, + ZedPrimitive, + ZedRecord, + ZedSet, + ZedUnion, + ZedField +} + +export type ZedData = + | ZedPrimitive + | ZedRecord + | ZedArray + | ZedSet + | ZedUnion + | ZedEnum + | ZedMap diff --git a/zealot/zed/json.ts b/zealot/zed/json.ts new file mode 100644 index 0000000000..d20e2dae64 --- /dev/null +++ b/zealot/zed/json.ts @@ -0,0 +1,76 @@ +import { + ZedArray, + ZedPrimitive, + ZedEnum, + ZedField, + ZedMap, + ZedRecord, + ZedSet, + ZedUnion +} from "./index" + +export function deserialize(data) { + switch (data.kind) { + case "primitive": + return new ZedPrimitive({ + typeName: data.typeName, + value: data.value, + type: data.type + }) + + case "array": + return new ZedArray({ + typeName: data.typeName, + type: data.type, + items: data.items.map(deserialize) + }) + + case "set": + return new ZedSet({ + typeName: data.typeName, + type: data.type, + items: data.items.map(deserialize) + }) + + case "record": + return new ZedRecord({ + typeName: data.typeName, + fields: data.fields.map(deserialize) + }) + + case "field": + return new ZedField({ + name: data.name, + data: deserialize(data.data) + }) + + case "union": + return new ZedUnion({ + typeName: data.typeName, + types: data.types, + value: deserialize(data.value) + }) + + case "enum": + return new ZedEnum({ + typeName: data.typeName, + symbols: data.symbols, + value: data.value + }) + + case "map": + return new ZedMap({ + typeName: data.typeName, + keyType: data.keyType, + valueType: data.valueType, + value: new Map( + data.value.map(([key, value]) => { + return [deserialize(key), deserialize(value)] + }) + ) + }) + + default: + throw new Error(`Unknown Zed Type: ${JSON.stringify(data)}`) + } +} diff --git a/zealot/zed/map.ts b/zealot/zed/map.ts new file mode 100644 index 0000000000..b0453aac12 --- /dev/null +++ b/zealot/zed/map.ts @@ -0,0 +1,33 @@ +import {ZedData} from "./index" + +export class ZedMap { + keyType: string + valueType: string + typeName?: string + value: Map + + constructor(args: { + keyType: string + valueType: string + typeName?: string + value: Map + }) { + this.keyType = args.keyType + this.valueType = args.valueType + this.typeName = args.typeName + this.value = args.value + } + + serialize() { + return { + kind: "map", + keyType: this.keyType, + valueType: this.valueType, + typeName: this.typeName, + value: Array.from(this.value.entries()).map(([key, value]) => [ + key.serialize(), + value.serialize() + ]) + } + } +} diff --git a/zealot/zed/primitive.ts b/zealot/zed/primitive.ts new file mode 100644 index 0000000000..a8ceddc8bb --- /dev/null +++ b/zealot/zed/primitive.ts @@ -0,0 +1,50 @@ +export class ZedPrimitive { + public typeName?: string // the logical type name + public type: string // the actual type + public value: string | null + + constructor(args: {type: string; value: string | null; typeName?: string}) { + this.type = args.type + this.value = args.value + this.typeName = args.typeName + } + + isUnset() { + return this.value === null + } + + toDate() { + if (this.type !== "time") { + throw new Error(`Cannot make type: ${this.type} into a Date`) + } + return new Date(+this.value * 1000) + } + + toFloat() { + if (this.isUnset()) return 0 + + try { + return parseFloat(this.value) + } catch { + throw new Error(`Cannot make type: ${this.type} into a Float`) + } + } + + toInt() { + if (this.isUnset()) throw new Error("value is unset") + const int = parseInt(this.value) + if (isNaN(int)) { + throw new Error(`Cannot make type: ${this.type} into an Integer`) + } + return int + } + + serialize() { + return { + kind: "primitive", + type: this.type, + value: this.value, + typeName: this.typeName + } + } +} diff --git a/zealot/zed/record.ts b/zealot/zed/record.ts new file mode 100644 index 0000000000..10313550da --- /dev/null +++ b/zealot/zed/record.ts @@ -0,0 +1,105 @@ +import {ZedField} from "./field" + +export class ZedRecord { + fields: ZedField[] | null + typeName?: string + + constructor(args: {fields: ZedField[]; typeName?: string}) { + this.fields = args.fields + this.typeName = args.typeName + } + + [Symbol.iterator]() { + let index = 0 + const next = () => + index < this.fields.length + ? {done: false, value: this.at(index++)} + : {done: true, value: undefined} + + return {next} + } + + get columns() { + return this.fields.map((f) => f.name) + } + + at(index: number) { + return this.fields[index].data + } + + get(name: string) { + return this.getField(name).data + } + + try(name: string) { + try { + return this.get(name) + } catch { + return null + } + } + + isUnset() { + return this.fields === null + } + + private _getField(name: string) { + if (this.isUnset()) throw new Error("Record is unset") + const field = this.fields.find((f) => f.name == name) + if (!field) throw new UnknownColumnError(name, this.columns) + return field + } + + getField(name: string) { + return name + .split(".") + .reduce((field: ZedField, namePart: string) => { + if (field.data instanceof ZedRecord) { + return new ZedField({name, data: field.data._getField(namePart).data}) + } else { + throw new Error("Dot syntax is only for nested records") + } + }, new ZedField({name: "", data: this})) + } + + tryField(name: string) { + try { + return this.getField(name) + } catch { + return null + } + } + + has(name: string) { + return this.columns.includes(name) + } + + flatten(prefix = ""): ZedRecord { + let fields = [] + + this.fields.forEach((field) => { + if (field.data instanceof ZedRecord) { + const nested = field.data.flatten(field.name + ".") + fields = fields.concat(nested.fields) + } else { + fields.push({data: field.data, name: prefix + field.name}) + } + }) + return new ZedRecord({fields}) + } + + serialize() { + return { + kind: "record", + typeName: this.typeName, + fields: this.fields.map((f) => f.serialize()) + } + } +} + +class UnknownColumnError extends Error { + constructor(unknown: string, names: string[]) { + const available = names.map((n) => `"${n}"`).join(", ") + super(`"${unknown}" not present in [${available}]`) + } +} diff --git a/zealot/zed/set.ts b/zealot/zed/set.ts new file mode 100644 index 0000000000..a56410239f --- /dev/null +++ b/zealot/zed/set.ts @@ -0,0 +1,22 @@ +import {ZedData} from "./index" + +export class ZedSet { + type: string + items: ZedData[] + typeName?: string + + constructor(args: {type: string; typeName?: string; items: ZedData[]}) { + this.type = args.type + this.typeName = args.typeName + this.items = args.items + } + + serialize() { + return { + kind: "set", + type: this.type, + typeName: this.typeName, + items: this.items.map((item) => item.serialize()) + } + } +} diff --git a/zealot/zed/union.ts b/zealot/zed/union.ts new file mode 100644 index 0000000000..2c4a08eee5 --- /dev/null +++ b/zealot/zed/union.ts @@ -0,0 +1,22 @@ +import {ZedData} from "./index" + +export class ZedUnion { + types: string[] + value: ZedData + typeName?: string + + constructor(args: {types: string[]; value: ZedData; typeName?: string}) { + this.types = args.types + this.value = args.value + this.typeName = args.typeName + } + + serialize() { + return { + kind: "union", + types: this.types, + typeName: this.typeName, + value: this.value.serialize() + } + } +} diff --git a/zealot/zed/zjson.test.ts b/zealot/zed/zjson.test.ts index d2ea424e1f..b4cd74779a 100644 --- a/zealot/zed/zjson.test.ts +++ b/zealot/zed/zjson.test.ts @@ -1,4 +1,5 @@ import {execSync} from "child_process" +import {deserialize} from "./json" import {decode} from "./zjson" function zq(q, file) { @@ -11,7 +12,10 @@ function zq(q, file) { } test("can correlate?", () => { - const file = "test/data/count-by-typeof.zson" - const _list = decode(zq("*", file)) - const _chart = decode(zq("count() by typeof(.)", file)) + const file = "test/data/sample.zson" + const list = decode(zq("*", file)) + const json = list.rows.map((row) => row.serialize()) + const list2 = json.map(deserialize) + + expect(list2).toEqual(list.rows) }) diff --git a/zealot/zed/zjson.ts b/zealot/zed/zjson.ts index 1e276e37ac..af161a2f19 100644 --- a/zealot/zed/zjson.ts +++ b/zealot/zed/zjson.ts @@ -1,5 +1,13 @@ -import {construct} from "./construct" -import {ZedRecord} from "./data-types" +import { + ZedArray, + ZedEnum, + ZedField, + ZedMap, + ZedPrimitive, + ZedRecord, + ZedSet, + ZedUnion +} from "./index" import ZedTypeDef from "./type-def" type ID = string @@ -77,7 +85,7 @@ export type StreamObject = { export type TypeName = string -export type TypeContext = Map +export type TypeContext = {[typeName: string]: ZedTypeDef} export type DecodedZJSON = { rows: ZedRecord[] @@ -85,23 +93,123 @@ export type DecodedZJSON = { context: TypeContext } +export function construct( + context: TypeContext, + type: Type, + value: Value, + typeName?: TypeName +) { + switch (type.kind) { + case "primitive": + return new ZedPrimitive({ + typeName, + type: type.name, + value: value as string + }) + + case "array": + return new ZedArray({ + typeName, + type: simpleType(type.type), + items: (value as Value[]).map((value) => + construct(context, type.type, value) + ) + }) + + case "set": + return new ZedSet({ + typeName, + type: simpleType(type.type), + items: (value as Value[]).map((value) => + construct(context, type.type, value) + ) + }) + + case "record": + return new ZedRecord({ + typeName, + fields: type.fields.map((field, index) => { + return new ZedField({ + name: field.name, + data: construct(context, field.type, value[index]) + }) + }) + }) + + case "union": + var [index, unionValue] = value as [string, string] + return new ZedUnion({ + typeName, + types: type.types.map(simpleType), + value: construct(context, type.types[parseInt(index)], unionValue) + }) + + case "enum": + return new ZedEnum({ + typeName, + symbols: type.symbols, + value: value as string + }) + + case "map": + return new ZedMap({ + typeName, + keyType: simpleType(type.key_type), + valueType: simpleType(type.val_type), + value: new Map( + createEntries(value as Value[]).map(([key, value]) => { + return [ + construct(context, type.key_type, key), + construct(context, type.val_type, value) + ] + }) + ) + }) + + case "typename": + var def = context[type.name] + if (!def) throw new Error("No typedef for: " + type.name) + return construct(context, def.innerType, value, type.name) + + case "typedef": + var def = new ZedTypeDef({type}) + context[def.name] = def + return construct(context, def.innerType, value, def.name) + + default: + throw new Error(`Unknown ZJSON Type: ${JSON.stringify(type)}`) + } +} + +function simpleType(type) { + return type.kind === "primitive" ? type.name : type.kind +} + +function createEntries(array) { + const entries = [] + for (let i = 0; i < array.length; i += 2) { + entries.push([array[i], array[i + 1]]) + } + return entries +} + export function decode( zjson: StreamObject[], - context: TypeContext = new Map(), - schemas: TypeContext = new Map() + context: TypeContext = {}, + schemas: TypeContext = {} ): DecodedZJSON { const rows = [] for (const {schema, types, values} of zjson) { // Add all the types to the context types && - types.forEach((typedef) => - context.set(typedef.name, new ZedTypeDef({type: typedef})) + types.forEach( + (typedef) => (context[typedef.name] = new ZedTypeDef({type: typedef})) ) // Add the root record type to the schemas - const typedef = context.get(schema) - if (!schemas.has(schema)) schemas.set(schema, typedef) + const typedef = context[schema] + if (!(schema in schemas)) schemas[schema] = typedef // Construct the row and save it const type = typedef.innerType From a7b447e345ad54f8af30877e5efba306f963c3ed Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 20 Apr 2021 14:09:06 -0700 Subject: [PATCH 05/41] No typescript errors --- app/core/hooks/useSearch.ts | 2 +- app/core/utils/type-class-names.ts | 11 +++ app/detail/Fields.tsx | 15 ++-- app/detail/Pane.tsx | 2 +- app/detail/flows/contextMenu.ts | 2 +- .../flows/next-page-viewer-search.test.ts | 18 ++--- app/search/flows/next-page-viewer-search.ts | 2 +- app/search/flows/viewer-search.test.ts | 2 +- app/search/flows/viewer-search.ts | 4 +- app/tile/viz.tsx | 2 +- errors.txt | 20 +++++ map.zson | 1 + ppl/detail/Correlation.tsx | 2 +- ppl/detail/RelatedAlerts.tsx | 2 +- ppl/detail/RelatedConns.tsx | 2 +- ppl/detail/flows/fetch.test.ts | 39 +++++---- ppl/detail/flows/fetch.ts | 9 ++- .../flows/get-correlation-query.test.ts | 74 ++++++----------- ppl/detail/flows/get-correlation-query.ts | 2 +- ppl/detail/flows/test-this.test.ts | 7 ++ ppl/detail/models/BrimEvent.ts | 2 +- ppl/detail/models/Correlation.ts | 15 ++-- ppl/detail/models/SuricataEvent.ts | 2 +- ppl/detail/models/UnknownEvent.ts | 2 +- ppl/detail/models/ZeekEvent.ts | 2 +- ppl/detail/util/sort.ts | 2 +- ppl/menus/detailFieldContextMenu.ts | 7 +- ppl/menus/searchFieldContextMenu.ts | 7 +- ppl/summary/bar-chart.tsx | 2 +- ppl/summary/horizontal-bar-chart.tsx | 2 +- scripts/test/responses.js | 33 ++++++++ .../search.ts => scripts/test/search.deno.ts | 11 ++- src/js/brim/cell.ts | 2 +- src/js/brim/complexCell.ts | 2 +- src/js/brim/primitiveCell.ts | 6 +- src/js/brim/program.test.ts | 50 ++++-------- src/js/brim/program.ts | 2 +- src/js/components/ConnVersation.tsx | 12 ++- src/js/components/FieldCell.tsx | 10 +-- src/js/components/LogCell/CompoundField.tsx | 2 +- src/js/components/LogCell/SingleField.tsx | 2 +- src/js/components/LogCell/index.tsx | 5 +- src/js/components/LogDetails/ConnPanel.tsx | 2 +- src/js/components/LogDetails/Md5Panel.tsx | 2 +- src/js/components/LogRow.tsx | 2 +- src/js/components/RightPane.test.tsx | 4 +- src/js/components/RightPane.tsx | 2 +- .../components/SearchResults/ResultsTable.tsx | 2 +- src/js/components/Tables/HorizontalTable.tsx | 2 +- src/js/components/Tables/Table.tsx | 10 ++- src/js/components/Tables/VerticalTable.tsx | 2 +- src/js/components/Viewer/Chunk.tsx | 2 +- src/js/components/Viewer/Viewer.tsx | 2 +- src/js/electron/menu/actions/detailActions.ts | 73 +++++++++-------- src/js/electron/menu/actions/searchActions.ts | 60 +++++++------- src/js/flows/downloadPcap.ts | 2 +- src/js/flows/rightclick/cellMenu.test.ts | 41 +++------- src/js/flows/scrollToLog.ts | 2 +- src/js/flows/searchBar/actions.ts | 2 +- src/js/flows/viewLogDetail.ts | 2 +- src/js/models/TableColumns.ts | 4 +- src/js/searches/programs.test.ts | 28 ++----- src/js/searches/programs.ts | 2 +- src/js/state/Chart/actions.ts | 2 +- src/js/state/Chart/test.ts | 14 +--- src/js/state/Columns/models/columnSet.ts | 10 ++- src/js/state/Columns/selectors.ts | 2 +- src/js/state/Columns/touch.test.ts | 56 +++++++------ src/js/state/LogDetails/actions.ts | 2 +- src/js/state/LogDetails/selectors.ts | 2 +- src/js/state/LogDetails/test.ts | 17 +--- src/js/state/LogDetails/types.ts | 8 +- src/js/state/Packets/flows.ts | 2 +- src/js/state/SearchBar/test.ts | 17 ++-- src/js/state/Viewer/actions.ts | 2 +- src/js/state/Viewer/reducer.ts | 4 +- src/js/state/Viewer/selectors.ts | 2 +- src/js/state/Viewer/test.ts | 70 ++++++++-------- src/js/state/Viewer/types.ts | 2 +- src/js/types/index.ts | 2 +- src/js/zql/toZql.ts | 6 +- test/data/correlation.zson | 2 + test/{api => }/data/custom-sample.ndjson | 0 test/{api => }/data/custom-schema.json | 0 test/{api => }/data/plain.txt | 0 test/{api => }/data/sample.ndjson | 0 test/{api => }/data/sample.pcap | Bin test/{api => }/data/sample.pcapng | Bin test/{api => }/data/sample.tsv | 0 test/{api => }/data/sample.zng | Bin test/{api => }/data/sampleTypes.json | 0 test/{api => }/data/types.tsv | 0 test/factories/record.ts | 40 +++++++++ test/responses/config.js | 35 ++++++++ .../correlation-uid-community-id.response | 14 ++++ test/responses/correlation-uid.response | 14 ++++ test/responses/count-by-path.response | 14 ++++ test/responses/count.response | 14 ++++ test/responses/dns.response | 14 ++++ test/responses/index.ts | 19 ++++- test/responses/sample.response | 14 ++++ zealot/fetcher/records_callback.ts | 6 +- zealot/fetcher/stream.ts | 2 +- zealot/zed/array.ts | 4 + zealot/zed/enum.ts | 4 + zealot/zed/field.ts | 5 ++ zealot/zed/index.ts | 2 + zealot/zed/json.ts | 76 ++++++++++++++++-- zealot/zed/map.ts | 4 + zealot/zed/primitive.ts | 4 + zealot/zed/record.ts | 5 ++ zealot/zed/set.ts | 4 + zealot/zed/union.ts | 4 + 113 files changed, 708 insertions(+), 458 deletions(-) create mode 100644 app/core/utils/type-class-names.ts create mode 100644 errors.txt create mode 100644 map.zson create mode 100644 ppl/detail/flows/test-this.test.ts create mode 100644 scripts/test/responses.js rename test/api/scripts/search.ts => scripts/test/search.deno.ts (60%) create mode 100644 test/data/correlation.zson rename test/{api => }/data/custom-sample.ndjson (100%) rename test/{api => }/data/custom-schema.json (100%) rename test/{api => }/data/plain.txt (100%) rename test/{api => }/data/sample.ndjson (100%) rename test/{api => }/data/sample.pcap (100%) rename test/{api => }/data/sample.pcapng (100%) rename test/{api => }/data/sample.tsv (100%) rename test/{api => }/data/sample.zng (100%) rename test/{api => }/data/sampleTypes.json (100%) rename test/{api => }/data/types.tsv (100%) create mode 100644 test/factories/record.ts create mode 100644 test/responses/config.js create mode 100644 test/responses/correlation-uid-community-id.response create mode 100644 test/responses/correlation-uid.response create mode 100644 test/responses/count-by-path.response create mode 100644 test/responses/count.response create mode 100644 test/responses/dns.response create mode 100644 test/responses/sample.response diff --git a/app/core/hooks/useSearch.ts b/app/core/hooks/useSearch.ts index 3fa55ca36f..5132d55f18 100644 --- a/app/core/hooks/useSearch.ts +++ b/app/core/hooks/useSearch.ts @@ -2,7 +2,7 @@ import {useEffect, useState} from "react" import {useDispatch} from "react-redux" import {search} from "src/js/flows/search/mod" import {AppDispatch} from "src/js/state/types" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" type R = [ZedRecord[], boolean] diff --git a/app/core/utils/type-class-names.ts b/app/core/utils/type-class-names.ts new file mode 100644 index 0000000000..d93f5c3094 --- /dev/null +++ b/app/core/utils/type-class-names.ts @@ -0,0 +1,11 @@ +import {ZedData, ZedPrimitive} from "zealot/zed" + +export function typeClassNames(data: ZedData) { + const classNames = [] + if (data instanceof ZedPrimitive) { + classNames.push(data.type) + } + if (data.typeName) classNames.push(data.typeName) + if (data.isUnset()) classNames.push("null") + return classNames.join(" ") +} diff --git a/app/detail/Fields.tsx b/app/detail/Fields.tsx index 8a46cbfed3..f06e24eb5b 100644 --- a/app/detail/Fields.tsx +++ b/app/detail/Fields.tsx @@ -1,15 +1,14 @@ -import {useDispatch} from "react-redux" -import React, {memo, useCallback, useMemo, useState} from "react" - import {Data, Name, Value} from "app/core/Data" +import {typeClassNames} from "app/core/utils/type-class-names" +import React, {memo, useCallback, useMemo, useState} from "react" +import {useDispatch} from "react-redux" import {createCell} from "src/js/brim/cell" import BrimTooltip from "src/js/components/BrimTooltip" import ColumnDescription from "src/js/components/LogDetails/ColumnDescription" - -import PanelHeading from "./PanelHeading" -import Panel from "./Panel" +import {ZedField, ZedRecord} from "zealot/zed" import contextMenu from "./flows/contextMenu" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import Panel from "./Panel" +import PanelHeading from "./PanelHeading" type Props = { record: ZedRecord @@ -34,7 +33,7 @@ const DataPanel = React.memo(function DataTable({ {field.name} onRightClick(field)} > {createCell(field).display()} diff --git a/app/detail/Pane.tsx b/app/detail/Pane.tsx index 1c25f49bf4..ea85c4e413 100644 --- a/app/detail/Pane.tsx +++ b/app/detail/Pane.tsx @@ -13,7 +13,7 @@ import LogDetails from "src/js/state/LogDetails" import styled from "styled-components" import Fields from "./Fields" import NoSelection from "./NoSelection" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" const BG = styled.div` padding: 12px; diff --git a/app/detail/flows/contextMenu.ts b/app/detail/flows/contextMenu.ts index a0116dd03f..574c2be5ee 100644 --- a/app/detail/flows/contextMenu.ts +++ b/app/detail/flows/contextMenu.ts @@ -4,7 +4,7 @@ import {showContextMenu} from "src/js/lib/System" import Columns from "src/js/state/Columns" import Current from "src/js/state/Current" import SearchBar from "src/js/state/SearchBar" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" const contextMenu = (field: ZedField, record: ZedRecord) => (_, getState) => { const space = Current.mustGetSpace(getState()) diff --git a/app/search/flows/next-page-viewer-search.test.ts b/app/search/flows/next-page-viewer-search.test.ts index eda820c361..3560010eb0 100644 --- a/app/search/flows/next-page-viewer-search.test.ts +++ b/app/search/flows/next-page-viewer-search.test.ts @@ -9,23 +9,15 @@ import Viewer from "src/js/state/Viewer" import Workspaces from "src/js/state/Workspaces" import fixtures from "src/js/test/fixtures" import initTestStore from "src/js/test/initTestStore" -import {STRING, TIME} from "test/fixtures/zjson-types" +import {createRecord} from "test/factories/record" import {createZealotMock} from "zealot" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" -import {RecordType} from "zealot/zed/zjson" +import {ZedPrimitive} from "zealot/zed" import nextPageViewerSearch from "./next-page-viewer-search" -const type = { - kind: "record", - fields: [ - {name: "td", type: STRING}, - {name: "ts", type: TIME} - ] -} as RecordType const records = [ - new ZedRecord({type, value: ["1", "100"]}), - new ZedRecord({type, value: ["1", "200"]}), - new ZedRecord({type, value: ["1", "300"]}) + createRecord({td: "1", ts: new Date(100)}), + createRecord({td: "1", ts: new Date(200)}), + createRecord({td: "1", ts: new Date(300)}) ] let store, zealot, tabId diff --git a/app/search/flows/next-page-viewer-search.ts b/app/search/flows/next-page-viewer-search.ts index 1fd7e9b7fd..56fc2eeb5a 100644 --- a/app/search/flows/next-page-viewer-search.ts +++ b/app/search/flows/next-page-viewer-search.ts @@ -8,7 +8,7 @@ import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import Url from "src/js/state/Url" import Viewer from "src/js/state/Viewer" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed" import {viewerSearch} from "./viewer-search" /** diff --git a/app/search/flows/viewer-search.test.ts b/app/search/flows/viewer-search.test.ts index 0005da1434..dc48838e30 100644 --- a/app/search/flows/viewer-search.test.ts +++ b/app/search/flows/viewer-search.test.ts @@ -11,7 +11,7 @@ // import responses from "src/js/test/responses" // import {STRING} from "test/fixtures/zjson-types" // import {createZealotMock} from "zealot" -// import {ZedRecord} from "zealot/zed/data-types" +// import {ZedRecord} from "zealot/zed" // import {viewerSearch} from "./viewer-search" // const dnsResp = responses("dns.txt") diff --git a/app/search/flows/viewer-search.ts b/app/search/flows/viewer-search.ts index d2d2b552e9..a1f4a64bf7 100644 --- a/app/search/flows/viewer-search.ts +++ b/app/search/flows/viewer-search.ts @@ -9,7 +9,7 @@ import SearchBar from "src/js/state/SearchBar" import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import Viewer from "src/js/state/Viewer" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {TypeContext} from "zealot/zed/zjson" type Args = { @@ -40,7 +40,7 @@ function handle( append = false ): Thunk { return function(dispatch) { - let allColumns: TypeContext = new Map() + let allColumns: TypeContext = {} let allRecords: ZedRecord[] = [] let count = 0 diff --git a/app/tile/viz.tsx b/app/tile/viz.tsx index a1c19aad55..d54105e8e5 100644 --- a/app/tile/viz.tsx +++ b/app/tile/viz.tsx @@ -6,7 +6,7 @@ import {TileFormat} from "ppl/summary/summary" import Table from "ppl/summary/table" import React, {memo} from "react" import styled from "styled-components" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" const BG = styled.div` flex: 1; diff --git a/errors.txt b/errors.txt new file mode 100644 index 0000000000..aab8ee749e --- /dev/null +++ b/errors.txt @@ -0,0 +1,20 @@ +app/search/flows/next-page-viewer-search.test.ts +ppl/detail/flows/fetch.test.ts +ppl/detail/flows/get-correlation-query.test.ts +src/js/brim/program.test.ts +src/js/components/RightPane.test.tsx +src/js/electron/menu/actions/detailActions.ts +src/js/electron/menu/actions/searchActions.ts +src/js/flows/rightclick/cellMenu.test.ts +src/js/models/TableColumns.ts +src/js/searches/programs.test.ts +src/js/state/Chart/test.ts +src/js/state/Columns/models/columnSet.ts +src/js/state/Columns/selectors.ts +src/js/state/Columns/touch.test.ts +src/js/state/Columns/touch.ts +src/js/state/LogDetails/test.ts +src/js/state/SearchBar/test.ts +src/js/state/Viewer/reducer.ts +src/js/state/Viewer/test.ts +zealot/fetcher/records_callback.ts diff --git a/map.zson b/map.zson new file mode 100644 index 0000000000..931756f0d6 --- /dev/null +++ b/map.zson @@ -0,0 +1 @@ +{ my_map: |{ {"name", "james"}, {"age", "thirty"} }| } diff --git a/ppl/detail/Correlation.tsx b/ppl/detail/Correlation.tsx index 4d24f069e3..f23c6612f6 100644 --- a/ppl/detail/Correlation.tsx +++ b/ppl/detail/Correlation.tsx @@ -15,7 +15,7 @@ import Panel from "app/detail/Panel" import {getCorrelationQuery} from "./flows/get-correlation-query" import EventLimit from "./EventLimit" import {showContextMenu} from "src/js/lib/System" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" export default memo(function UidPanel({record}: {record: ZedRecord}) { const query = useMemo(() => getCorrelationQuery(record), [record]) diff --git a/ppl/detail/RelatedAlerts.tsx b/ppl/detail/RelatedAlerts.tsx index 3c95227306..ebfdcbf3ac 100644 --- a/ppl/detail/RelatedAlerts.tsx +++ b/ppl/detail/RelatedAlerts.tsx @@ -9,7 +9,7 @@ import React, {memo, useCallback, useMemo} from "react" import brim from "src/js/brim" import {showContextMenu} from "src/js/lib/System" import zql from "src/js/zql" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import EventLimit from "./EventLimit" import EventTimeline from "./EventTimeline" import firstLast from "./util/firstLast" diff --git a/ppl/detail/RelatedConns.tsx b/ppl/detail/RelatedConns.tsx index ad96779d9e..7aa4a9b97b 100644 --- a/ppl/detail/RelatedConns.tsx +++ b/ppl/detail/RelatedConns.tsx @@ -8,7 +8,7 @@ import React, {memo, useCallback, useMemo} from "react" import brim from "src/js/brim" import {showContextMenu} from "src/js/lib/System" import zql from "src/js/zql" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import EventLimit from "./EventLimit" import EventTimeline from "./EventTimeline" import firstLast from "./util/firstLast" diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index 5feea22fe2..fb6e51d075 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -1,26 +1,19 @@ import {last} from "lodash" import loginTo from "src/js/test/helpers/loginTo" -import {ZedRecord} from "zealot/zed/data-types" +import {recordOf} from "test/factories/record" +import {useResponse} from "test/responses" import {fetchCorrelation} from "./fetch" -const zeek = new ZedRecord({ - type: { - kind: "record", - fields: [ - {name: "_path", type: {kind: "primitive", name: "string"}}, - {name: "uid", type: {kind: "primitive", name: "string"}} - ] - }, - value: ["dns", "CbOjYpkXn9LfqV51c"] -}) +const zeek = recordOf( + ["_path", "string", "dns"], + ["uid", "string", "CbOjYpkXn9LfqV51c"] +) -const suricata = new ZedRecord({ - type: { - fields: [{name: "community_id", type: {kind: "primitive", name: "string"}}], - kind: "record" - }, - value: ["1:N7YGmWjwTmMKNhsZHBR618n3ReA="] -}) +const suricata = recordOf([ + "community_id", + "string", + "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" +]) const uidOrCommunityIdZql = 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' @@ -30,8 +23,14 @@ const uidZql = const cidZql = 'community_id="1:N7YGmWjwTmMKNhsZHBR618n3ReA=" | head 100' -const stubs: any = {} -describe("zeek log when community_id is found", () => { +const stubs = { + uidResult: useResponse("correlationUid"), + uidAndCommunityResult: useResponse("correlationUidCommunityId"), + alertResults: "FILL ME IN", + noCommunityIdInConn: "FILL ME IN" +} + +describe.only("zeek log when community_id is found", () => { let setup beforeEach(async () => { setup = await loginTo("workspace1", "space1") diff --git a/ppl/detail/flows/fetch.ts b/ppl/detail/flows/fetch.ts index 4171f38b69..bfd3062545 100644 --- a/ppl/detail/flows/fetch.ts +++ b/ppl/detail/flows/fetch.ts @@ -1,5 +1,5 @@ import {search} from "src/js/flows/search/mod" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {Correlation} from "../models/Correlation" import {getCorrelationQuery} from "./get-correlation-query" @@ -11,16 +11,16 @@ function findConn(records) { const collect = ({response, promise}) => { let records: ZedRecord[] = [] - response.chan(0, ({rows}) => (records = rows)) + response.chan(0, ({rows}) => { + records = rows + }) return promise.then(() => records) } export const fetchCorrelation = (record: ZedRecord) => async (dispatch) => { const query = getCorrelationQuery(record) const {uid, cid} = new Correlation(record).getIds() - const run = () => collect(dispatch(search({query, id}))) - if (!uid && !cid) return [] if (cid && uid) return run() if (cid) return run() @@ -28,6 +28,7 @@ export const fetchCorrelation = (record: ZedRecord) => async (dispatch) => { // If there is only a uid and not a cid const records = await run() const conn = findConn(records) + console.log(conn) if (conn && conn.has("community_id")) return dispatch(fetchCorrelation(conn)) else return records } diff --git a/ppl/detail/flows/get-correlation-query.test.ts b/ppl/detail/flows/get-correlation-query.test.ts index 81aff0b4aa..033fef6336 100644 --- a/ppl/detail/flows/get-correlation-query.test.ts +++ b/ppl/detail/flows/get-correlation-query.test.ts @@ -3,22 +3,16 @@ import { connCorrelation, uidCorrelation } from "src/js/searches/programs" -import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" -import {RecordFieldType, RecordType} from "zealot/zed/zjson" +import {createRecord} from "test/factories/record" +import {ZedPrimitive} from "zealot/zed" import {getCorrelationQuery} from "./get-correlation-query" test("returns uid query if ts and duration are missing", () => { - const type = { - kind: "record", - fields: [ - {name: "_path", type: STRING}, - {name: "uid", type: STRING}, - {name: "community_id", type: STRING} - ] - } as RecordType - const value = ["conn", "CHem0e2rJqHiwjhgq7", "1:NYgcI8mLerCC20GwJVV5AftL0uY="] - const record = new ZedRecord({type, value}) + const record = createRecord({ + _path: "conn", + uid: "CHem0e2rJqHiwjhgq7", + community_id: "1:NYgcI8mLerCC20GwJVV5AftL0uY=" + }) expect(getCorrelationQuery(record)).toBe( uidCorrelation(record.get("uid") as ZedPrimitive) @@ -26,22 +20,13 @@ test("returns uid query if ts and duration are missing", () => { }) test("returns conn query if ts and duration are present", () => { - const type = [ - {name: "_path", type: STRING}, - {name: "uid", type: STRING}, - {name: "community_id", type: STRING}, - {name: "ts", type: TIME}, - {name: "duration", type: INTERVAL} - ] as RecordFieldType[] - const value = [ - "conn", - "CHem0e2rJqHiwjhgq7", - "1:NYgcI8mLerCC20GwJVV5AftL0uY=", - "1585852166.003543", - null - ] - const record = ZedRecord.of(type, value) - + const record = createRecord({ + _path: "conn", + uid: "CHem0e2rJqHiwjhgq7", + community_id: "1:NYgcI8mLerCC20GwJVV5AftL0uY=", + ts: new Date(1585852166.003543 * 1000), + duration: null + }) expect(getCorrelationQuery(record)).toBe( connCorrelation( record.get("uid") as ZedPrimitive, @@ -53,19 +38,12 @@ test("returns conn query if ts and duration are present", () => { }) test("returns cid query if only cid present", () => { - const type = [ - {name: "_path", type: STRING}, - {name: "community_id", type: STRING}, - {name: "ts", type: TIME}, - {name: "duration", type: INTERVAL} - ] as RecordFieldType[] - const value = [ - "conn", - "1:NYgcI8mLerCC20GwJVV5AftL0uY=", - "1585852166.003543", - null - ] - const record = ZedRecord.of(type, value) + const record = createRecord({ + _path: "conn", + community_id: "1:NYgcI8mLerCC20GwJVV5AftL0uY=", + ts: new Date(1585852166.003543 * 1000), + duration: null + }) expect(getCorrelationQuery(record)).toBe( cidCorrelation(record.get("community_id") as ZedPrimitive) @@ -73,13 +51,11 @@ test("returns cid query if only cid present", () => { }) test("returns null if no cid or uid", () => { - const type = [ - {name: "_path", type: STRING}, - {name: "ts", type: TIME}, - {name: "duration", type: INTERVAL} - ] as RecordFieldType[] - const value = ["conn", "1585852166.003543", null] - const record = ZedRecord.of(type, value) + const record = createRecord({ + _path: "conn", + ts: new Date(1585852166.003543 * 1000), + duration: null + }) expect(getCorrelationQuery(record)).toBe(null) }) diff --git a/ppl/detail/flows/get-correlation-query.ts b/ppl/detail/flows/get-correlation-query.ts index 2dec7b2655..8ff9432477 100644 --- a/ppl/detail/flows/get-correlation-query.ts +++ b/ppl/detail/flows/get-correlation-query.ts @@ -3,7 +3,7 @@ import { connCorrelation, uidCorrelation } from "src/js/searches/programs" -import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" +import {ZedRecord, ZedPrimitive} from "zealot/zed" import {Correlation} from "../models/Correlation" export function getCorrelationQuery(record: ZedRecord) { diff --git a/ppl/detail/flows/test-this.test.ts b/ppl/detail/flows/test-this.test.ts new file mode 100644 index 0000000000..6664790072 --- /dev/null +++ b/ppl/detail/flows/test-this.test.ts @@ -0,0 +1,7 @@ +import {useResponse} from "test/responses" + +const dns = useResponse("dns") + +test("hi", () => { + console.log(dns) +}) diff --git a/ppl/detail/models/BrimEvent.ts b/ppl/detail/models/BrimEvent.ts index 357c47fdd6..b43b388de4 100644 --- a/ppl/detail/models/BrimEvent.ts +++ b/ppl/detail/models/BrimEvent.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {SuricataEvent} from "./SuricataEvent" import {UnknownEvent} from "./UnknownEvent" import {ZeekEvent} from "./ZeekEvent" diff --git a/ppl/detail/models/Correlation.ts b/ppl/detail/models/Correlation.ts index bfcdc2552f..7c34afb227 100644 --- a/ppl/detail/models/Correlation.ts +++ b/ppl/detail/models/Correlation.ts @@ -1,5 +1,5 @@ -import {get, isString} from "lodash" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {get} from "lodash" +import {ZedArray, ZedPrimitive, ZedRecord, ZedSet} from "zealot/zed" const specialUids = { files: "conn_uids", @@ -31,11 +31,12 @@ export class Correlation { const data = this.r.get(name) if (data instanceof ZedPrimitive) { return data.toString() - } else { - const value = data._value - if (!value) return null - if (isString(value)) return value - return value.map((v) => v.toString()).join(" ") + } else if (data instanceof ZedArray || data instanceof ZedSet) { + const uids = data.items.map((item) => { + if (item instanceof ZedPrimitive) return item.toString() + else return "" + }) + return uids.join(" ") } } } diff --git a/ppl/detail/models/SuricataEvent.ts b/ppl/detail/models/SuricataEvent.ts index 650e3a219b..c25ba1c16e 100644 --- a/ppl/detail/models/SuricataEvent.ts +++ b/ppl/detail/models/SuricataEvent.ts @@ -1,4 +1,4 @@ -import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" +import {ZedRecord, ZedPrimitive} from "zealot/zed" import {BrimEventInterface} from "./BrimEvent" export class SuricataEvent implements BrimEventInterface { diff --git a/ppl/detail/models/UnknownEvent.ts b/ppl/detail/models/UnknownEvent.ts index 9b0ebe37ab..fef506673b 100644 --- a/ppl/detail/models/UnknownEvent.ts +++ b/ppl/detail/models/UnknownEvent.ts @@ -1,4 +1,4 @@ -import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" +import {ZedRecord, ZedPrimitive} from "zealot/zed" import {BrimEventInterface} from "./BrimEvent" export class UnknownEvent implements BrimEventInterface { diff --git a/ppl/detail/models/ZeekEvent.ts b/ppl/detail/models/ZeekEvent.ts index e93597c2cd..8a3ba75e19 100644 --- a/ppl/detail/models/ZeekEvent.ts +++ b/ppl/detail/models/ZeekEvent.ts @@ -1,4 +1,4 @@ -import {ZedRecord, ZedPrimitive} from "zealot/zed/data-types" +import {ZedRecord, ZedPrimitive} from "zealot/zed" import {BrimEventInterface} from "./BrimEvent" export class ZeekEvent implements BrimEventInterface { diff --git a/ppl/detail/util/sort.ts b/ppl/detail/util/sort.ts index b101f8b244..07c2450207 100644 --- a/ppl/detail/util/sort.ts +++ b/ppl/detail/util/sort.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" export const sort = (logs: ZedRecord[]) => { const findConn = (log) => log.try("_path")?.toString() === "conn" diff --git a/ppl/menus/detailFieldContextMenu.ts b/ppl/menus/detailFieldContextMenu.ts index 3660bb48bc..abb7f96449 100644 --- a/ppl/menus/detailFieldContextMenu.ts +++ b/ppl/menus/detailFieldContextMenu.ts @@ -3,7 +3,7 @@ import {isEqual} from "lodash" import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" import {Space} from "src/js/state/Spaces/types" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" export default function detailFieldContextMenu( program: string, @@ -15,10 +15,11 @@ export default function detailFieldContextMenu( log: ZedRecord, compound: boolean ): MenuItemConstructorOptions[] { - const isTime = field.data.kind === "time" + const isTime = + field.data instanceof ZedPrimitive && field.data.type === "time" const isConn = log.try("_path")?.toString() === "conn" const isGroupBy = hasGroupByProc(program) - const isIp = ["addr", "set[addr]"].includes(field.data.kind) + const isIp = field.data instanceof ZedPrimitive && field.data.type === "ip" const hasCol = columns.includes(field.name) const sameCols = isEqual(log.columns.sort(), columns.sort()) const hasPackets = space && space.pcap_support diff --git a/ppl/menus/searchFieldContextMenu.ts b/ppl/menus/searchFieldContextMenu.ts index e5c1df1bd4..c6c0d42dbb 100644 --- a/ppl/menus/searchFieldContextMenu.ts +++ b/ppl/menus/searchFieldContextMenu.ts @@ -4,7 +4,7 @@ import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" import {Space} from "src/js/state/Spaces/types" import {RightClickBuilder} from "src/js/types" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" export default function searchFieldContextMenu( program: string, @@ -16,10 +16,11 @@ export default function searchFieldContextMenu( log: ZedRecord, compound: boolean ): MenuItemConstructorOptions[] { - const isTime = field.data.kind === "time" + const isTime = + field.data instanceof ZedPrimitive && field.data.type === "time" const isConn = log.try("_path")?.toString() === "conn" const isGroupBy = hasGroupByProc(program) - const isIp = ["addr", "ip"].includes(field.data.kind) + const isIp = field.data instanceof ZedPrimitive && field.data.type === "ip" const hasCol = columns.includes(field.name) const flatColNames = log.flatten().columns const sameCols = isEqual(flatColNames.sort(), columns.sort()) diff --git a/ppl/summary/bar-chart.tsx b/ppl/summary/bar-chart.tsx index 8f7b2d30e2..8d71c6dc10 100644 --- a/ppl/summary/bar-chart.tsx +++ b/ppl/summary/bar-chart.tsx @@ -5,7 +5,7 @@ import {BarRounded} from "@vx/shape" import React from "react" import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed" type Props = { records: ZedRecord[] diff --git a/ppl/summary/horizontal-bar-chart.tsx b/ppl/summary/horizontal-bar-chart.tsx index c6db7eddde..2fd8cabd05 100644 --- a/ppl/summary/horizontal-bar-chart.tsx +++ b/ppl/summary/horizontal-bar-chart.tsx @@ -5,7 +5,7 @@ import {BarRounded} from "@vx/shape" import React from "react" import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed" type Props = { records: ZedRecord[] diff --git a/scripts/test/responses.js b/scripts/test/responses.js new file mode 100644 index 0000000000..671b4fd19a --- /dev/null +++ b/scripts/test/responses.js @@ -0,0 +1,33 @@ +const {spawn} = require("child_process") +const fs = require("fs-extra") +const glob = require("glob") +const config = require("../../test/responses/config") + +function saveResponse(input, output, query) { + const deno = spawn( + "deno", + [`run --allow-all scripts/test/search.deno.ts '${input}' '${query}'`], + {shell: true} + ) + const out = fs.createWriteStream(output) + deno.stdout.pipe(out) + + return new Promise((resolve) => { + deno.on("close", () => { + resolve() + }) + }) +} + +async function main() { + glob.sync("test/responses/*.response").forEach((file) => fs.removeSync(file)) + + for (const key in config) { + const {input, output, query} = config[key] + const inFile = `test/data/${input}` + const outFile = `test/responses/${output}` + await saveResponse(inFile, outFile, query) + } +} + +main() diff --git a/test/api/scripts/search.ts b/scripts/test/search.deno.ts similarity index 60% rename from test/api/scripts/search.ts rename to scripts/test/search.deno.ts index 976b3926d7..de1c37bebd 100644 --- a/test/api/scripts/search.ts +++ b/scripts/test/search.deno.ts @@ -3,14 +3,14 @@ import { fromFileUrl, dirname } from "https://deno.land/std@0.70.0/path/mod.ts" -import {withZqd} from "../helper/test_api.ts" +import {withZqd} from "../../test/api/helper/test_api.ts" -const DIR = dirname(fromFileUrl(import.meta.url)) -const FILE = join(DIR, "../test_api/data/sample.tsv") +const FILE = join(Deno.cwd(), Deno.args[0]) +const QUERY = Deno.args[1] async function ingest(zealot: any) { const space = await zealot.spaces.create({name: "gen space"}) - const ingest = await zealot.logs.post({paths: [FILE], spaceId: space.id}) + const ingest = await zealot.logs.postPaths({paths: [FILE], spaceId: space.id}) await ingest.origResp.text() return space.id } @@ -20,8 +20,7 @@ withZqd(async (zealot) => { const spaceId = await ingest(zealot) const from = new Date(0) const to = new Date() - const zql = Deno.args[0] - const search = await zealot.search(zql, {spaceId, from, to}) + const search = await zealot.search(QUERY, {spaceId, from, to}) const text = await search.origResp.text() console.log(text) } catch (e) { diff --git a/src/js/brim/cell.ts b/src/js/brim/cell.ts index 10f6b14a22..c94b3870b5 100644 --- a/src/js/brim/cell.ts +++ b/src/js/brim/cell.ts @@ -1,4 +1,4 @@ -import {ZedData, ZedPrimitive} from "zealot/zed/data-types" +import {ZedData, ZedPrimitive} from "zealot/zed" import {createComplexCell} from "./complexCell" import {createPrimitiveCell} from "./primitiveCell" diff --git a/src/js/brim/complexCell.ts b/src/js/brim/complexCell.ts index fdd0adcbaf..abc5cc7f34 100644 --- a/src/js/brim/complexCell.ts +++ b/src/js/brim/complexCell.ts @@ -5,7 +5,7 @@ import { ZedRecord, ZedSet, ZedUnion -} from "zealot/zed/data-types" +} from "zealot/zed" import {createCell} from "./cell" import {ONE_CHAR} from "./primitiveCell" diff --git a/src/js/brim/primitiveCell.ts b/src/js/brim/primitiveCell.ts index f263cfb852..61193763f9 100644 --- a/src/js/brim/primitiveCell.ts +++ b/src/js/brim/primitiveCell.ts @@ -1,5 +1,5 @@ import {isEqual} from "lodash" -import {ZedPrimitive} from "zealot/zed/data-types" +import {ZedPrimitive} from "zealot/zed" import {withCommas} from "../lib/fmt" import brim from "./" @@ -20,8 +20,8 @@ interface PrimitiveField { } export function createPrimitiveCell({name, data}: PrimitiveField) { - const type = data.kind - const value = data._value + const type = data.type + const value = data.value return { name, diff --git a/src/js/brim/program.test.ts b/src/js/brim/program.test.ts index d90fe3a8a8..0c8313e8aa 100644 --- a/src/js/brim/program.test.ts +++ b/src/js/brim/program.test.ts @@ -1,6 +1,5 @@ -import {COUNT, IP, STRING} from "test/fixtures/zjson-types" -import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" -import {RecordFieldType} from "zealot/zed/zjson" +import {createData, createField, createRecord} from "test/factories/record" +import {ZedField, ZedPrimitive} from "zealot/zed" import { addHeadProc, getHeadCount, @@ -13,12 +12,7 @@ import brim from "./" import {createCell} from "./cell" describe("excluding and including", () => { - const field = createCell( - new ZedField({ - name: "uid", - data: new ZedPrimitive({type: STRING, value: "123"}) - }) - ) + const field = createCell(createField("uid", "123")) test("excluding a field", () => { const program = brim @@ -30,7 +24,7 @@ describe("excluding and including", () => { }) test("excluding a field with a pipe", () => { - const data = new ZedPrimitive({type: STRING, value: "HTTP"}) + const data = createData("HTTP") const program = brim .program( 'tx_hosts=2606:4700:30::681c:135e fuid!="F2nyqx46YRDAYe4c73" | sort' @@ -63,13 +57,12 @@ describe("excluding and including", () => { }) describe("drill down", () => { - const columns = [ - {name: "id", type: {kind: "record", fields: [{name: "orig_h", type: IP}]}}, - {name: "proto", type: STRING}, - {name: "query", type: STRING}, - {name: "count", type: COUNT} - ] as RecordFieldType[] - const result = ZedRecord.of(columns, [["192.168.0.54"], "udp", "WPAD", "24"]) + const result = createRecord({ + id: {orig_h: "192.168.0.54"}, + proto: "udp", + query: "WPAD", + count: 24 + }) test("when there is no leading filter", () => { const program = brim @@ -110,13 +103,7 @@ describe("drill down", () => { }) test("count by and filter the same", () => { - const result = ZedRecord.of( - [ - {type: STRING, name: "md5"}, - {type: COUNT, name: "count"} - ], - ["123", "1"] - ) + const result = createRecord({md5: "123", count: 1}) const program = brim .program("md5=123 | count() by md5 | sort -r | head 5") @@ -127,13 +114,10 @@ describe("drill down", () => { }) test("filter query", () => { - const result = ZedRecord.of( - [ - {name: "md5", type: STRING}, - {name: "count", type: COUNT} - ], - ["9f51ef98c42df4430a978e4157c43dd5", "21"] - ) + const result = createRecord({ + md5: "9f51ef98c42df4430a978e4157c43dd5", + count: 21 + }) const program = brim .program( @@ -150,7 +134,7 @@ describe("drill down", () => { describe("count by", () => { test("empty program", () => { - const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const data = new ZedPrimitive({type: "string", value: "heyo"}) const field = createCell(new ZedField({name: "_path", data})) const program = brim .program() @@ -161,7 +145,7 @@ describe("count by", () => { }) test("append a count to an existing query", () => { - const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const data = new ZedPrimitive({type: "string", value: "heyo"}) const field = createCell(new ZedField({name: "query", data})) const program = brim .program("dns") diff --git a/src/js/brim/program.ts b/src/js/brim/program.ts index 4978613f7e..c11e55ef5b 100644 --- a/src/js/brim/program.ts +++ b/src/js/brim/program.ts @@ -1,6 +1,6 @@ import {isEqual} from "lodash" import {parse} from "zealot" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {trim} from "../lib/Str" import stdlib from "../stdlib" import brim from "./" diff --git a/src/js/components/ConnVersation.tsx b/src/js/components/ConnVersation.tsx index 07c874dfcf..28db63eb46 100644 --- a/src/js/components/ConnVersation.tsx +++ b/src/js/components/ConnVersation.tsx @@ -2,7 +2,7 @@ import contextMenu from "app/detail/flows/contextMenu" import {every} from "lodash" import React from "react" import {useDispatch} from "react-redux" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import connHistoryView from "../lib/connHistoryView" import VerticalTable from "./Tables/VerticalTable" import {Fieldset} from "./Typography" @@ -15,16 +15,14 @@ type Props = { } function filter(record: ZedRecord, names: string[]) { - const cols = [] - const vals = [] + const fields = [] names.forEach((n) => { - const i = record.columns.indexOf(n) - cols.push(record._type[i]) - vals.push(record._value[i]) + const field = record.tryField(n) + if (field) fields.push(field) }) - return ZedRecord.of(cols, vals) + return new ZedRecord({fields}) } const ConnVersation = ({record}: Props) => { diff --git a/src/js/components/FieldCell.tsx b/src/js/components/FieldCell.tsx index 0f06fa46f1..3e15576e8f 100644 --- a/src/js/components/FieldCell.tsx +++ b/src/js/components/FieldCell.tsx @@ -1,6 +1,7 @@ +import {typeClassNames} from "app/core/utils/type-class-names" import classNames from "classnames" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import {createCell} from "../brim/cell" type Props = {field: ZedField; record: ZedRecord} @@ -22,11 +23,8 @@ export default function FieldCell({field, record}: Props) { className={classNames( "field-cell", field.name, - field.data.kind, - getBackground(field, record), - { - null: field.data.isUnset() - } + typeClassNames(field.data), + getBackground(field, record) )} > {cell.display()} diff --git a/src/js/components/LogCell/CompoundField.tsx b/src/js/components/LogCell/CompoundField.tsx index 9340191c26..714e868ab8 100644 --- a/src/js/components/LogCell/CompoundField.tsx +++ b/src/js/components/LogCell/CompoundField.tsx @@ -1,6 +1,6 @@ import classNames from "classnames" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import {createComplexCell} from "../../brim/complexCell" import SingleField from "./SingleField" diff --git a/src/js/components/LogCell/SingleField.tsx b/src/js/components/LogCell/SingleField.tsx index e6ad43b97e..af71471705 100644 --- a/src/js/components/LogCell/SingleField.tsx +++ b/src/js/components/LogCell/SingleField.tsx @@ -1,6 +1,6 @@ import classNames from "classnames" import React, {useEffect, useRef, useState} from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import {$Menu} from "../../electron/menu" import lib from "../../lib" import {showContextMenu} from "../../lib/System" diff --git a/src/js/components/LogCell/index.tsx b/src/js/components/LogCell/index.tsx index 2b84f936b4..2f18605902 100644 --- a/src/js/components/LogCell/index.tsx +++ b/src/js/components/LogCell/index.tsx @@ -1,6 +1,7 @@ +import {typeClassNames} from "app/core/utils/type-class-names" import classNames from "classnames" import React, {useState} from "react" -import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" import {RightClickBuilder} from "../../types" import Tooltip from "../Tooltip" import CompoundField from "./CompoundField" @@ -35,7 +36,7 @@ export default function LogCell({field, style, rightClick, log}: Props) { } return (
{ store.dispatch(Layout.showRightSidebar()) store.dispatch(tabHistory.push(workspacesPath())) - store.dispatch(LogDetails.push(ZedRecord.of([], []))) + store.dispatch(LogDetails.push(new ZedRecord({fields: []}))) const el = provide(store, ) expect(el.html()).toBe("") }) diff --git a/src/js/components/RightPane.tsx b/src/js/components/RightPane.tsx index 0864834e7d..45ab7aac79 100644 --- a/src/js/components/RightPane.tsx +++ b/src/js/components/RightPane.tsx @@ -1,7 +1,7 @@ import DetailPane from "app/detail/Pane" import React from "react" import {connect} from "react-redux" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {openLogDetailsWindow} from "../flows/openLogDetailsWindow" import ExpandWindow from "../icons/ExpandWindow" import dispatchToProps from "../lib/dispatchToProps" diff --git a/src/js/components/SearchResults/ResultsTable.tsx b/src/js/components/SearchResults/ResultsTable.tsx index 334515f8aa..94c9b3acd0 100644 --- a/src/js/components/SearchResults/ResultsTable.tsx +++ b/src/js/components/SearchResults/ResultsTable.tsx @@ -3,7 +3,7 @@ import {isEmpty} from "lodash" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" import React, {useEffect} from "react" import {connect, useDispatch} from "react-redux" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {openLogDetailsWindow} from "../../flows/openLogDetailsWindow" import {viewLogDetail} from "../../flows/viewLogDetail" import dispatchToProps from "../../lib/dispatchToProps" diff --git a/src/js/components/Tables/HorizontalTable.tsx b/src/js/components/Tables/HorizontalTable.tsx index 7be6c89305..3b70d39f65 100644 --- a/src/js/components/Tables/HorizontalTable.tsx +++ b/src/js/components/Tables/HorizontalTable.tsx @@ -1,5 +1,5 @@ import * as React from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import Table, {TableData, TableHeader} from "./Table" type Props = { diff --git a/src/js/components/Tables/Table.tsx b/src/js/components/Tables/Table.tsx index f2ca786f1b..62160e453a 100644 --- a/src/js/components/Tables/Table.tsx +++ b/src/js/components/Tables/Table.tsx @@ -1,5 +1,6 @@ +import {typeClassNames} from "app/core/utils/type-class-names" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import FieldCell from "../FieldCell" export default function Table({className, ...props}: any) { @@ -7,7 +8,7 @@ export default function Table({className, ...props}: any) { } export function TableHeader({column}: {column: ZedField}) { - return
+ return } type Props = { @@ -24,7 +25,10 @@ export function TableData({field, record, onRightClick}: Props) { } return ( - ) diff --git a/src/js/components/Tables/VerticalTable.tsx b/src/js/components/Tables/VerticalTable.tsx index 9abf866c3a..e0e303b229 100644 --- a/src/js/components/Tables/VerticalTable.tsx +++ b/src/js/components/Tables/VerticalTable.tsx @@ -1,6 +1,6 @@ import classNames from "classnames" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedRecord} from "zealot/zed" import Table, {TableData, TableHeader} from "./Table" type Props = { diff --git a/src/js/components/Viewer/Chunk.tsx b/src/js/components/Viewer/Chunk.tsx index d9626e4363..43dcc3e75d 100644 --- a/src/js/components/Viewer/Chunk.tsx +++ b/src/js/components/Viewer/Chunk.tsx @@ -1,5 +1,5 @@ import React from "react" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import TableColumns from "../../models/TableColumns" import {RowRenderer, ViewerDimens} from "../../types" import * as Styler from "./Styler" diff --git a/src/js/components/Viewer/Viewer.tsx b/src/js/components/Viewer/Viewer.tsx index 0906959073..ad020704e3 100644 --- a/src/js/components/Viewer/Viewer.tsx +++ b/src/js/components/Viewer/Viewer.tsx @@ -1,5 +1,5 @@ import React, {useEffect, useRef, useState} from "react" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import lib from "../../lib" import ScrollHooks from "../../lib/ScrollHooks" import TableColumns from "../../models/TableColumns" diff --git a/src/js/electron/menu/actions/detailActions.ts b/src/js/electron/menu/actions/detailActions.ts index 3066abe896..ef00667e32 100644 --- a/src/js/electron/menu/actions/detailActions.ts +++ b/src/js/electron/menu/actions/detailActions.ts @@ -1,3 +1,14 @@ +import { + ZedField, + SerializedZedField, + ZedPrimitive, + ZedRecord, + SerializedZedRecord +} from "zealot/zed" +import brim from "../../../brim" +import {createCell} from "../../../brim/cell" +import {downloadPcap} from "../../../flows/downloadPcap" +import {openNewSearchTab} from "../../../flows/openNewSearchWindow" import { appendQueryCountBy, appendQueryExclude, @@ -6,32 +17,21 @@ import { appendQueryNotIn, appendQuerySortBy } from "../../../flows/searchBar/actions" +import {viewLogDetail} from "../../../flows/viewLogDetail" import lib from "../../../lib" import open from "../../../lib/open" -import {viewLogDetail} from "../../../flows/viewLogDetail" +import virusTotal from "../../../services/virusTotal" import Modal from "../../../state/Modal" import SearchBar from "../../../state/SearchBar" -import action from "./action" -import brim from "../../../brim" import tab from "../../../state/Tab" -import virusTotal from "../../../services/virusTotal" -import {downloadPcap} from "../../../flows/downloadPcap" -import {openNewSearchTab} from "../../../flows/openNewSearchWindow" -import {createCell} from "../../../brim/cell" -import { - ZedField, - ZedRecordSpec, - ZedRecord, - ZedPrimitive, - ZedFieldSpec -} from "zealot/zed/data-types" +import action from "./action" function buildDetailActions() { return { copy: action({ name: "detail-cell-menu-copy", label: "Copy", - listener(_dispatch, data: ZedFieldSpec) { + listener(_dispatch, data: SerializedZedField) { const f = ZedField.deserialize(data) lib.doc.copyToClipboard(f.data.toString()) } @@ -39,7 +39,7 @@ function buildDetailActions() { countBy: action({ name: "detail-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQueryCountBy(ZedField.deserialize(data))) dispatch(openNewSearchTab()) @@ -48,14 +48,14 @@ function buildDetailActions() { detail: action({ name: "detail-cell-menu-detail", label: "View details", - listener(dispatch, log: ZedRecordSpec) { + listener(dispatch, log: SerializedZedRecord) { dispatch(viewLogDetail(ZedRecord.deserialize(log))) } }), exclude: action({ name: "detail-cell-menu-exclude", label: "Filter != value in new search", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQueryExclude(ZedField.deserialize(field))) dispatch(openNewSearchTab()) @@ -64,7 +64,7 @@ function buildDetailActions() { freshInclude: action({ name: "detail-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { const cell = createCell(ZedField.deserialize(field)) dispatch(SearchBar.clearSearchBar()) dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) @@ -74,13 +74,12 @@ function buildDetailActions() { fromTime: action({ name: "detail-cell-menu-from-time", label: 'Use as "start" time in new search', - listener(dispatch, data: ZedFieldSpec) { - const field = ZedField.deserialize(data) - if (field.data.kind === "time") { + listener(dispatch, fieldJSON: SerializedZedField) { + const field = ZedField.deserialize(fieldJSON) + const data = field.data + if (data instanceof ZedPrimitive && data.type === "time") { dispatch(SearchBar.clearSearchBar()) - dispatch( - tab.setFrom(brim.time((field.data as ZedPrimitive).toDate()).toTs()) - ) + dispatch(tab.setFrom(brim.time(data.toDate()).toTs())) dispatch(openNewSearchTab()) } } @@ -88,7 +87,7 @@ function buildDetailActions() { groupByDrillDown: action({ name: "detail-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program, log: ZedRecordSpec) { + listener(dispatch, program, log: SerializedZedRecord) { const newProgram = brim .program(program) .drillDown(ZedRecord.deserialize(log)) @@ -104,7 +103,7 @@ function buildDetailActions() { include: action({ name: "detail-cell-menu-include", label: "Filter = value in new search", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQueryInclude(ZedField.deserialize(field))) dispatch(openNewSearchTab()) @@ -113,7 +112,7 @@ function buildDetailActions() { in: action({ name: "detail-cell-menu-in", label: "Filter in field in new search", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQueryIn(createCell(ZedField.deserialize(field)))) dispatch(openNewSearchTab()) @@ -122,7 +121,7 @@ function buildDetailActions() { notIn: action({ name: "detail-cell-menu-not-in", label: "Filter not in field in new search", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQueryNotIn(createCell(ZedField.deserialize(field)))) dispatch(openNewSearchTab()) @@ -139,14 +138,14 @@ function buildDetailActions() { pcaps: action({ name: "detail-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, log: ZedRecordSpec) { + listener(dispatch, log: SerializedZedRecord) { dispatch(downloadPcap(ZedRecord.deserialize(log))) } }), sortAsc: action({ name: "detail-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQuerySortBy(field.name, "asc")) dispatch(openNewSearchTab()) @@ -155,7 +154,7 @@ function buildDetailActions() { sortDesc: action({ name: "detail-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, field: ZedFieldSpec) { + listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) dispatch(appendQuerySortBy(field.name, "desc")) dispatch(openNewSearchTab()) @@ -164,14 +163,14 @@ function buildDetailActions() { toTime: action({ name: "detail-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) - if (field.data.kind === "time") { + if (field.data instanceof ZedPrimitive && field.data.type === "time") { dispatch(SearchBar.clearSearchBar()) dispatch( tab.setTo( brim - .time((field.data as ZedPrimitive).toDate()) + .time(field.data.toDate()) .add(1, "ms") .toTs() ) @@ -183,7 +182,7 @@ function buildDetailActions() { virusTotalRightclick: action({ name: "detail-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(_dispatch, data: ZedFieldSpec) { + listener(_dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) open(virusTotal.url(field.data.toString())) } @@ -191,7 +190,7 @@ function buildDetailActions() { whoisRightclick: action({ name: "detail-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) dispatch(Modal.show("whois", {addr: field.data.toString()})) } diff --git a/src/js/electron/menu/actions/searchActions.ts b/src/js/electron/menu/actions/searchActions.ts index 87d35b5881..13fc9c6d40 100644 --- a/src/js/electron/menu/actions/searchActions.ts +++ b/src/js/electron/menu/actions/searchActions.ts @@ -1,11 +1,11 @@ import lib from "src/js/lib" import { + SerializedZedField, + SerializedZedRecord, ZedField, - ZedFieldSpec, ZedPrimitive, - ZedRecord, - ZedRecordSpec -} from "zealot/zed/data-types" + ZedRecord +} from "zealot/zed" import brim from "../../../brim" import {createCell} from "../../../brim/cell" import {downloadPcap} from "../../../flows/downloadPcap" @@ -35,7 +35,7 @@ function buildSearchActions() { copy: action({ name: "search-cell-menu-copy", label: "Copy", - listener(_dispatch, data: ZedFieldSpec) { + listener(_dispatch, data: SerializedZedField) { const f = ZedField.deserialize(data) lib.doc.copyToClipboard(f.data.toString()) } @@ -43,7 +43,7 @@ function buildSearchActions() { countBy: action({ name: "search-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const f = ZedField.deserialize(data) dispatch(appendQueryCountBy(f)) dispatch(submitSearch()) @@ -52,7 +52,7 @@ function buildSearchActions() { detail: action({ name: "search-cell-menu-detail", label: "Open details", - listener(dispatch, data: ZedRecordSpec) { + listener(dispatch, data: SerializedZedRecord) { const record = ZedRecord.deserialize(data) dispatch(Layout.showRightSidebar()) dispatch(viewLogDetail(record)) @@ -61,7 +61,7 @@ function buildSearchActions() { exclude: action({ name: "search-cell-menu-exclude", label: "Filter != value", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { dispatch(appendQueryExclude(ZedField.deserialize(data))) dispatch(submitSearch()) } @@ -69,7 +69,7 @@ function buildSearchActions() { freshInclude: action({ name: "search-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const cell = createCell(ZedField.deserialize(data)) dispatch(SearchBar.clearSearchBar()) dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) @@ -79,12 +79,10 @@ function buildSearchActions() { fromTime: action({ name: "search-cell-menu-from-time", label: 'Use as "start" time', - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) - if (field.data.kind === "time") { - dispatch( - tab.setFrom(brim.time((field.data as ZedPrimitive).toDate()).toTs()) - ) + if (field.data instanceof ZedPrimitive && field.data.type === "time") { + dispatch(tab.setFrom(brim.time(field.data.toDate()).toTs())) dispatch(submitSearch()) } } @@ -92,7 +90,7 @@ function buildSearchActions() { groupByDrillDown: action({ name: "search-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program: string, data: ZedRecordSpec) { + listener(dispatch, program: string, data: SerializedZedRecord) { const record = ZedRecord.deserialize(data) const newProgram = brim .program(program) @@ -109,7 +107,7 @@ function buildSearchActions() { include: action({ name: "search-cell-menu-include", label: "Filter = value", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { dispatch(appendQueryInclude(ZedField.deserialize(data))) dispatch(submitSearch()) } @@ -117,7 +115,7 @@ function buildSearchActions() { in: action({ name: "search-cell-menu-in", label: "Filter in field", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { dispatch(appendQueryIn(createCell(ZedField.deserialize(data)))) dispatch(submitSearch()) } @@ -125,11 +123,15 @@ function buildSearchActions() { jumpToTime: action({ name: "search-cell-menu-show-context", label: "View in full context", - listener(dispatch, fieldData: ZedFieldSpec, recordData: ZedRecordSpec) { + listener( + dispatch, + fieldData: SerializedZedField, + recordData: SerializedZedRecord + ) { const field = ZedField.deserialize(fieldData) const record = ZedRecord.deserialize(recordData) - const brimTime = brim.time((field.data as ZedPrimitive).toDate()) - if (field.data.kind === "time") { + if (field.data instanceof ZedPrimitive && field.data.type === "time") { + const brimTime = brim.time(field.data.toDate()) dispatch(tab.setFrom(brimTime.subtract(1, "minutes").toTs())) dispatch(tab.setTo(brimTime.add(1, "minutes").toTs())) dispatch(SearchBar.clearSearchBar()) @@ -147,7 +149,7 @@ function buildSearchActions() { notIn: action({ name: "search-cell-menu-not-in", label: "Filter not in field", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { dispatch(appendQueryNotIn(createCell(ZedField.deserialize(data)))) dispatch(submitSearch()) } @@ -155,7 +157,7 @@ function buildSearchActions() { logResult: action({ name: "search-cell-menu-log-result", label: "Log result to console", - listener(_dispatch, field: ZedFieldSpec, log: ZedRecordSpec) { + listener(_dispatch, field: SerializedZedField, log: SerializedZedRecord) { console.log(JSON.stringify(log)) console.log(JSON.stringify(field)) } @@ -163,14 +165,14 @@ function buildSearchActions() { pcaps: action({ name: "search-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, data: ZedRecordSpec) { + listener(dispatch, data: SerializedZedRecord) { dispatch(downloadPcap(ZedRecord.deserialize(data))) } }), sortAsc: action({ name: "search-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) dispatch(appendQuerySortBy(field.name, "asc")) dispatch(submitSearch()) @@ -179,7 +181,7 @@ function buildSearchActions() { sortDesc: action({ name: "search-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) dispatch(appendQuerySortBy(field.name, "desc")) dispatch(submitSearch()) @@ -188,9 +190,9 @@ function buildSearchActions() { toTime: action({ name: "search-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) - if (field.data.kind === "time") { + if (field.data instanceof ZedPrimitive && field.data.type === "time") { dispatch( tab.setTo( brim @@ -206,7 +208,7 @@ function buildSearchActions() { virusTotalRightclick: action({ name: "search-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) if (field.data instanceof ZedPrimitive && !field.data.isUnset()) { open(virusTotal.url(field.data.toString() as string)) @@ -216,7 +218,7 @@ function buildSearchActions() { whoisRightclick: action({ name: "search-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: ZedFieldSpec) { + listener(dispatch, data: SerializedZedField) { const field = ZedField.deserialize(data) dispatch(Modal.show("whois", {addr: field.data.toString()})) } diff --git a/src/js/flows/downloadPcap.ts b/src/js/flows/downloadPcap.ts index 1d32ed85c4..542b385a50 100644 --- a/src/js/flows/downloadPcap.ts +++ b/src/js/flows/downloadPcap.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import open from "../lib/open" import Packets from "../state/Packets" import {Thunk} from "../state/types" diff --git a/src/js/flows/rightclick/cellMenu.test.ts b/src/js/flows/rightclick/cellMenu.test.ts index c825e7ff3c..7629130d69 100644 --- a/src/js/flows/rightclick/cellMenu.test.ts +++ b/src/js/flows/rightclick/cellMenu.test.ts @@ -1,18 +1,15 @@ import {MenuItemConstructorOptions} from "electron" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" -import {COUNT, IP, STRING, TIME} from "test/fixtures/zjson-types" -import {ZedRecord} from "zealot/zed/data-types" +import {createRecord} from "test/factories/record" import fixtures from "../../test/fixtures" -const conn = ZedRecord.of( - [ - {name: "id", type: {kind: "record", fields: [{name: "orig_h", type: IP}]}}, - {name: "_path", type: STRING}, - {name: "ts", type: TIME} - ], - [["192.168.0.1"], "conn", "1234513"] -) -const dns = ZedRecord.of([{name: "query", type: STRING}], ["dns.query.yo"]) +const conn = createRecord({ + _path: "conn", + id: {orig_h: "192.168.0.1"}, + ts: new Date(1234513 * 1000) +}) + +const dns = createRecord({query: "dns.query.yo"}) function menuText(menu: MenuItemConstructorOptions[]) { return menu @@ -83,16 +80,10 @@ describe("Analysis Right Click", () => { const columnNames = ["count", "id.orig_h"] test("nested field", () => { - const log = ZedRecord.of( - [ - {name: "count", type: COUNT}, - { - name: "id", - type: {kind: "record", fields: [{name: "orig_h", type: IP}]} - } - ], - ["300", ["192.168.0.51"]] - ) + const log = createRecord({ + count: 300, + id: {orig_h: "192.168.0.51"} + }) const field = log.getField("id.orig_h") const ctxMenu = searchFieldContextMenu(program, columnNames, space)( field, @@ -104,13 +95,7 @@ describe("Analysis Right Click", () => { }) test("non-address field", () => { - const log = ZedRecord.of( - [ - {name: "count", type: COUNT}, - {name: "proto", type: STRING} - ], - ["100", "tcp"] - ) + const log = createRecord({count: 100, proto: "tcp"}) const field = log.getField("proto") const ctxMenu = searchFieldContextMenu( "* | count() by proto", diff --git a/src/js/flows/scrollToLog.ts b/src/js/flows/scrollToLog.ts index 5786337aad..8fce624c88 100644 --- a/src/js/flows/scrollToLog.ts +++ b/src/js/flows/scrollToLog.ts @@ -1,5 +1,5 @@ import {isEqual} from "lodash" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {Thunk} from "../state/types" import Viewer from "../state/Viewer" diff --git a/src/js/flows/searchBar/actions.ts b/src/js/flows/searchBar/actions.ts index b7849c9f65..2b8a585dff 100644 --- a/src/js/flows/searchBar/actions.ts +++ b/src/js/flows/searchBar/actions.ts @@ -1,4 +1,4 @@ -import {ZedField} from "zealot/zed/data-types" +import {ZedField} from "zealot/zed" import brim from "../../brim" import {Cell, createCell} from "../../brim/cell" import {onlyWhitespace} from "../../lib/Str" diff --git a/src/js/flows/viewLogDetail.ts b/src/js/flows/viewLogDetail.ts index 7499a180ab..7cdd522e1e 100644 --- a/src/js/flows/viewLogDetail.ts +++ b/src/js/flows/viewLogDetail.ts @@ -1,6 +1,6 @@ import {isEqual} from "lodash" import {fetchCorrelation} from "ppl/detail/flows/fetch" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import ErrorFactory from "../models/ErrorFactory" import Current from "../state/Current" import LogDetails from "../state/LogDetails" diff --git a/src/js/models/TableColumns.ts b/src/js/models/TableColumns.ts index 0e0121dcae..ad99786265 100644 --- a/src/js/models/TableColumns.ts +++ b/src/js/models/TableColumns.ts @@ -1,4 +1,4 @@ -import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" import {createCell} from "../brim/cell" import columnOrder from "../lib/columnOrder" import {$Column} from "../state/Columns/models/column" @@ -42,7 +42,7 @@ export default class TableColumns { new ZedField({ name: "", data: new ZedPrimitive({ - type: {kind: "primitive", name: "string"}, + type: "string", value: col.name }) }) diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index ae9e2b8489..9a0d6f9c20 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -1,26 +1,14 @@ -import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" -import {ZedPrimitive, ZedRecord, ZedRecordSpec} from "zealot/zed/data-types" +import {createRecord} from "test/factories/record" +import {ZedPrimitive} from "zealot/zed" import {connCorrelation} from "./programs" test("conn correlation", () => { - const conn: ZedRecordSpec = { - type: { - kind: "record", - fields: [ - {name: "ts", type: TIME}, - {name: "uid", type: STRING}, - {name: "duration", type: INTERVAL}, - {name: "community_id", type: STRING} - ] - }, - value: [ - "1425568032.998178", - "CbOjYpkXn9LfqV51c", - "0.70995", - "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" - ] - } - const record = ZedRecord.deserialize(conn) + const record = createRecord({ + ts: new Date(1425568032.998178 * 1000), + uid: "CbOjYpkXn9LfqV51c", + duration: 0.70995, + community_id: "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" + }) expect( connCorrelation( diff --git a/src/js/searches/programs.ts b/src/js/searches/programs.ts index 357ec19c8b..14912f046e 100644 --- a/src/js/searches/programs.ts +++ b/src/js/searches/programs.ts @@ -1,4 +1,4 @@ -import {ZedPrimitive} from "zealot/zed/data-types" +import {ZedPrimitive} from "zealot/zed" import zql from "../zql" export function md5Correlation(md5: string) { diff --git a/src/js/state/Chart/actions.ts b/src/js/state/Chart/actions.ts index 22babe5289..9c0fadd36a 100644 --- a/src/js/state/Chart/actions.ts +++ b/src/js/state/Chart/actions.ts @@ -1,4 +1,4 @@ -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed" import MergeHash from "../../models/MergeHash" import UniqArray from "../../models/UniqArray" import {SearchStatus} from "../../types/searches" diff --git a/src/js/state/Chart/test.ts b/src/js/state/Chart/test.ts index 793d6491cc..980fe74c4f 100644 --- a/src/js/state/Chart/test.ts +++ b/src/js/state/Chart/test.ts @@ -1,6 +1,4 @@ -import {COUNT, STRING, TIME} from "test/fixtures/zjson-types" -import {ZedRecord} from "zealot/zed/data-types" -import {RecordFieldType} from "zealot/zed/zjson" +import {createRecord} from "test/factories/record" import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import chart from "./" @@ -11,15 +9,9 @@ beforeEach(() => { tabId = Tabs.getActive(store.getState()) }) -const fields = [ - {name: "ts", type: TIME}, - {name: "_path", type: STRING}, - {name: "count", type: COUNT} -] as RecordFieldType[] - const records = [ - ZedRecord.of(fields, ["0", "conn", "500"]), - ZedRecord.of(fields, ["100", "dns", "300"]) + createRecord({ts: new Date(0), _path: "conn", count: 500}), + createRecord({ts: new Date(100 * 1000), _path: "dns", count: 300}) ] test("chart records append", () => { diff --git a/src/js/state/Columns/models/columnSet.ts b/src/js/state/Columns/models/columnSet.ts index 93dca4c80c..a16e632295 100644 --- a/src/js/state/Columns/models/columnSet.ts +++ b/src/js/state/Columns/models/columnSet.ts @@ -5,17 +5,19 @@ import {$Column, createColumn} from "./column" export function createColumnSet(c: TypeContext) { return { getName() { - if (c.size === 0) { + const keys = Object.keys(c) + const size = keys.length + if (size === 0) { return "none" - } else if (c.size === 1) { - return Array.from(c.keys())[0] + } else if (size === 1) { + return keys[0] } else { return "temp" } }, getUniqColumns() { let allCols = [] - for (const typedef of c.values()) { + for (const typedef of Object.values(c)) { let inner = typedef.flatten().innerType if (inner.kind === "record") { allCols = [...allCols, ...inner.fields] diff --git a/src/js/state/Columns/selectors.ts b/src/js/state/Columns/selectors.ts index 958e1b42de..f6d2badddc 100644 --- a/src/js/state/Columns/selectors.ts +++ b/src/js/state/Columns/selectors.ts @@ -1,5 +1,5 @@ import {createSelector} from "reselect" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import TableColumns from "../../models/TableColumns" import activeTabSelect from "../Tab/activeTabSelect" import {State} from "../types" diff --git a/src/js/state/Columns/touch.test.ts b/src/js/state/Columns/touch.test.ts index c35fb33b4d..34f33f977b 100644 --- a/src/js/state/Columns/touch.test.ts +++ b/src/js/state/Columns/touch.test.ts @@ -1,41 +1,39 @@ -import {createColumn} from "./models/column" +import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" +import ZedTypeDef from "zealot/zed/type-def" +import initTestStore from "../../test/initTestStore" import Columns from "./" import actions from "./actions" -import initTestStore from "../../test/initTestStore" +import {createColumn} from "./models/column" import touch from "./touch" -import ZedTypeDef from "zealot/zed/type-def" -import {STRING, INTERVAL, TIME} from "test/fixtures/zjson-types" -const columns = new Map( - Object.entries({ - "1": new ZedTypeDef({ +const columns = { + "1": new ZedTypeDef({ + type: { + name: "1", + kind: "typedef", type: { - name: "1", - kind: "typedef", - type: { - kind: "record", - fields: [ - {name: "_path", type: STRING}, - {name: "duration", type: INTERVAL} - ] - } + kind: "record", + fields: [ + {name: "_path", type: STRING}, + {name: "duration", type: INTERVAL} + ] } - }), - "2": new ZedTypeDef({ + } + }), + "2": new ZedTypeDef({ + type: { + name: "2", + kind: "typedef", type: { - name: "2", - kind: "typedef", - type: { - kind: "record", - fields: [ - {name: "_path", type: STRING}, - {name: "ts", type: TIME} - ] - } + kind: "record", + fields: [ + {name: "_path", type: STRING}, + {name: "ts", type: TIME} + ] } - }) + } }) -) +} let store beforeEach(() => { diff --git a/src/js/state/LogDetails/actions.ts b/src/js/state/LogDetails/actions.ts index 1737a0a655..02a0718b81 100644 --- a/src/js/state/LogDetails/actions.ts +++ b/src/js/state/LogDetails/actions.ts @@ -1,5 +1,5 @@ import {SearchStatus} from "src/js/types/searches" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import { LOG_DETAIL_BACK, LOG_DETAIL_CLEAR, diff --git a/src/js/state/LogDetails/selectors.ts b/src/js/state/LogDetails/selectors.ts index 13b561e232..9699364480 100644 --- a/src/js/state/LogDetails/selectors.ts +++ b/src/js/state/LogDetails/selectors.ts @@ -6,7 +6,7 @@ import {State} from "../types" import {LogDetailHistory, toHistory} from "./reducer" import {LogDetailsState} from "./types" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" const getLogDetails = activeTabSelect((state: TabState) => { return state.logDetails diff --git a/src/js/state/LogDetails/test.ts b/src/js/state/LogDetails/test.ts index 6b4bb13243..6d3ed80134 100644 --- a/src/js/state/LogDetails/test.ts +++ b/src/js/state/LogDetails/test.ts @@ -1,19 +1,10 @@ -import {ZedRecord} from "zealot/zed/data-types" -import {RecordType} from "zealot/zed/zjson" +import {createRecord} from "test/factories/record" import initTestStore from "../../test/initTestStore" import LogDetails from "./" -const type = { - kind: "record", - fields: [ - {name: "_td", type: {kind: "primitive", name: "count"}}, - {name: "letter", type: {kind: "primitive", name: "string"}} - ] -} as RecordType - -const record = new ZedRecord({type, value: ["1", "a"]}) -const record2 = new ZedRecord({type, value: ["1", "b"]}) -const record3 = new ZedRecord({type, value: ["1", "c"]}) +const record = createRecord({_td: "1", letter: "a"}) +const record2 = createRecord({_td: "1", letter: "b"}) +const record3 = createRecord({_td: "1", letter: "c"}) let store beforeEach(() => { diff --git a/src/js/state/LogDetails/types.ts b/src/js/state/LogDetails/types.ts index b720f68ac8..6b9471a8f3 100644 --- a/src/js/state/LogDetails/types.ts +++ b/src/js/state/LogDetails/types.ts @@ -1,4 +1,4 @@ -import {ZedRecordSpec} from "zealot/zed/data-types" +import {SerializedZedRecord} from "zealot/zed" import {SearchStatus} from "../../types/searches" export type LogDetailsState = { @@ -7,8 +7,8 @@ export type LogDetailsState = { } export type LogDetails = { - log: ZedRecordSpec - uidLogs: ZedRecordSpec[] + log: SerializedZedRecord + uidLogs: SerializedZedRecord[] uidStatus: SearchStatus } @@ -21,7 +21,7 @@ export type LogDetailsAction = export type LOG_DETAIL_PUSH = { type: "LOG_DETAIL_PUSH" - record: ZedRecordSpec + record: SerializedZedRecord } export type LOG_DETAIL_UPDATE = { diff --git a/src/js/state/Packets/flows.ts b/src/js/state/Packets/flows.ts index d8deb6ea68..823a9a6f16 100644 --- a/src/js/state/Packets/flows.ts +++ b/src/js/state/Packets/flows.ts @@ -1,6 +1,6 @@ import {remote} from "electron" import {join} from "path" -import {ZedPrimitive, ZedRecord} from "zealot/zed/data-types" +import {ZedPrimitive, ZedRecord} from "zealot/zed" import {getZealot} from "../../flows/getZealot" import {saveToFile} from "../../lib/response" import Current from "../Current" diff --git a/src/js/state/SearchBar/test.ts b/src/js/state/SearchBar/test.ts index 50ea8b005e..b82459181c 100644 --- a/src/js/state/SearchBar/test.ts +++ b/src/js/state/SearchBar/test.ts @@ -17,8 +17,7 @@ import Url from "../Url" import Workspaces from "../Workspaces" import SearchBar from "./" import {SearchBarState} from "./types" -import {ZedPrimitive, ZedField} from "zealot/zed/data-types" -import {STRING} from "test/fixtures/zjson-types" +import {ZedPrimitive, ZedField} from "zealot/zed" let store, mock beforeEach(() => { @@ -129,7 +128,7 @@ test("search bar pin remove when out of bounds", () => { }) test("append an include field", () => { - const data = new ZedPrimitive({type: STRING, value: "conn"}) + const data = new ZedPrimitive({type: "string", value: "conn"}) const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryInclude(field)]) @@ -137,7 +136,7 @@ test("append an include field", () => { }) test("append an include field when some text already exists", () => { - const data = new ZedPrimitive({type: STRING, value: "conn"}) + const data = new ZedPrimitive({type: "string", value: "conn"}) const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), @@ -147,14 +146,14 @@ test("append an include field when some text already exists", () => { }) test("append an exclude field", () => { - const data = new ZedPrimitive({type: STRING, value: "conn"}) + const data = new ZedPrimitive({type: "string", value: "conn"}) const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryExclude(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe('_path!="conn"') }) test("append an exclude field when some text already exists", () => { - const data = new ZedPrimitive({type: STRING, value: "conn"}) + const data = new ZedPrimitive({type: "string", value: "conn"}) const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), @@ -164,14 +163,14 @@ test("append an exclude field when some text already exists", () => { }) test("append a count by field", () => { - const data = new ZedPrimitive({type: STRING, value: "conn"}) + const data = new ZedPrimitive({type: "string", value: "conn"}) const field = new ZedField({name: "_path", data}) const state = store.dispatchAll([appendQueryCountBy(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe("* | count() by _path") }) test("append a count to an existing query", () => { - const data = new ZedPrimitive({type: STRING, value: "ho ho"}) + const data = new ZedPrimitive({type: "string", value: "ho ho"}) const field = new ZedField({name: "query", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), @@ -181,7 +180,7 @@ test("append a count to an existing query", () => { }) test("append a count to an existing query with a pin", () => { - const data = new ZedPrimitive({type: STRING, value: "heyo"}) + const data = new ZedPrimitive({type: "string", value: "heyo"}) const field = new ZedField({name: "query", data}) const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), diff --git a/src/js/state/Viewer/actions.ts b/src/js/state/Viewer/actions.ts index 0274c5424d..823e3e7e8e 100644 --- a/src/js/state/Viewer/actions.ts +++ b/src/js/state/Viewer/actions.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" import { diff --git a/src/js/state/Viewer/reducer.ts b/src/js/state/Viewer/reducer.ts index 9c336f46f8..e9f040ebd2 100644 --- a/src/js/state/Viewer/reducer.ts +++ b/src/js/state/Viewer/reducer.ts @@ -8,7 +8,7 @@ const init = (): ViewerState => ({ records: [], endStatus: "INCOMPLETE", status: "INIT", - columns: new Map(), + columns: {}, scrollPos: {x: 0, y: 0}, selection: { rows: {}, @@ -34,7 +34,7 @@ export default function( case "VIEWER_STATUS": return {...state, status: action.status} case "VIEWER_COLUMNS": - return {...state, columns: new Map([...state.columns, ...action.columns])} + return {...state, columns: {...state.columns, ...action.columns}} case "VIEWER_SET_COLUMNS": return {...state, columns: action.columns} case "VIEWER_SCROLL": diff --git a/src/js/state/Viewer/selectors.ts b/src/js/state/Viewer/selectors.ts index 8aebbd72c5..d7ae630365 100644 --- a/src/js/state/Viewer/selectors.ts +++ b/src/js/state/Viewer/selectors.ts @@ -1,7 +1,7 @@ import {createSelector} from "reselect" import {ScrollPosition} from "src/js/types" import {SearchStatus} from "src/js/types/searches" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {TabState} from "../Tab/types" import Tabs from "../Tabs" import {State} from "../types" diff --git a/src/js/state/Viewer/test.ts b/src/js/state/Viewer/test.ts index abafba2564..acd347ea35 100644 --- a/src/js/state/Viewer/test.ts +++ b/src/js/state/Viewer/test.ts @@ -1,7 +1,5 @@ -import {STRING} from "test/fixtures/zjson-types" -import {ZedRecord} from "zealot/zed/data-types" +import {createRecord} from "test/factories/record" import ZedTypeDef from "zealot/zed/type-def" -import {RecordType} from "zealot/zed/zjson" import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import Viewer from "../Viewer" @@ -14,21 +12,9 @@ beforeEach(() => { tabId = Tabs.getActive(store.getState()) }) -const type = { - kind: "record", - fields: [ - { - name: "ts", - type: { - kind: "primitive", - name: "time" - } - } - ] -} as RecordType -const conn = new ZedRecord({type, value: ["1"]}) -const dns = new ZedRecord({type, value: ["2"]}) -const http = new ZedRecord({type, value: ["3"]}) +const conn = createRecord({ts: new Date(1000)}) +const dns = createRecord({ts: new Date(2000)}) +const http = createRecord({ts: new Date(3000)}) test("adding logs to the viewer", () => { const state = store.dispatchAll([ @@ -80,33 +66,41 @@ test("results limited", () => { test("update columns with same tds", () => { const cols1 = { "9d14c2039a78d76760aae879c7fd2c82": new ZedTypeDef({ - type: {kind: "typedef", name: "hello", type: STRING} + type: { + kind: "typedef", + name: "hello", + type: {kind: "primitive", name: "string"} + } }) } const cols2 = { "71f1b421963d31952e15edf7e3957a81": new ZedTypeDef({ - type: {kind: "typedef", name: "hello", type: STRING} + type: { + kind: "typedef", + name: "hello", + type: {kind: "primitive", name: "string"} + } }) } const state = store.dispatchAll([ - Viewer.updateColumns(tabId, new Map(Object.entries(cols1))), - Viewer.updateColumns(tabId, new Map(Object.entries(cols2))) + Viewer.updateColumns(tabId, cols1), + Viewer.updateColumns(tabId, cols2) ]) - expect(Viewer.getColumns(state)).toEqual( - new Map([ - [ - "9d14c2039a78d76760aae879c7fd2c82", - new ZedTypeDef({ - type: {kind: "typedef", name: "hello", type: STRING} - }) - ], - [ - "71f1b421963d31952e15edf7e3957a81", - new ZedTypeDef({ - type: {kind: "typedef", name: "hello", type: STRING} - }) - ] - ]) - ) + expect(Viewer.getColumns(state)).toEqual({ + "9d14c2039a78d76760aae879c7fd2c82": new ZedTypeDef({ + type: { + kind: "typedef", + name: "hello", + type: {kind: "primitive", name: "string"} + } + }), + "71f1b421963d31952e15edf7e3957a81": new ZedTypeDef({ + type: { + kind: "typedef", + name: "hello", + type: {kind: "primitive", name: "string"} + } + }) + }) }) diff --git a/src/js/state/Viewer/types.ts b/src/js/state/Viewer/types.ts index 53e487380c..16465d2a8b 100644 --- a/src/js/state/Viewer/types.ts +++ b/src/js/state/Viewer/types.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {TypeContext} from "zealot/zed/zjson" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" diff --git a/src/js/types/index.ts b/src/js/types/index.ts index 10f7d3548e..de1d3ce77d 100644 --- a/src/js/types/index.ts +++ b/src/js/types/index.ts @@ -2,7 +2,7 @@ import {SpanArgs} from "../state/Search/types" import {TimeUnit} from "../lib" import AppError from "../models/AppError" import {MenuItemConstructorOptions} from "electron" -import {ZedRecord, ZedField} from "zealot/zed/data-types" +import {ZedRecord, ZedField} from "zealot/zed" export type Notification = | AppError diff --git a/src/js/zql/toZql.ts b/src/js/zql/toZql.ts index 37f4cf644e..3390674311 100644 --- a/src/js/zql/toZql.ts +++ b/src/js/zql/toZql.ts @@ -1,5 +1,5 @@ import isString from "lodash/isString" -import {ZedPrimitive} from "zealot/zed/data-types" +import {ZedPrimitive} from "zealot/zed" export function toZql(object: unknown): string { if (object instanceof ZedPrimitive) return toZqlZngPrimitive(object) @@ -30,7 +30,7 @@ function toZqlBool(bool: boolean) { } function toZqlZngPrimitive(data: ZedPrimitive) { - if (data.kind === "string" || data.kind === "bstring") + if (data.type === "string" || data.type === "bstring") return toZqlString(data.toString()) - throw new Error(`Can't convert Zng Type: ${data.kind} to zql`) + throw new Error(`Can't convert Zng Type: ${data.type} to zql`) } diff --git a/test/data/correlation.zson b/test/data/correlation.zson new file mode 100644 index 0000000000..db09261079 --- /dev/null +++ b/test/data/correlation.zson @@ -0,0 +1,2 @@ +{_path:"conn",ts:2020-02-25T16:03:13.978298Z,uid:"CbOjYpkXn9LfqV51c" (bstring),id:{orig_h:192.168.1.110,orig_p:51848 (port=(uint16)),resp_h:192.168.1.254,resp_p:53 (port)} (=0),proto:"udp" (=zenum),service:"dns" (bstring),duration:16.907ms,orig_bytes:38 (uint64),resp_bytes:54 (uint64),conn_state:"SF" (bstring),local_orig:null (bool),local_resp:null (bool),missed_bytes:0 (uint64),history:"Dd" (bstring),orig_pkts:1 (uint64),orig_ip_bytes:66 (uint64),resp_pkts:1 (uint64),resp_ip_bytes:82 (uint64),tunnel_parents:null (1=(|[bstring]|)),community_id:"1:N7YGmWjwTmMKNhsZHBR618n3ReA="} (=2) +{_path:"dns",ts:2020-02-25T16:03:13.978298Z,uid:"CbOjYpkXn9LfqV51c" (bstring),id:{orig_h:192.168.1.110,orig_p:51848,resp_h:192.168.1.254,resp_p:53} (0),proto:"udp" (zenum),trans_id:47856 (uint64),rtt:16.907ms,query:"news.ycombinator.com" (bstring),qclass:1 (uint64),qclass_name:"C_INTERNET" (bstring),qtype:1 (uint64),qtype_name:"A" (bstring),rcode:0 (uint64),rcode_name:"NOERROR" (bstring),AA:false,TC:false,RD:true,RA:true,Z:0 (uint64),answers:["209.216.230.240" (bstring)] (=3),TTLs:[35s],rejected:false} (=4) diff --git a/test/api/data/custom-sample.ndjson b/test/data/custom-sample.ndjson similarity index 100% rename from test/api/data/custom-sample.ndjson rename to test/data/custom-sample.ndjson diff --git a/test/api/data/custom-schema.json b/test/data/custom-schema.json similarity index 100% rename from test/api/data/custom-schema.json rename to test/data/custom-schema.json diff --git a/test/api/data/plain.txt b/test/data/plain.txt similarity index 100% rename from test/api/data/plain.txt rename to test/data/plain.txt diff --git a/test/api/data/sample.ndjson b/test/data/sample.ndjson similarity index 100% rename from test/api/data/sample.ndjson rename to test/data/sample.ndjson diff --git a/test/api/data/sample.pcap b/test/data/sample.pcap similarity index 100% rename from test/api/data/sample.pcap rename to test/data/sample.pcap diff --git a/test/api/data/sample.pcapng b/test/data/sample.pcapng similarity index 100% rename from test/api/data/sample.pcapng rename to test/data/sample.pcapng diff --git a/test/api/data/sample.tsv b/test/data/sample.tsv similarity index 100% rename from test/api/data/sample.tsv rename to test/data/sample.tsv diff --git a/test/api/data/sample.zng b/test/data/sample.zng similarity index 100% rename from test/api/data/sample.zng rename to test/data/sample.zng diff --git a/test/api/data/sampleTypes.json b/test/data/sampleTypes.json similarity index 100% rename from test/api/data/sampleTypes.json rename to test/data/sampleTypes.json diff --git a/test/api/data/types.tsv b/test/data/types.tsv similarity index 100% rename from test/api/data/types.tsv rename to test/data/types.tsv diff --git a/test/factories/record.ts b/test/factories/record.ts new file mode 100644 index 0000000000..f6f4ecb378 --- /dev/null +++ b/test/factories/record.ts @@ -0,0 +1,40 @@ +import {isDate, isString} from "lodash" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" + +// Only primitive values for now +export function recordOf(...fields) { + return new ZedRecord({ + fields: fields.map(([name, type, value]) => { + return new ZedField({name, data: new ZedPrimitive({type, value})}) + }) + }) +} + +// Convert a js object into a zed record +export function createRecord(object): ZedRecord { + let fields = [] + for (let name in object) { + fields.push(createField(name, object[name])) + } + return new ZedRecord({fields}) +} + +export function createField(name, value) { + return new ZedField({ + name, + data: createData(value) + }) +} + +export function createData(value) { + if (isString(value)) { + return new ZedPrimitive({type: "string", value}) + } else if (isDate(value)) { + return new ZedPrimitive({ + type: "time", + value: (value.getTime() / 1000).toString() + }) + } else { + throw new Error(`Implement this: ${JSON.stringify(value)}`) + } +} diff --git a/test/responses/config.js b/test/responses/config.js new file mode 100644 index 0000000000..2c20b2e971 --- /dev/null +++ b/test/responses/config.js @@ -0,0 +1,35 @@ +// All input files are relative to the test/data directory +// All output files are relative to the test/responses directory + +module.exports = { + dns: { + query: "_path=dns", + input: "sample.tsv", + output: "dns.response" + }, + count: { + query: "count()", + input: "sample.tsv", + output: "count.response" + }, + countByPath: { + query: "count() by _path", + input: "sample.tsv", + output: "count-by-path.response" + }, + sample: { + query: "*", + input: "sample.tsv", + output: "sample.response" + }, + correlationUid: { + query: "_path=dns", + input: "correlation.zson", + output: "correlation-uid.response" + }, + correlationUidCommunityId: { + query: "*", + input: "correlation.zson", + output: "correlation-uid-community-id.response" + } +} diff --git a/test/responses/correlation-uid-community-id.response b/test/responses/correlation-uid-community-id.response new file mode 100644 index 0000000000..341d2d7618 --- /dev/null +++ b/test/responses/correlation-uid-community-id.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892506,"ns":909626000},"update_time":{"sec":1618892506,"ns":911052000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/correlation-uid.response b/test/responses/correlation-uid.response new file mode 100644 index 0000000000..5ec26abcfd --- /dev/null +++ b/test/responses/correlation-uid.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892506,"ns":668674000},"update_time":{"sec":1618892506,"ns":669317000},"bytes_read":255,"bytes_matched":141,"records_read":2,"records_matched":1} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/count-by-path.response b/test/responses/count-by-path.response new file mode 100644 index 0000000000..1c2dda8db1 --- /dev/null +++ b/test/responses/count-by-path.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]},{"schema":"23","values":["dns","2"]},{"schema":"23","values":["weird","1"]},{"schema":"23","values":["stats","1"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892506,"ns":137594000},"update_time":{"sec":1618892506,"ns":140560000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/count.response b/test/responses/count.response new file mode 100644 index 0000000000..51764a3802 --- /dev/null +++ b/test/responses/count.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["30"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892505,"ns":822464000},"update_time":{"sec":1618892505,"ns":823472000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/dns.response b/test/responses/dns.response new file mode 100644 index 0000000000..41bbf47ee9 --- /dev/null +++ b/test/responses/dns.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]},{"schema":"28","values":["dns","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","0.017946","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["173.194.201.189"],["213"],"F"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892505,"ns":556065000},"update_time":{"sec":1618892505,"ns":557955000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/index.ts b/test/responses/index.ts index 4628c1464a..590a9022a7 100644 --- a/test/responses/index.ts +++ b/test/responses/index.ts @@ -1,5 +1,16 @@ -const _thing = { - input: "test/data/sample.zson", - query: "* | count()", - output: "test/responses/sample-count.response" +import {readFileSync} from "fs-extra" +import {join} from "path" +import * as config from "test/responses/config" + +const cache = {} + +export function useResponse(name: string) { + if (name in cache) return cache[name] + + const {output} = config[name] + const path = join(__dirname, output) + const data = readFileSync(path, {encoding: "utf-8"}) + + cache[name] = data.split("\n\n\n").map((json) => JSON.parse(json)) + return data } diff --git a/test/responses/sample.response b/test/responses/sample.response new file mode 100644 index 0000000000..2a12902018 --- /dev/null +++ b/test/responses/sample.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"ts_delta","type":{"kind":"primitive","name":"duration"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"gaps","type":{"kind":"primitive","name":"uint64"}},{"name":"acks","type":{"kind":"primitive","name":"uint64"}},{"name":"percent_lost","type":{"kind":"primitive","name":"float64"}}]}}],"values":["capture_loss","1582646597.838527","11.854892","zeek","0","6","0"]},{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","1582646595.986756","Cl4Rbf2czFCAu7bc23",["192.168.1.110","57540","172.217.1.138","443"],"tcp",null,"0.060027","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","1582646595.784014","C2J5XS1KgpVaWCJpRg",["192.168.1.110","57572","172.217.6.138","443"],"tcp",null,"0.053917","63","0","SF",null,null,"0","DFafA","4","271","3","156",null]},{"schema":"28","values":["conn","1582646595.481957","C1dB0oBEPvJUas0dg",["192.168.1.179","51524","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"tx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"rx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"conn_uids","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"source","type":{"kind":"primitive","name":"bstring"}},{"name":"depth","type":{"kind":"primitive","name":"uint64"}},{"name":"analyzers","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"mime_type","type":{"kind":"primitive","name":"bstring"}},{"name":"filename","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"is_orig","type":{"kind":"primitive","name":"bool"}},{"name":"seen_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"total_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"missing_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"overflow_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"timedout","type":{"kind":"primitive","name":"bool"}},{"name":"parent_fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"md5","type":{"kind":"primitive","name":"bstring"}},{"name":"sha1","type":{"kind":"primitive","name":"bstring"}},{"name":"sha256","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted_cutoff","type":{"kind":"primitive","name":"bool"}},{"name":"extracted_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["files","1582646594.050188","FMa5RD1f8J80UyJKQe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1"],"application/ocsp-response",null,"0",null,"F","471",null,"0","0","F",null,"fca4341c673e74dc3e330e1640e9b7d4","570304f7776ed6ca29464404554fb927395328e6",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","FXkLzR25cwYgeI5yI7",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-user-cert",null,"0",null,"F","1711",null,"0","0","F",null,"d16bf2ff6647cb8bccce2594e9237e50","987050ffb905cad3a79a8596c2120db97c03a165",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","Frbo5s1CRsV02JUoSe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0",null,"F","1176",null,"0","0","F",null,"345eff15b7a49add451b65a7f4bdc6ae","1fb86b1168ec743154062e8c9cc5b171a4b7ccb4",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","F1VUER3CattO8PiHWc",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0",null,"F","947",null,"0","0","F",null,"79e4a9840d7d3a96d7c04fe2434c892e","a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436",null,null,null,null]},{"schema":"36","types":[{"kind":"typedef","name":"36","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"id","type":{"kind":"primitive","name":"bstring"}},{"name":"certificate","type":{"kind":"record","fields":[{"name":"version","type":{"kind":"primitive","name":"uint64"}},{"name":"serial","type":{"kind":"primitive","name":"bstring"}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"not_valid_before","type":{"kind":"primitive","name":"time"}},{"name":"not_valid_after","type":{"kind":"primitive","name":"time"}},{"name":"key_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"sig_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"key_type","type":{"kind":"primitive","name":"bstring"}},{"name":"key_length","type":{"kind":"primitive","name":"uint64"}},{"name":"exponent","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}}]}},{"name":"san","type":{"kind":"record","fields":[{"name":"dns","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"uri","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"email","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"ip","type":{"kind":"array","type":{"kind":"primitive","name":"ip"}}}]}},{"name":"basic_constraints","type":{"kind":"record","fields":[{"name":"ca","type":{"kind":"primitive","name":"bool"}},{"name":"path_len","type":{"kind":"primitive","name":"uint64"}}]}}]}}],"values":["x509","1582646594.050187","FXkLzR25cwYgeI5yI7",["3","074FE902C6B7D619884E1CA1854D9178","CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","1562569200","1631300400","rsaEncryption","sha256WithRSAEncryption","rsa","2048","65537",null],[["news.ycombinator.com"],null,null,null],["F",null]]},{"schema":"37","types":[{"kind":"typedef","name":"37","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"version","type":{"kind":"primitive","name":"bstring"}},{"name":"cipher","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}},{"name":"server_name","type":{"kind":"primitive","name":"bstring"}},{"name":"resumed","type":{"kind":"primitive","name":"bool"}},{"name":"last_alert","type":{"kind":"primitive","name":"bstring"}},{"name":"next_protocol","type":{"kind":"primitive","name":"bstring"}},{"name":"established","type":{"kind":"primitive","name":"bool"}},{"name":"cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"client_cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"client_subject","type":{"kind":"primitive","name":"bstring"}},{"name":"client_issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"validation_status","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["ssl","1582646594.021637","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"TLSv12","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","secp256r1","news.ycombinator.com","F",null,"http/1.1","T",["FXkLzR25cwYgeI5yI7","Frbo5s1CRsV02JUoSe","F1VUER3CattO8PiHWc"],[],"CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US",null,null,"ok"]},{"schema":"28","values":["conn","1582646593.996366","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"tcp","ssl","0.104626","1088","6425","S1",null,null,"0","ShADda","10","1620","8","6849",null]},{"schema":"28","values":["conn","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"39","types":[{"kind":"typedef","name":"39","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]},{"schema":"28","values":["conn","1582646592.860963","CK9UVX1h8kJ6Wrmju6",["192.168.1.110","57635","17.125.252.5","443"],"tcp",null,"0.593878","699","305","OTH",null,null,"0","DadA","5","899","5","505",null]},{"schema":"28","values":["conn","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","dns","0.017946","46","62","SF",null,null,"0","Dd","1","74","1","90",null]},{"schema":"39","values":["dns","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","0.017946","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["173.194.201.189"],["213"],"F"]},{"schema":"28","values":["conn","1582646590.938093","CFH9bc1tw1tM6UNPjl",["192.168.1.110","55351","18.205.93.211","443"],"tcp",null,"0.088679","215","193","OTH",null,null,"0","DadA","2","319","2","297",null]},{"schema":"28","values":["conn","1582646590.264455","C7sBkQ1LVS7gbqDtzk",["192.168.1.110","57332","64.233.179.189","443"],"tcp",null,"1.079187","363","382","OTH",null,null,"0","^dADa","7","727","7","746",null]},{"schema":"28","values":["conn","1582646589.440467","CcIQuP2iS3J8Gl3td7",["192.168.1.179","47783","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"28","values":["conn","1582646588.807682","CkqJmu2oNuiB3zZ0ta",["192.168.1.110","55354","52.37.243.173","443"],"tcp",null,"0.761817","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","1582646588.4727","C6aipo1N64FtzIpSqc",["192.168.1.110","57487","192.30.253.125","443"],"tcp",null,"0.077944","28","24","OTH",null,null,"0","^dADa","2","132","2","128",null]},{"schema":"28","values":["conn","1582646588.449312","CpH9k34gcifnZmAH3h",["192.168.1.110","57326","173.194.201.189","443"],"tcp",null,"1.152367","403","380","OTH",null,null,"0","^dADa","7","767","7","744",null]},{"schema":"28","values":["conn","1582646588.334796","C1VePj3MBeuyPB31wi",["192.168.1.110","55346","52.37.243.173","443"],"tcp",null,"1.230297","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","1582646588.210507","CLuXz9mjBQJrNe822",["192.168.1.110","55344","52.37.243.173","443"],"tcp",null,"0.034261","56","56","OTH",null,null,"0","DadA","2","160","2","160",null]},{"schema":"28","values":["conn","1582646587.715839","Cy3dlK3PhYKCwG227k",["192.168.1.110","55747","13.52.5.22","443"],"tcp",null,"0.017643","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","1582646587.715728","CFwwj51CZ17Px2g2rl",["192.168.1.110","55635","18.246.31.137","443"],"tcp",null,"0.040702","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"40","types":[{"kind":"typedef","name":"40","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"name","type":{"kind":"primitive","name":"bstring"}},{"name":"addl","type":{"kind":"primitive","name":"bstring"}},{"name":"notice","type":{"kind":"primitive","name":"bool"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["weird","1582646586.880512",null,[null,null,null,null],"unknown_protocol","2","F","zeek"]},{"schema":"28","values":["conn","1582646586.154443","Cki8nOlkkWyYzyqx2",["192.168.1.110","57591","172.217.9.142","443"],"tcp",null,"0.216412","2511","3263","OTH",null,null,"0","DadA","11","3083","9","3731",null]},{"schema":"41","types":[{"kind":"typedef","name":"41","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"mem","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"bytes_recv","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_dropped","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_link","type":{"kind":"primitive","name":"uint64"}},{"name":"pkt_lag","type":{"kind":"primitive","name":"duration"}},{"name":"events_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"events_queued","type":{"kind":"primitive","name":"uint64"}},{"name":"active_tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"timers","type":{"kind":"primitive","name":"uint64"}},{"name":"active_timers","type":{"kind":"primitive","name":"uint64"}},{"name":"files","type":{"kind":"primitive","name":"uint64"}},{"name":"active_files","type":{"kind":"primitive","name":"uint64"}},{"name":"dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"active_dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_tcp_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_file_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_frag_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_unknown_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["stats","1582646585.983635","zeek","71","1","1001",null,null,null,"403","12","1","0","0","1","0","0","38","34","0","0","0","0","1008","0","0","0"]},{"schema":"28","values":["conn","1582646585.983635","CMa2ZP3LnKsjzJCDy7",["192.168.1.110","56625","172.217.9.142","443"],"tcp",null,"11.317514","2801","1306","OTH",null,null,"0","DadA","16","3633","16","2138",null]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618892506,"ns":407720000},"update_time":{"sec":1618892506,"ns":411426000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} + + +{"type":"TaskEnd","task_id":0} + diff --git a/zealot/fetcher/records_callback.ts b/zealot/fetcher/records_callback.ts index 5db93a7b52..fb30855b1b 100644 --- a/zealot/fetcher/records_callback.ts +++ b/zealot/fetcher/records_callback.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" import {decode, TypeContext} from "zealot/zed/zjson" import * as zqd from "../zqd" @@ -19,8 +19,8 @@ function getChannel(id: number, channels: ChannelMap): Channel { if (!channels.has(id)) { channels.set(id, { rows: [], - schemas: new Map(), - context: new Map() + schemas: {}, + context: {} } as Channel) } return channels.get(id) as Channel diff --git a/zealot/fetcher/stream.ts b/zealot/fetcher/stream.ts index ded8dc371a..be7067c295 100644 --- a/zealot/fetcher/stream.ts +++ b/zealot/fetcher/stream.ts @@ -1,7 +1,7 @@ import {createCallbacks} from "./callbacks" import {ZCallbacks, ZIterator} from "../types" import {decode, StreamObject} from "zealot/zed/zjson" -import {ZedRecord} from "zealot/zed/data-types" +import {ZedRecord} from "zealot/zed" async function emitCallbacks(iterator: ZIterator, callbacks: ZCallbacks) { try { diff --git a/zealot/zed/array.ts b/zealot/zed/array.ts index ef6476ed17..94d82834fe 100644 --- a/zealot/zed/array.ts +++ b/zealot/zed/array.ts @@ -11,6 +11,10 @@ export class ZedArray { this.items = args.items } + isUnset() { + return this.items === null + } + serialize() { return { kind: "array", diff --git a/zealot/zed/enum.ts b/zealot/zed/enum.ts index 400193f8ed..40a2955d77 100644 --- a/zealot/zed/enum.ts +++ b/zealot/zed/enum.ts @@ -9,6 +9,10 @@ export class ZedEnum { this.typeName = args.typeName } + isUnset() { + return this.value === null + } + serialize() { return { kind: "enum", diff --git a/zealot/zed/field.ts b/zealot/zed/field.ts index bf3e4625bb..d164f8bae4 100644 --- a/zealot/zed/field.ts +++ b/zealot/zed/field.ts @@ -1,9 +1,14 @@ import {ZedData} from "./index" +import {deserialize} from "./json" export class ZedField { name: string data: ZedData + static deserialize(data): ZedField { + return deserialize(data) + } + constructor({name, data}: {name: string; data: ZedData}) { this.name = name this.data = data diff --git a/zealot/zed/index.ts b/zealot/zed/index.ts index 75351220c5..ed30845088 100644 --- a/zealot/zed/index.ts +++ b/zealot/zed/index.ts @@ -7,6 +7,8 @@ import {ZedRecord} from "./record" import {ZedSet} from "./set" import {ZedUnion} from "./union" +export {SerializedZedRecord, SerializedZedField} from "./json" + export { ZedArray, ZedEnum, diff --git a/zealot/zed/json.ts b/zealot/zed/json.ts index d20e2dae64..4467a688e7 100644 --- a/zealot/zed/json.ts +++ b/zealot/zed/json.ts @@ -2,14 +2,80 @@ import { ZedArray, ZedPrimitive, ZedEnum, - ZedField, ZedMap, ZedRecord, ZedSet, - ZedUnion + ZedUnion, + ZedData, + ZedField } from "./index" -export function deserialize(data) { +export type SerializedZed = + | SerializedZedPrimitive + | SerializedZedArray + | SerializedZedSet + | SerializedZedRecord + | SerializedZedUnion + | SerializedZedEnum + | SerializedZedMap + | SerializedZedField + +export type SerializedZedField = { + kind: "field" + name: string + data: SerializedZed +} + +export type SerializedZedPrimitive = { + kind: "primitive" + value: string | null + type: string + typeName?: string +} + +export type SerializedZedArray = { + kind: "array" + type: string + items: SerializedZed[] + typeName?: string +} + +export type SerializedZedSet = { + kind: "set" + type: string + items: SerializedZed[] + typeName?: string +} + +export type SerializedZedRecord = { + kind: "record" + fields: SerializedZedField[] + typeName?: string +} + +export type SerializedZedUnion = { + kind: "union" + types: string[] + value: SerializedZed + typeName?: string +} + +export type SerializedZedEnum = { + kind: "enum" + symbols: string[] + value: string + typeName?: string +} + +export type SerializedZedMap = { + kind: "map" + keyType: string + valueType: string + value: [SerializedZed, SerializedZed][] + typeName?: string +} + +export function deserialize(data: SerializedZed) { switch (data.kind) { case "primitive": return new ZedPrimitive({ @@ -29,13 +95,13 @@ export function deserialize(data) { return new ZedSet({ typeName: data.typeName, type: data.type, - items: data.items.map(deserialize) + items: data.items.map(deserialize) as ZedData[] }) case "record": return new ZedRecord({ typeName: data.typeName, - fields: data.fields.map(deserialize) + fields: data.fields.map(deserialize) as ZedField[] }) case "field": diff --git a/zealot/zed/map.ts b/zealot/zed/map.ts index b0453aac12..4c8ca6afe6 100644 --- a/zealot/zed/map.ts +++ b/zealot/zed/map.ts @@ -18,6 +18,10 @@ export class ZedMap { this.value = args.value } + isUnset() { + return this.value === null + } + serialize() { return { kind: "map", diff --git a/zealot/zed/primitive.ts b/zealot/zed/primitive.ts index a8ceddc8bb..710652e04a 100644 --- a/zealot/zed/primitive.ts +++ b/zealot/zed/primitive.ts @@ -39,6 +39,10 @@ export class ZedPrimitive { return int } + toString(): string { + return this.value || "" + } + serialize() { return { kind: "primitive", diff --git a/zealot/zed/record.ts b/zealot/zed/record.ts index 10313550da..a456e0a7c4 100644 --- a/zealot/zed/record.ts +++ b/zealot/zed/record.ts @@ -1,4 +1,5 @@ import {ZedField} from "./field" +import {deserialize} from "./json" export class ZedRecord { fields: ZedField[] | null @@ -9,6 +10,10 @@ export class ZedRecord { this.typeName = args.typeName } + static deserialize(data): ZedRecord { + return deserialize(data) + } + [Symbol.iterator]() { let index = 0 const next = () => diff --git a/zealot/zed/set.ts b/zealot/zed/set.ts index a56410239f..3e25e29b7e 100644 --- a/zealot/zed/set.ts +++ b/zealot/zed/set.ts @@ -11,6 +11,10 @@ export class ZedSet { this.items = args.items } + isUnset() { + return this.items === null + } + serialize() { return { kind: "set", diff --git a/zealot/zed/union.ts b/zealot/zed/union.ts index 2c4a08eee5..0b5b567915 100644 --- a/zealot/zed/union.ts +++ b/zealot/zed/union.ts @@ -11,6 +11,10 @@ export class ZedUnion { this.typeName = args.typeName } + isUnset() { + return this.value === null + } + serialize() { return { kind: "union", From d144e80291df28fd578dcf301c7f237a749e9b11 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 20 Apr 2021 16:30:42 -0700 Subject: [PATCH 06/41] Tests green --- ppl/detail/flows/fetch.test.ts | 37 ++++++------- ppl/detail/flows/fetch.ts | 2 +- ppl/detail/flows/test-this.test.ts | 7 --- scripts/test/responses.js | 1 + src/js/flows/search/handler.ts | 1 - src/js/flows/search/mod.ts | 1 - src/js/searches/programs.test.ts | 1 - test/data/no-community-id-in-conn.zson | 2 + test/data/only-alerts.zng | Bin 0 -> 2716 bytes test/factories/record.ts | 51 ++++++++++++------ test/responses/config.js | 12 ++++- .../correlation-uid-community-id.response | 2 +- test/responses/correlation-uid.response | 4 +- test/responses/count-by-path.response | 4 +- test/responses/count.response | 2 +- test/responses/dns.response | 2 +- test/responses/index.ts | 4 +- .../no-community-id-in-conn.response | 14 +++++ test/responses/only-alerts.response | 14 +++++ test/responses/sample.response | 2 +- zealot/zealot_mock.ts | 1 + zealot/zed/array.ts | 4 +- zealot/zed/set.ts | 2 +- zealot/zed/zjson.ts | 38 +++++++------ 24 files changed, 134 insertions(+), 74 deletions(-) delete mode 100644 ppl/detail/flows/test-this.test.ts create mode 100644 test/data/no-community-id-in-conn.zson create mode 100644 test/data/only-alerts.zng create mode 100644 test/responses/no-community-id-in-conn.response create mode 100644 test/responses/only-alerts.response diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index fb6e51d075..d26b3c6b24 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -1,22 +1,23 @@ import {last} from "lodash" import loginTo from "src/js/test/helpers/loginTo" -import {recordOf} from "test/factories/record" +import {createRecord} from "test/factories/record" import {useResponse} from "test/responses" import {fetchCorrelation} from "./fetch" -const zeek = recordOf( - ["_path", "string", "dns"], - ["uid", "string", "CbOjYpkXn9LfqV51c"] -) +const zeek = createRecord({ + _path: "dns", + uid: "CbOjYpkXn9LfqV51c", + duration: null, + ts: new Date(0) +}) -const suricata = recordOf([ - "community_id", - "string", - "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" -]) +const suricata = createRecord({ + ts: new Date(0), + community_id: "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" +}) const uidOrCommunityIdZql = - 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' + 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" and ts >= 1582646593.978 and ts < 1582646683.994) | head 100' const uidZql = 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" | head 100' @@ -26,16 +27,16 @@ const cidZql = 'community_id="1:N7YGmWjwTmMKNhsZHBR618n3ReA=" | head 100' const stubs = { uidResult: useResponse("correlationUid"), uidAndCommunityResult: useResponse("correlationUidCommunityId"), - alertResults: "FILL ME IN", - noCommunityIdInConn: "FILL ME IN" + alertResults: useResponse("onlyAlerts"), + noCommunityIdInConn: useResponse("noCommunityIdInConn") } -describe.only("zeek log when community_id is found", () => { +describe("zeek log when community_id is found", () => { let setup beforeEach(async () => { setup = await loginTo("workspace1", "space1") setup.zealot.stubStream("search", stubs.uidResult) - setup.zealot.stubStream(" search", stubs.uidAndCommunityResult) + setup.zealot.stubStream("search", stubs.uidAndCommunityResult) }) test("runs two queries ", async () => { @@ -57,7 +58,7 @@ describe.only("zeek log when community_id is found", () => { test("returns the records", async () => { const {store} = setup const records = await store.dispatch(fetchCorrelation(zeek)) - expect(records.length).toBe(10) + expect(records.length).toBe(2) }) }) @@ -84,7 +85,7 @@ describe("zeek log when community_id is not found", () => { test("returns the records", async () => { const {store} = setup const records = await store.dispatch(fetchCorrelation(zeek)) - expect(records.length).toBe(8) + expect(records.length).toBe(2) }) }) @@ -112,6 +113,6 @@ describe("suricata alert when community_id found", () => { test("returns the records", async () => { const {store} = setup const records = await store.dispatch(fetchCorrelation(suricata)) - expect(records.length).toBe(9) + expect(records.length).toBe(11) }) }) diff --git a/ppl/detail/flows/fetch.ts b/ppl/detail/flows/fetch.ts index bfd3062545..dbe7ca8c38 100644 --- a/ppl/detail/flows/fetch.ts +++ b/ppl/detail/flows/fetch.ts @@ -21,6 +21,7 @@ export const fetchCorrelation = (record: ZedRecord) => async (dispatch) => { const query = getCorrelationQuery(record) const {uid, cid} = new Correlation(record).getIds() const run = () => collect(dispatch(search({query, id}))) + if (!uid && !cid) return [] if (cid && uid) return run() if (cid) return run() @@ -28,7 +29,6 @@ export const fetchCorrelation = (record: ZedRecord) => async (dispatch) => { // If there is only a uid and not a cid const records = await run() const conn = findConn(records) - console.log(conn) if (conn && conn.has("community_id")) return dispatch(fetchCorrelation(conn)) else return records } diff --git a/ppl/detail/flows/test-this.test.ts b/ppl/detail/flows/test-this.test.ts deleted file mode 100644 index 6664790072..0000000000 --- a/ppl/detail/flows/test-this.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {useResponse} from "test/responses" - -const dns = useResponse("dns") - -test("hi", () => { - console.log(dns) -}) diff --git a/scripts/test/responses.js b/scripts/test/responses.js index 671b4fd19a..99b93229b0 100644 --- a/scripts/test/responses.js +++ b/scripts/test/responses.js @@ -3,6 +3,7 @@ const fs = require("fs-extra") const glob = require("glob") const config = require("../../test/responses/config") +// In order for this to work, rollup the zealot module first function saveResponse(input, output, query) { const deno = spawn( "deno", diff --git a/src/js/flows/search/handler.ts b/src/js/flows/search/handler.ts index 3504d1471a..8b808c5999 100644 --- a/src/js/flows/search/handler.ts +++ b/src/js/flows/search/handler.ts @@ -30,7 +30,6 @@ export function handle(request: any) { reject(e) } } - request .then((stream) => { stream diff --git a/src/js/flows/search/mod.ts b/src/js/flows/search/mod.ts index a66c5e040f..a78536f212 100644 --- a/src/js/flows/search/mod.ts +++ b/src/js/flows/search/mod.ts @@ -39,7 +39,6 @@ export function search({ to = to || defaultTo from = from || defaultFrom const req = zealot.search(query, {from, to, spaceId, signal: ctl.signal}) - dispatch(Handlers.abort(id, false)) dispatch(Handlers.register(id, searchHandle)) diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index 9a0d6f9c20..62db26cdd7 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -9,7 +9,6 @@ test("conn correlation", () => { duration: 0.70995, community_id: "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" }) - expect( connCorrelation( record.get("uid") as ZedPrimitive, diff --git a/test/data/no-community-id-in-conn.zson b/test/data/no-community-id-in-conn.zson new file mode 100644 index 0000000000..56a95777c3 --- /dev/null +++ b/test/data/no-community-id-in-conn.zson @@ -0,0 +1,2 @@ +{_path:"conn",ts:2020-02-25T16:03:13.978298Z,uid:"CbOjYpkXn9LfqV51c" (bstring),id:{orig_h:192.168.1.110,orig_p:51848 (port=(uint16)),resp_h:192.168.1.254,resp_p:53 (port)} (=0),proto:"udp" (=zenum),service:"dns" (bstring),duration:16.907ms,orig_bytes:38 (uint64),resp_bytes:54 (uint64),conn_state:"SF" (bstring),local_orig:null (bool),local_resp:null (bool),missed_bytes:0 (uint64),history:"Dd" (bstring),orig_pkts:1 (uint64),orig_ip_bytes:66 (uint64),resp_pkts:1 (uint64),resp_ip_bytes:82 (uint64),tunnel_parents:null (1=(|[bstring]|))} (=2) +{_path:"dns",ts:2020-02-25T16:03:13.978298Z,uid:"CbOjYpkXn9LfqV51c" (bstring),id:{orig_h:192.168.1.110,orig_p:51848,resp_h:192.168.1.254,resp_p:53} (0),proto:"udp" (zenum),trans_id:47856 (uint64),rtt:16.907ms,query:"news.ycombinator.com" (bstring),qclass:1 (uint64),qclass_name:"C_INTERNET" (bstring),qtype:1 (uint64),qtype_name:"A" (bstring),rcode:0 (uint64),rcode_name:"NOERROR" (bstring),AA:false,TC:false,RD:true,RA:true,Z:0 (uint64),answers:["209.216.230.240" (bstring)] (=3),TTLs:[35s],rejected:false} (=4) diff --git a/test/data/only-alerts.zng b/test/data/only-alerts.zng new file mode 100644 index 0000000000000000000000000000000000000000..a4d7c3de52d425ff46123cc2f61de67ffed6616b GIT binary patch literal 2716 zcmdVWPe>F|7y$5DcU@-}%h@@o+dn2K6g{-N3R+=dDzT-j?5+t#J|8o0*TMPM%v;yB z%Ss^#LLRcnk|>G-Av_fw3W*}LE>RJMkfE0}qO6ocsP~ri+_9O%%s1aR^L^j%t+@={ z#Ew1TXIr^IAtPWp5%zl}WGH$@ zMH=>VFp)qaMiMYgT}p|hDw>G=C;^E5yn#&G3~C|*vT8tDPZ`pBeX|OcG$gYVP?eR8 z--%(;zrwQXt2jIx>-pIAb$o8pM%Fc7;YJb2lGB-_Hxu0oD}sHxJu7HC-_)C39P+{xEA<^pme zf}Rkd5J5Ptn>|8|e8fS8tcxVJZF+~x;d0kA^eQP11%g!|7y!ZA+)F3`0@WZ?!!TUu z`1U)ezPs4EyF?bPTE+nY`&JwZ)_V^KaE z34Ut7H*R~=nYrgLsp;3FW}b7Oc<{0OitSCO7aqT%rttnFe0FC=@pAgI?M=c=>nt^W jeR7x|iI!eC_;k$nCVhD6A~g-vR`48ncYmH6{Rh(@$VNn1 literal 0 HcmV?d00001 diff --git a/test/factories/record.ts b/test/factories/record.ts index f6f4ecb378..35d0ad74c7 100644 --- a/test/factories/record.ts +++ b/test/factories/record.ts @@ -1,15 +1,6 @@ -import {isDate, isString} from "lodash" +import {isDate, isInteger, isNumber, isObject, isString} from "lodash" import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" -// Only primitive values for now -export function recordOf(...fields) { - return new ZedRecord({ - fields: fields.map(([name, type, value]) => { - return new ZedField({name, data: new ZedPrimitive({type, value})}) - }) - }) -} - // Convert a js object into a zed record export function createRecord(object): ZedRecord { let fields = [] @@ -27,14 +18,44 @@ export function createField(name, value) { } export function createData(value) { - if (isString(value)) { - return new ZedPrimitive({type: "string", value}) - } else if (isDate(value)) { + if (value === null) { + return new ZedPrimitive({type: "null", value: null}) + } + + if (isDate(value)) { return new ZedPrimitive({ type: "time", value: (value.getTime() / 1000).toString() }) - } else { - throw new Error(`Implement this: ${JSON.stringify(value)}`) } + + if (isInteger(value)) { + return new ZedPrimitive({type: "uint64", value: value.toString()}) + } + + if (isNumber(value)) { + return new ZedPrimitive({type: "float64", value: value.toString()}) + } + + if (isString(value) && isIp(value)) { + return new ZedPrimitive({type: "ip", value}) + } + + if (isString(value)) { + return new ZedPrimitive({type: "string", value}) + } + + if (isObject(value)) { + return createRecord(value) + } + + throw new Error(`Implement this: ${JSON.stringify(value)}`) +} + +function isIp(string) { + const blocks: any[] = string.split(".") + if (blocks.length !== 4) return false + return blocks.every((block) => { + return Number(block) >= 0 && Number(block) <= 255 + }) } diff --git a/test/responses/config.js b/test/responses/config.js index 2c20b2e971..6941573b24 100644 --- a/test/responses/config.js +++ b/test/responses/config.js @@ -23,7 +23,7 @@ module.exports = { output: "sample.response" }, correlationUid: { - query: "_path=dns", + query: "*", input: "correlation.zson", output: "correlation-uid.response" }, @@ -31,5 +31,15 @@ module.exports = { query: "*", input: "correlation.zson", output: "correlation-uid-community-id.response" + }, + noCommunityIdInConn: { + query: "*", + input: "no-community-id-in-conn.zson", + output: "no-community-id-in-conn.response" + }, + onlyAlerts: { + query: "*", + input: "only-alerts.zng", + output: "only-alerts.response" } } diff --git a/test/responses/correlation-uid-community-id.response b/test/responses/correlation-uid-community-id.response index 341d2d7618..8dd01f21ea 100644 --- a/test/responses/correlation-uid-community-id.response +++ b/test/responses/correlation-uid-community-id.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892506,"ns":909626000},"update_time":{"sec":1618892506,"ns":911052000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1618961329,"ns":341415000},"update_time":{"sec":1618961329,"ns":342521000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/correlation-uid.response b/test/responses/correlation-uid.response index 5ec26abcfd..68dbc4a505 100644 --- a/test/responses/correlation-uid.response +++ b/test/responses/correlation-uid.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892506,"ns":668674000},"update_time":{"sec":1618892506,"ns":669317000},"bytes_read":255,"bytes_matched":141,"records_read":2,"records_matched":1} +{"type":"SearchStats","start_time":{"sec":1618961329,"ns":92713000},"update_time":{"sec":1618961329,"ns":93862000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count-by-path.response b/test/responses/count-by-path.response index 1c2dda8db1..76ed673f1e 100644 --- a/test/responses/count-by-path.response +++ b/test/responses/count-by-path.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]},{"schema":"23","values":["dns","2"]},{"schema":"23","values":["weird","1"]},{"schema":"23","values":["stats","1"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["dns","2"]},{"schema":"23","values":["weird","1"]},{"schema":"23","values":["stats","1"]},{"schema":"23","values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892506,"ns":137594000},"update_time":{"sec":1618892506,"ns":140560000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1618961328,"ns":598653000},"update_time":{"sec":1618961328,"ns":600533000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count.response b/test/responses/count.response index 51764a3802..4fae8e84e5 100644 --- a/test/responses/count.response +++ b/test/responses/count.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892505,"ns":822464000},"update_time":{"sec":1618892505,"ns":823472000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1618961328,"ns":357440000},"update_time":{"sec":1618961328,"ns":360568000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/dns.response b/test/responses/dns.response index 41bbf47ee9..65084e7e0d 100644 --- a/test/responses/dns.response +++ b/test/responses/dns.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892505,"ns":556065000},"update_time":{"sec":1618892505,"ns":557955000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1618961328,"ns":111091000},"update_time":{"sec":1618961328,"ns":113691000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/index.ts b/test/responses/index.ts index 590a9022a7..5234b52bcc 100644 --- a/test/responses/index.ts +++ b/test/responses/index.ts @@ -7,10 +7,10 @@ const cache = {} export function useResponse(name: string) { if (name in cache) return cache[name] + if (!(name in config)) throw new Error(`Unknown response: ${name}`) const {output} = config[name] const path = join(__dirname, output) const data = readFileSync(path, {encoding: "utf-8"}) - cache[name] = data.split("\n\n\n").map((json) => JSON.parse(json)) - return data + return cache[name] } diff --git a/test/responses/no-community-id-in-conn.response b/test/responses/no-community-id-in-conn.response new file mode 100644 index 0000000000..4a4af75912 --- /dev/null +++ b/test/responses/no-community-id-in-conn.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618961329,"ns":585595000},"update_time":{"sec":1618961329,"ns":586740000},"bytes_read":224,"bytes_matched":224,"records_read":2,"records_matched":2} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/only-alerts.response b/test/responses/only-alerts.response new file mode 100644 index 0000000000..9666d4fb15 --- /dev/null +++ b/test/responses/only-alerts.response @@ -0,0 +1,14 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"event_type","type":{"kind":"primitive","name":"bstring"}},{"name":"src_ip","type":{"kind":"primitive","name":"ip"}},{"name":"src_port","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"dest_ip","type":{"kind":"primitive","name":"ip"}},{"name":"dest_port","type":{"kind":"typename","name":"port"}},{"name":"vlan","type":{"kind":"array","type":{"kind":"primitive","name":"uint16"}}},{"name":"proto","type":{"kind":"primitive","name":"bstring"}},{"name":"app_proto","type":{"kind":"primitive","name":"bstring"}},{"name":"alert","type":{"kind":"record","fields":[{"name":"severity","type":{"kind":"primitive","name":"uint16"}},{"name":"signature","type":{"kind":"primitive","name":"bstring"}},{"name":"category","type":{"kind":"primitive","name":"bstring"}},{"name":"action","type":{"kind":"primitive","name":"bstring"}},{"name":"signature_id","type":{"kind":"primitive","name":"uint64"}},{"name":"gid","type":{"kind":"primitive","name":"uint64"}},{"name":"rev","type":{"kind":"primitive","name":"uint64"}},{"name":"metadata","type":{"kind":"record","fields":[{"name":"signature_severity","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"former_category","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"attack_target","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"deployment","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"affected_product","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"created_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"performance_impact","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"updated_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"malware_family","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"tag","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}}]}}]}},{"name":"flow_id","type":{"kind":"primitive","name":"uint64"}},{"name":"pcap_cnt","type":{"kind":"primitive","name":"uint64"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"tx_id","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_code","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_type","type":{"kind":"primitive","name":"uint64"}},{"name":"community_id","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668996500830","9025085","1428696616.318954","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668255955402","8015332","1428390396.903542","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394670027691253","7847662","1428286359.587171","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669830007389","7174571","1427922895.913599","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668388559068","6196400","1427802619.776149","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668479470212","5719123","1427509072.835646","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668951075328","5413510","1427319659.24276","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669628743811","5030650","1427100624.107701","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669695329403","4864841","1427003335.253496","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669771541024","4703729","1426906194.903837","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668624120943","2766977","1426102254.635397","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1618961329,"ns":835445000},"update_time":{"sec":1618961329,"ns":836198000},"bytes_read":2299,"bytes_matched":2299,"records_read":11,"records_matched":11} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/sample.response b/test/responses/sample.response index 2a12902018..7f1d112212 100644 --- a/test/responses/sample.response +++ b/test/responses/sample.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618892506,"ns":407720000},"update_time":{"sec":1618892506,"ns":411426000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1618961328,"ns":846342000},"update_time":{"sec":1618961328,"ns":848980000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/zealot/zealot_mock.ts b/zealot/zealot_mock.ts index 91bbcf5f6f..ac251af5ae 100644 --- a/zealot/zealot_mock.ts +++ b/zealot/zealot_mock.ts @@ -3,6 +3,7 @@ import {FetchArgs} from "./fetcher/fetcher" import {createStream} from "./fetcher/stream" import {createError} from "./util/error" import {Zealot, ZealotPayload} from "./types" +import {isConstructorDeclaration} from "typescript" type StubMode = "always" | "once" type RespWrap = typeof stream | typeof promise diff --git a/zealot/zed/array.ts b/zealot/zed/array.ts index 94d82834fe..eb43f86b6e 100644 --- a/zealot/zed/array.ts +++ b/zealot/zed/array.ts @@ -2,7 +2,7 @@ import {ZedData} from "./index" export class ZedArray { type: string - items: ZedData[] + items: ZedData[] | null typeName?: string constructor(args: {type: string; typeName?: string; items: ZedData[]}) { @@ -20,7 +20,7 @@ export class ZedArray { kind: "array", type: this.type, typeName: this.typeName, - items: this.items.map((item) => item.serialize()) + items: this.isUnset() ? null : this.items.map((item) => item.serialize()) } } } diff --git a/zealot/zed/set.ts b/zealot/zed/set.ts index 3e25e29b7e..b07211307c 100644 --- a/zealot/zed/set.ts +++ b/zealot/zed/set.ts @@ -20,7 +20,7 @@ export class ZedSet { kind: "set", type: this.type, typeName: this.typeName, - items: this.items.map((item) => item.serialize()) + items: this.isUnset() ? null : this.items.map((item) => item.serialize()) } } } diff --git a/zealot/zed/zjson.ts b/zealot/zed/zjson.ts index af161a2f19..f55153a583 100644 --- a/zealot/zed/zjson.ts +++ b/zealot/zed/zjson.ts @@ -104,25 +104,29 @@ export function construct( return new ZedPrimitive({ typeName, type: type.name, - value: value as string + value: value as string | null }) case "array": return new ZedArray({ typeName, type: simpleType(type.type), - items: (value as Value[]).map((value) => - construct(context, type.type, value) - ) + items: value + ? (value as Value[]).map((value) => + construct(context, type.type, value) + ) + : null }) case "set": return new ZedSet({ typeName, type: simpleType(type.type), - items: (value as Value[]).map((value) => - construct(context, type.type, value) - ) + items: value + ? (value as Value[]).map((value) => + construct(context, type.type, value) + ) + : null }) case "record": @@ -148,7 +152,7 @@ export function construct( return new ZedEnum({ typeName, symbols: type.symbols, - value: value as string + value: value as string | null }) case "map": @@ -156,14 +160,16 @@ export function construct( typeName, keyType: simpleType(type.key_type), valueType: simpleType(type.val_type), - value: new Map( - createEntries(value as Value[]).map(([key, value]) => { - return [ - construct(context, type.key_type, key), - construct(context, type.val_type, value) - ] - }) - ) + value: value + ? new Map( + createEntries(value as Value[]).map(([key, value]) => { + return [ + construct(context, type.key_type, key), + construct(context, type.val_type, value) + ] + }) + ) + : null }) case "typename": From 95db2db74106423c3efedfaf2758741789ff887f Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 20 Apr 2021 17:08:30 -0700 Subject: [PATCH 07/41] Removed most of cell.ts only need to guess width --- app/core/formatters/format-zed.ts | 10 +++++ app/detail/Fields.tsx | 6 +-- ppl/summary/number.tsx | 10 +++-- ppl/summary/table.tsx | 9 ++-- scripts/rename.js | 3 +- scripts/test/search.deno.ts | 6 +-- src/js/brim/cell.ts | 5 --- src/js/brim/complexCell.ts | 7 +-- src/js/brim/primitiveCell.ts | 39 ----------------- src/js/brim/program.test.ts | 9 ++-- src/js/brim/program.ts | 14 +++--- src/js/brim/syntax.ts | 21 ++++----- src/js/components/FieldCell.tsx | 7 ++- src/js/electron/menu/actions/detailActions.ts | 15 +++---- src/js/electron/menu/actions/searchActions.ts | 10 ++--- src/js/flows/searchBar/actions.ts | 11 +++-- src/js/state/Boards/test.ts | 43 ------------------- src/js/state/Tiles/test.ts | 41 ------------------ zealot/zealot_mock.ts | 5 +-- 19 files changed, 72 insertions(+), 199 deletions(-) create mode 100644 app/core/formatters/format-zed.ts delete mode 100644 src/js/state/Boards/test.ts delete mode 100644 src/js/state/Tiles/test.ts diff --git a/app/core/formatters/format-zed.ts b/app/core/formatters/format-zed.ts new file mode 100644 index 0000000000..3e9bd33735 --- /dev/null +++ b/app/core/formatters/format-zed.ts @@ -0,0 +1,10 @@ +import brim from "src/js/brim" +import {withCommas} from "src/js/lib/fmt" +import {ZedPrimitive} from "zealot/zed" + +export function formatPrimitive(data: ZedPrimitive) { + if (data.isUnset()) return "⦻" + if (data.type.match(/int/)) return withCommas(data.toString()) + if (data.type === "time") return brim.time(this.toDate()).format() + return data.toString() +} diff --git a/app/detail/Fields.tsx b/app/detail/Fields.tsx index f06e24eb5b..ccd82c1324 100644 --- a/app/detail/Fields.tsx +++ b/app/detail/Fields.tsx @@ -1,11 +1,11 @@ import {Data, Name, Value} from "app/core/Data" +import {formatPrimitive} from "app/core/formatters/format-zed" import {typeClassNames} from "app/core/utils/type-class-names" import React, {memo, useCallback, useMemo, useState} from "react" import {useDispatch} from "react-redux" -import {createCell} from "src/js/brim/cell" import BrimTooltip from "src/js/components/BrimTooltip" import ColumnDescription from "src/js/components/LogDetails/ColumnDescription" -import {ZedField, ZedRecord} from "zealot/zed" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" import contextMenu from "./flows/contextMenu" import Panel from "./Panel" import PanelHeading from "./PanelHeading" @@ -36,7 +36,7 @@ const DataPanel = React.memo(function DataTable({ className={typeClassNames(field.data)} onContextMenu={() => onRightClick(field)} > - {createCell(field).display()} + {formatPrimitive(field.data as ZedPrimitive)} ))} diff --git a/ppl/summary/number.tsx b/ppl/summary/number.tsx index 6051a9cc1a..fb2b1a0a74 100644 --- a/ppl/summary/number.tsx +++ b/ppl/summary/number.tsx @@ -1,6 +1,8 @@ +import {formatPrimitive} from "app/core/formatters/format-zed" +import {typeClassNames} from "app/core/utils/type-class-names" import React from "react" -import {createCell} from "src/js/brim/cell" import styled from "styled-components" +import {ZedField, ZedPrimitive} from "zealot/zed" const Num = styled.div` font-size: 24px; @@ -14,9 +16,11 @@ const Num = styled.div` ` export default function Number({record}) { - const field = record?.getFields()[0] + const field = record?.getFields()[0] as ZedField if (!field) return null return ( - {createCell(field).display()} + + {formatPrimitive(field.data as ZedPrimitive)} + ) } diff --git a/ppl/summary/table.tsx b/ppl/summary/table.tsx index 7c54816d27..029dd8fc03 100644 --- a/ppl/summary/table.tsx +++ b/ppl/summary/table.tsx @@ -1,8 +1,9 @@ -import React from "react" -import {createCell} from "src/js/brim/cell" -import styled from "styled-components" import {scaleLinear} from "@vx/scale" +import {formatPrimitive} from "app/core/formatters/format-zed" import {cssVar, transparentize} from "polished" +import React from "react" +import styled from "styled-components" +import {ZedPrimitive} from "zealot/zed" type GridProps = {templateColumns: string | undefined} @@ -127,7 +128,7 @@ export default function Table({records, x}) { .getFields() .map((field, i) => ( - {createCell(field).display()} + {formatPrimitive(field.data as ZedPrimitive)} ))} diff --git a/scripts/rename.js b/scripts/rename.js index 2c9919b49b..f5f96063c1 100644 --- a/scripts/rename.js +++ b/scripts/rename.js @@ -1,12 +1,11 @@ const {Project} = require("ts-morph") const _ = require("lodash") -const {kebabCase} = require("lodash") const project = new Project({ tsConfigFilePath: "./tsconfig.json" }) -project.getSourceFiles().forEach((srcFile, i) => { +project.getSourceFiles().forEach((srcFile) => { // const oldName = srcFile.getBaseName() // const name = srcFile.getBaseNameWithoutExtension() // const newName = kebabCase(name) + srcFile.getExtension() diff --git a/scripts/test/search.deno.ts b/scripts/test/search.deno.ts index de1c37bebd..e5fcdf98f3 100644 --- a/scripts/test/search.deno.ts +++ b/scripts/test/search.deno.ts @@ -1,8 +1,4 @@ -import { - join, - fromFileUrl, - dirname -} from "https://deno.land/std@0.70.0/path/mod.ts" +import {join} from "https://deno.land/std@0.70.0/path/mod.ts" import {withZqd} from "../../test/api/helper/test_api.ts" const FILE = join(Deno.cwd(), Deno.args[0]) diff --git a/src/js/brim/cell.ts b/src/js/brim/cell.ts index c94b3870b5..cdce2f4c10 100644 --- a/src/js/brim/cell.ts +++ b/src/js/brim/cell.ts @@ -3,11 +3,6 @@ import {createComplexCell} from "./complexCell" import {createPrimitiveCell} from "./primitiveCell" export interface Cell { - name: string - queryableValue: () => string - stringValue: () => string - display: () => string - compound: () => boolean guessWidth: () => number } diff --git a/src/js/brim/complexCell.ts b/src/js/brim/complexCell.ts index abc5cc7f34..a741c4e070 100644 --- a/src/js/brim/complexCell.ts +++ b/src/js/brim/complexCell.ts @@ -29,12 +29,7 @@ export function createComplexCell({name, data}: Args) { container: data.constructor.name, length: items.length, item: (index: number) => items[index], - queryableValue() { - return items.map((cell) => cell.queryableValue()).join(" ") - }, - display() { - return items.map((cell) => cell.display()).join(", ") - }, + stringValue() { return data.toString() }, diff --git a/src/js/brim/primitiveCell.ts b/src/js/brim/primitiveCell.ts index 61193763f9..b7458aa1dd 100644 --- a/src/js/brim/primitiveCell.ts +++ b/src/js/brim/primitiveCell.ts @@ -1,18 +1,8 @@ -import {isEqual} from "lodash" import {ZedPrimitive} from "zealot/zed" -import {withCommas} from "../lib/fmt" -import brim from "./" export const ONE_CHAR = 7.39 export const FIELD_PAD = 14 export const PATH_PAD = 12 -const WHITE_SPACE = /\s+/ -const COMMA = /,/ -const STRING_TYPE = /^b?string$/ -const DOUBLE_QUOTE = /"/g -const ESCAPED_DOUBLE_QUOTE = '\\"' -const BACK_SLASH = /\\/g -const ESCAPED_BACK_SLASH = "\\\\" interface PrimitiveField { name: string @@ -30,17 +20,6 @@ export function createPrimitiveCell({name, data}: PrimitiveField) { serialize() { return {name, type, value} }, - queryableValue() { - if (this.value === null) return "null" - if (this.value === undefined) return "null" - if (this.type === "bool") return this.value === "T" ? "true" : "false" - let quote = [WHITE_SPACE, COMMA].some((reg) => reg.test(this.value)) - if (STRING_TYPE.test(this.type)) quote = true - const str = this.value - .replace(BACK_SLASH, ESCAPED_BACK_SLASH) - .replace(DOUBLE_QUOTE, ESCAPED_DOUBLE_QUOTE) - return quote ? `"${str}"` : str - }, stringValue(): string { if (value === null) return "null" else if (Array.isArray(value)) return value.join(",") @@ -50,24 +29,6 @@ export function createPrimitiveCell({name, data}: PrimitiveField) { toDate() { return new Date(+this.value * 1000) }, - - display() { - if (value === "(empty)") { - return "" - } else if (value === null) { - return "⦻" - } else if (type === "count") { - return withCommas(value.toString()) - } else if (type === "time") { - return brim.time(this.toDate()).format() - } else if (isEqual(value, {})) { - return "" - } else if (value === undefined) { - return "" - } else { - return value.toString() - } - }, compound() { return false }, diff --git a/src/js/brim/program.test.ts b/src/js/brim/program.test.ts index 0c8313e8aa..0880e37616 100644 --- a/src/js/brim/program.test.ts +++ b/src/js/brim/program.test.ts @@ -9,10 +9,9 @@ import { splitParts } from "../lib/Program" import brim from "./" -import {createCell} from "./cell" describe("excluding and including", () => { - const field = createCell(createField("uid", "123")) + const field = createField("uid", "123") test("excluding a field", () => { const program = brim @@ -29,7 +28,7 @@ describe("excluding and including", () => { .program( 'tx_hosts=2606:4700:30::681c:135e fuid!="F2nyqx46YRDAYe4c73" | sort' ) - .exclude(createCell(new ZedField({name: "source", data}))) + .exclude(new ZedField({name: "source", data})) .string() expect(program).toEqual( @@ -135,7 +134,7 @@ describe("drill down", () => { describe("count by", () => { test("empty program", () => { const data = new ZedPrimitive({type: "string", value: "heyo"}) - const field = createCell(new ZedField({name: "_path", data})) + const field = new ZedField({name: "_path", data}) const program = brim .program() .countBy(field) @@ -146,7 +145,7 @@ describe("count by", () => { test("append a count to an existing query", () => { const data = new ZedPrimitive({type: "string", value: "heyo"}) - const field = createCell(new ZedField({name: "query", data})) + const field = new ZedField({name: "query", data}) const program = brim .program("dns") .countBy(field) diff --git a/src/js/brim/program.ts b/src/js/brim/program.ts index c11e55ef5b..6c7f0a0984 100644 --- a/src/js/brim/program.ts +++ b/src/js/brim/program.ts @@ -1,32 +1,32 @@ import {isEqual} from "lodash" import {parse} from "zealot" -import {ZedRecord} from "zealot/zed" +import {ZedField, ZedRecord} from "zealot/zed" import {trim} from "../lib/Str" import stdlib from "../stdlib" import brim from "./" import {EVERYTHING_FILTER, FILTER_PROC, TUPLE_PROCS} from "./ast" -import {Cell, createCell} from "./cell" +import {createCell} from "./cell" export default function(p = "", pins: string[] = []) { p = concatPins(p, pins) return { - exclude(field: Cell) { + exclude(field: ZedField) { p = insertFilter(p, brim.syntax.exclude(field)) return this }, - include(field: Cell) { + include(field: ZedField) { p = insertFilter(p, brim.syntax.include(field)) return this }, - in(field: Cell) { + in(field: ZedField) { p = insertFilter(p, brim.syntax.in(field)) return this }, - notIn(field: Cell) { + notIn(field: ZedField) { p = insertFilter(p, brim.syntax.notIn(field)) return this }, @@ -62,7 +62,7 @@ export default function(p = "", pins: string[] = []) { return this }, - countBy(field: Cell) { + countBy(field: ZedField) { p = stdlib .string(p) .append(" | " + brim.syntax.countBy(field)) diff --git a/src/js/brim/syntax.ts b/src/js/brim/syntax.ts index 522af5034b..2301013c69 100644 --- a/src/js/brim/syntax.ts +++ b/src/js/brim/syntax.ts @@ -1,19 +1,20 @@ -import {Cell} from "./cell" +import {ZedField} from "zealot/zed" +import {toZql} from "../zql/toZql" export default { - exclude(field: Cell) { - return `${field.name}!=${field.queryableValue()}` + exclude(field: ZedField) { + return `${field.name}!=${toZql(field.data)}` }, - include(field: Cell) { - return `${field.name}=${field.queryableValue()}` + include(field: ZedField) { + return `${field.name}=${toZql(field.data)}` }, - in(field: Cell) { - return `${field.queryableValue()} in ${field.name}` + in(field: ZedField) { + return `${toZql(field.data)} in ${field.name}` }, - notIn(field: Cell) { - return `!${field.queryableValue()} in ${field.name}` + notIn(field: ZedField) { + return `!${toZql(field.data)} in ${field.name}` }, - countBy(field: Cell) { + countBy(field: ZedField) { return `count() by ${field.name}` }, sortBy(name: string, direction: "asc" | "desc") { diff --git a/src/js/components/FieldCell.tsx b/src/js/components/FieldCell.tsx index 3e15576e8f..85a2f959af 100644 --- a/src/js/components/FieldCell.tsx +++ b/src/js/components/FieldCell.tsx @@ -1,8 +1,8 @@ +import {formatPrimitive} from "app/core/formatters/format-zed" import {typeClassNames} from "app/core/utils/type-class-names" import classNames from "classnames" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed" -import {createCell} from "../brim/cell" +import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" type Props = {field: ZedField; record: ZedRecord} @@ -17,7 +17,6 @@ function getBackground(field, record) { } export default function FieldCell({field, record}: Props) { - const cell = createCell(field) return (
- {cell.display()} + {formatPrimitive(field.data as ZedPrimitive)}
) } diff --git a/src/js/electron/menu/actions/detailActions.ts b/src/js/electron/menu/actions/detailActions.ts index ef00667e32..a81c2cc63f 100644 --- a/src/js/electron/menu/actions/detailActions.ts +++ b/src/js/electron/menu/actions/detailActions.ts @@ -1,12 +1,12 @@ +import {toZql} from "src/js/zql/toZql" import { - ZedField, SerializedZedField, + SerializedZedRecord, + ZedField, ZedPrimitive, - ZedRecord, - SerializedZedRecord + ZedRecord } from "zealot/zed" import brim from "../../../brim" -import {createCell} from "../../../brim/cell" import {downloadPcap} from "../../../flows/downloadPcap" import {openNewSearchTab} from "../../../flows/openNewSearchWindow" import { @@ -65,9 +65,8 @@ function buildDetailActions() { name: "detail-cell-menu-fresh-include", label: "New search with this value", listener(dispatch, field: SerializedZedField) { - const cell = createCell(ZedField.deserialize(field)) dispatch(SearchBar.clearSearchBar()) - dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) + dispatch(SearchBar.changeSearchBarInput(toZql(field.data))) dispatch(openNewSearchTab()) } }), @@ -114,7 +113,7 @@ function buildDetailActions() { label: "Filter in field in new search", listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryIn(createCell(ZedField.deserialize(field)))) + dispatch(appendQueryIn(ZedField.deserialize(field))) dispatch(openNewSearchTab()) } }), @@ -123,7 +122,7 @@ function buildDetailActions() { label: "Filter not in field in new search", listener(dispatch, field: SerializedZedField) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryNotIn(createCell(ZedField.deserialize(field)))) + dispatch(appendQueryNotIn(ZedField.deserialize(field))) dispatch(openNewSearchTab()) } }), diff --git a/src/js/electron/menu/actions/searchActions.ts b/src/js/electron/menu/actions/searchActions.ts index 13fc9c6d40..fee9fad626 100644 --- a/src/js/electron/menu/actions/searchActions.ts +++ b/src/js/electron/menu/actions/searchActions.ts @@ -1,4 +1,5 @@ import lib from "src/js/lib" +import {toZql} from "src/js/zql/toZql" import { SerializedZedField, SerializedZedRecord, @@ -7,7 +8,6 @@ import { ZedRecord } from "zealot/zed" import brim from "../../../brim" -import {createCell} from "../../../brim/cell" import {downloadPcap} from "../../../flows/downloadPcap" import scrollToLog from "../../../flows/scrollToLog" import { @@ -70,9 +70,9 @@ function buildSearchActions() { name: "search-cell-menu-fresh-include", label: "New search with this value", listener(dispatch, data: SerializedZedField) { - const cell = createCell(ZedField.deserialize(data)) + const field = ZedField.deserialize(data) dispatch(SearchBar.clearSearchBar()) - dispatch(SearchBar.changeSearchBarInput(cell.queryableValue())) + dispatch(SearchBar.changeSearchBarInput(toZql(field.data))) dispatch(submitSearch()) } }), @@ -116,7 +116,7 @@ function buildSearchActions() { name: "search-cell-menu-in", label: "Filter in field", listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryIn(createCell(ZedField.deserialize(data)))) + dispatch(appendQueryIn(ZedField.deserialize(data))) dispatch(submitSearch()) } }), @@ -150,7 +150,7 @@ function buildSearchActions() { name: "search-cell-menu-not-in", label: "Filter not in field", listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryNotIn(createCell(ZedField.deserialize(data)))) + dispatch(appendQueryNotIn(ZedField.deserialize(data))) dispatch(submitSearch()) } }), diff --git a/src/js/flows/searchBar/actions.ts b/src/js/flows/searchBar/actions.ts index 2b8a585dff..4257417744 100644 --- a/src/js/flows/searchBar/actions.ts +++ b/src/js/flows/searchBar/actions.ts @@ -1,6 +1,5 @@ import {ZedField} from "zealot/zed" import brim from "../../brim" -import {Cell, createCell} from "../../brim/cell" import {onlyWhitespace} from "../../lib/Str" import SearchBar from "../../state/SearchBar" import { @@ -15,7 +14,7 @@ export function appendQueryInclude(field: ZedField): Thunk { SearchBar.changeSearchBarInput( brim .program(getSearchBarInputValue(getState())) - .include(createCell(field)) + .include(field) .string() ) ) @@ -28,7 +27,7 @@ export function appendQueryExclude(field: ZedField): Thunk { SearchBar.changeSearchBarInput( brim .program(getSearchBarInputValue(getState())) - .exclude(createCell(field)) + .exclude(field) .string() ) ) @@ -45,7 +44,7 @@ export function appendQueryCountBy(field: ZedField): Thunk { SearchBar.changeSearchBarInput( brim .program(program) - .countBy(createCell(field)) + .countBy(field) .string() ) ) @@ -72,7 +71,7 @@ export function appendQuerySortBy( } } -export function appendQueryIn(field: Cell): Thunk { +export function appendQueryIn(field: ZedField): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -85,7 +84,7 @@ export function appendQueryIn(field: Cell): Thunk { } } -export function appendQueryNotIn(field: Cell): Thunk { +export function appendQueryNotIn(field: ZedField): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( diff --git a/src/js/state/Boards/test.ts b/src/js/state/Boards/test.ts deleted file mode 100644 index b919c6102b..0000000000 --- a/src/js/state/Boards/test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import initTestStore from "src/js/test/initTestStore" -import Boards from "./index" - -let d, s -beforeEach(() => { - const store = initTestStore() - d = store.dispatch - s = (f) => f(store.getState()) -}) - -/* Skipping tests until security summary is shipped */ - -test.skip("initial state", () => { - expect(s(Boards.all)).toHaveLength(1) - expect(s(Boards.all)[0]).toEqual( - expect.objectContaining({ - title: "Security Summary" - }) - ) -}) - -test.skip("adding a board", () => { - d(Boards.create({id: "1", title: "hi", tiles: []})) - - expect(s(Boards.get("1"))).toEqual({ - id: "1", - title: "hi", - tiles: [] - }) -}) - -test.skip("removing a board", () => { - d(Boards.create({id: "1", title: "hi", tiles: []})) - d(Boards.delete("1")) - expect(s(Boards.get("1"))).toBe(undefined) -}) - -test.skip("add a tile id", () => { - d(Boards.create({id: "1", title: "hi", tiles: []})) - d(Boards.appendTile({id: "1", tileId: "100"})) - - expect(s(Boards.get("1"))).toEqual({id: "1", title: "hi", tiles: ["100"]}) -}) diff --git a/src/js/state/Tiles/test.ts b/src/js/state/Tiles/test.ts deleted file mode 100644 index 71230a63e8..0000000000 --- a/src/js/state/Tiles/test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import initTestStore from "src/js/test/initTestStore" -import Tiles, {Tile} from "./index" - -let s, d -beforeEach(() => { - const store = initTestStore() - d = store.dispatch - s = (f) => f(store.getState()) -}) - -/* Skipping tests until security summary is shipped */ - -test.skip("initial state", () => { - const tiles = s(Tiles.all) - expect(tiles.length).toBe(4) -}) - -test.skip("Adding a tile", () => { - const tile: Tile = { - id: "1", - title: "Count by Path", - query: "count() by _path", - format: {type: "number"}, - layout: {x: 0, y: 0, w: 3, h: 3} - } - d(Tiles.create(tile)) - expect(s(Tiles.get(tile.id))).toEqual(tile) -}) - -test.skip("Removing a tile", () => { - const tile: Tile = { - id: "1", - title: "Count by Path", - query: "count() by _path", - format: {type: "number"}, - layout: {x: 0, y: 0, w: 3, h: 3} - } - d(Tiles.create(tile)) - d(Tiles.delete(tile.id)) - expect(s(Tiles.get(tile.id))).toBe(undefined) -}) diff --git a/zealot/zealot_mock.ts b/zealot/zealot_mock.ts index ac251af5ae..12db0017c1 100644 --- a/zealot/zealot_mock.ts +++ b/zealot/zealot_mock.ts @@ -1,9 +1,8 @@ -import {createZealot} from "./zealot" import {FetchArgs} from "./fetcher/fetcher" import {createStream} from "./fetcher/stream" -import {createError} from "./util/error" import {Zealot, ZealotPayload} from "./types" -import {isConstructorDeclaration} from "typescript" +import {createError} from "./util/error" +import {createZealot} from "./zealot" type StubMode = "always" | "once" type RespWrap = typeof stream | typeof promise From 918dc13a323f4b7d1cd22f3ee8f34752b852efcf Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 22 Apr 2021 08:58:54 -0700 Subject: [PATCH 08/41] Started zed types implementation --- zealot/zed/context.ts | 75 +++++++++++++++++++++++++++++ zealot/zed/types/type-bstring.ts | 10 ++++ zealot/zed/types/type-def.ts | 7 +++ zealot/zed/types/type-duration.ts | 10 ++++ zealot/zed/types/type-ip.ts | 10 ++++ zealot/zed/types/type-null.ts | 10 ++++ zealot/zed/types/type-primitives.ts | 20 ++++++++ zealot/zed/types/type-record.ts | 30 ++++++++++++ zealot/zed/types/type-string.ts | 10 ++++ zealot/zed/types/type-time.ts | 10 ++++ zealot/zed/types/type-typename.ts | 10 ++++ zealot/zed/types/type-uint16.ts | 10 ++++ zealot/zed/types/type-uint64.ts | 10 ++++ zealot/zed/types/utils.ts | 9 ++++ zealot/zed/zjson.test.ts | 13 ++--- 15 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 zealot/zed/context.ts create mode 100644 zealot/zed/types/type-bstring.ts create mode 100644 zealot/zed/types/type-def.ts create mode 100644 zealot/zed/types/type-duration.ts create mode 100644 zealot/zed/types/type-ip.ts create mode 100644 zealot/zed/types/type-null.ts create mode 100644 zealot/zed/types/type-primitives.ts create mode 100644 zealot/zed/types/type-record.ts create mode 100644 zealot/zed/types/type-string.ts create mode 100644 zealot/zed/types/type-time.ts create mode 100644 zealot/zed/types/type-typename.ts create mode 100644 zealot/zed/types/type-uint16.ts create mode 100644 zealot/zed/types/type-uint64.ts create mode 100644 zealot/zed/types/utils.ts diff --git a/zealot/zed/context.ts b/zealot/zed/context.ts new file mode 100644 index 0000000000..a10f49af19 --- /dev/null +++ b/zealot/zed/context.ts @@ -0,0 +1,75 @@ +import {TypeDef} from "./types/type-def" +import {TypeRecord} from "./types/type-record" +import {StreamObject, Type} from "./zjson" +import primitives from "./types/type-primitives" + +export class ZedContext { + id: number + typeByStringValue: {} + typeByName: {} + typedefs: {} + typetype: {} + + constructor() { + this.id = 0 + this.typeByStringValue = {} + this.typeByName = {} + this.typedefs = {} + this.typetype = {} + } + + decode(objects: StreamObject[]) { + return objects.map((object) => this.decodeRecord(object)) + } + + decodeRecord({schema, types, values}: StreamObject) { + const typedefs = {} + types && types.forEach((type) => this.decodeType(type, typedefs)) + const type = typedefs[schema] as TypeRecord + return type.create(values, typedefs) + } + + decodeType(obj: Type, typedefs) { + switch (obj.kind) { + case "record": + return this.lookupTypeRecord( + obj.fields.map(({name, type}) => ({ + name, + type: this.decodeType(type, typedefs) + })) + ) + case "typedef": + var type = this.decodeType(obj.type, typedefs) + typedefs[obj.name] = type + if (isNaN(obj.name as any)) { + type = new TypeDef(obj.name, type) + this.typedefs[obj.name] = type + } + return type + case "primitive": + var type = primitives[obj.name] + if (!type) throw `Implement primitive: ${obj.name}` + return type + case "typename": + return typedefs[obj.name] + default: + throw `Implement decoding: ${obj.kind}` + } + } + + lookupTypeRecord(fields) { + const string = TypeRecord.stringify(fields) + if (string in this.typeByStringValue) { + return this.typeByStringValue[string] + } else { + return this.alloc(string, new TypeRecord(fields)) + } + } + + alloc(key, type) { + type.id = this.id++ + this.typeByStringValue[key] = type + this.typeByName[type.id.toString()] = type + return type + } +} diff --git a/zealot/zed/types/type-bstring.ts b/zealot/zed/types/type-bstring.ts new file mode 100644 index 0000000000..dab68a865f --- /dev/null +++ b/zealot/zed/types/type-bstring.ts @@ -0,0 +1,10 @@ +class TypeOfBstring { + name = "bstring" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeBstring = new TypeOfBstring() diff --git a/zealot/zed/types/type-def.ts b/zealot/zed/types/type-def.ts new file mode 100644 index 0000000000..86adf97d81 --- /dev/null +++ b/zealot/zed/types/type-def.ts @@ -0,0 +1,7 @@ +export class TypeDef { + constructor(name, type) { + this.kind = "typedef" + this.name = name + this.type = type + } +} diff --git a/zealot/zed/types/type-duration.ts b/zealot/zed/types/type-duration.ts new file mode 100644 index 0000000000..cb3e0e23a1 --- /dev/null +++ b/zealot/zed/types/type-duration.ts @@ -0,0 +1,10 @@ +class TypeOfDuration { + name = "duration" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeDuration = new TypeOfDuration() diff --git a/zealot/zed/types/type-ip.ts b/zealot/zed/types/type-ip.ts new file mode 100644 index 0000000000..c88640abe5 --- /dev/null +++ b/zealot/zed/types/type-ip.ts @@ -0,0 +1,10 @@ +class TypeOfIp { + name = "ip" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeIp = new TypeOfIp() diff --git a/zealot/zed/types/type-null.ts b/zealot/zed/types/type-null.ts new file mode 100644 index 0000000000..89ce9364cf --- /dev/null +++ b/zealot/zed/types/type-null.ts @@ -0,0 +1,10 @@ +class TypeOfNull { + name = "null" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeNull = new TypeOfNull() diff --git a/zealot/zed/types/type-primitives.ts b/zealot/zed/types/type-primitives.ts new file mode 100644 index 0000000000..44a3b8d0df --- /dev/null +++ b/zealot/zed/types/type-primitives.ts @@ -0,0 +1,20 @@ +import {TypeBstring} from "./type-bstring" +import {TypeDuration} from "./type-duration" +import {TypeIp} from "./type-ip" +import {TypeNull} from "./type-null" +import {TypeString} from "./type-string" +import {TypeTime} from "./type-time" +import {TypeUint16} from "./type-uint16" +import {TypeUint64} from "./type-uint64" + +// These all point to a single instance of their type +export default { + string: TypeString, + bstring: TypeBstring, + time: TypeTime, + ip: TypeIp, + uint16: TypeUint16, + duration: TypeDuration, + uint64: TypeUint64, + null: TypeNull +} diff --git a/zealot/zed/types/type-record.ts b/zealot/zed/types/type-record.ts new file mode 100644 index 0000000000..7c07bf8280 --- /dev/null +++ b/zealot/zed/types/type-record.ts @@ -0,0 +1,30 @@ +import {ZedRecord} from ".." +import {typeId} from "./utils" + +export class TypeRecord { + kind = "record" + constructor(fields) { + this.fields = fields + } + + static stringify(fields) { + let s = "{" + let sep = "" + fields.forEach((f) => { + // XXX need to check if name has funny chars + s += sep + f.name + ":" + typeId(f.type) + sep = "," + }) + s += "}" + return s + } + create(values, typedefs) { + return new ZedRecord( + this, + this.fields.map((field, index) => ({ + name: field.name, + value: field.type.create(values[index]) + })) + ) + } +} diff --git a/zealot/zed/types/type-string.ts b/zealot/zed/types/type-string.ts new file mode 100644 index 0000000000..cd59c8c2c9 --- /dev/null +++ b/zealot/zed/types/type-string.ts @@ -0,0 +1,10 @@ +class TypeOfString { + name = "string" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeString = new TypeOfString() diff --git a/zealot/zed/types/type-time.ts b/zealot/zed/types/type-time.ts new file mode 100644 index 0000000000..6c097ade60 --- /dev/null +++ b/zealot/zed/types/type-time.ts @@ -0,0 +1,10 @@ +class TypeOfTime { + name = "time" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeTime = new TypeOfTime() diff --git a/zealot/zed/types/type-typename.ts b/zealot/zed/types/type-typename.ts new file mode 100644 index 0000000000..c0ae568d4d --- /dev/null +++ b/zealot/zed/types/type-typename.ts @@ -0,0 +1,10 @@ +class TypeOfUint16 { + name = "uint16" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeUint16 = new TypeOfUint16() diff --git a/zealot/zed/types/type-uint16.ts b/zealot/zed/types/type-uint16.ts new file mode 100644 index 0000000000..c0ae568d4d --- /dev/null +++ b/zealot/zed/types/type-uint16.ts @@ -0,0 +1,10 @@ +class TypeOfUint16 { + name = "uint16" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeUint16 = new TypeOfUint16() diff --git a/zealot/zed/types/type-uint64.ts b/zealot/zed/types/type-uint64.ts new file mode 100644 index 0000000000..8ec67d2e14 --- /dev/null +++ b/zealot/zed/types/type-uint64.ts @@ -0,0 +1,10 @@ +class TypeOfUint64 { + name = "uint64" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } +} + +export const TypeUint64 = new TypeOfUint64() diff --git a/zealot/zed/types/utils.ts b/zealot/zed/types/utils.ts new file mode 100644 index 0000000000..5ce201eafe --- /dev/null +++ b/zealot/zed/types/utils.ts @@ -0,0 +1,9 @@ +export function typeId(type) { + switch (type.kind) { + case "primitive": + case "typedef": + return type.name + default: + return type.id.toString() + } +} diff --git a/zealot/zed/zjson.test.ts b/zealot/zed/zjson.test.ts index b4cd74779a..49ca94a7ee 100644 --- a/zealot/zed/zjson.test.ts +++ b/zealot/zed/zjson.test.ts @@ -1,6 +1,5 @@ import {execSync} from "child_process" -import {deserialize} from "./json" -import {decode} from "./zjson" +import {ZedContext} from "./context" function zq(q, file) { const zed = "/Users/jkerr/tools/go/bin/zed" @@ -13,9 +12,11 @@ function zq(q, file) { test("can correlate?", () => { const file = "test/data/sample.zson" - const list = decode(zq("*", file)) - const json = list.rows.map((row) => row.serialize()) - const list2 = json.map(deserialize) + const input = zq("*", file) + const ctx = new ZedContext() + const list = ctx.decode(input) + // const json = list.rows.map((row) => row.serialize()) + // const list2 = json.map(deserialize) - expect(list2).toEqual(list.rows) + // expect(list2).toEqual(list.rows) }) From b4de42d35666603579ec2330f3c044964485234f Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 22 Apr 2021 17:10:28 -0700 Subject: [PATCH 09/41] got a test passing --- zealot/zed/context.ts | 70 +++++++++++++++++++++++++---- zealot/zed/types/type-array.ts | 23 ++++++++++ zealot/zed/types/type-bstring.ts | 11 ++++- zealot/zed/types/type-def.ts | 10 ++--- zealot/zed/types/type-duration.ts | 9 +++- zealot/zed/types/type-float64.ts | 17 +++++++ zealot/zed/types/type-int64.ts | 17 +++++++ zealot/zed/types/type-ip.ts | 9 +++- zealot/zed/types/type-map.ts | 24 ++++++++++ zealot/zed/types/type-net.ts | 17 +++++++ zealot/zed/types/type-null.ts | 9 +++- zealot/zed/types/type-primitives.ts | 32 +++++++++---- zealot/zed/types/type-record.ts | 29 +++++++++--- zealot/zed/types/type-set.ts | 23 ++++++++++ zealot/zed/types/type-string.ts | 8 +++- zealot/zed/types/type-time.ts | 9 +++- zealot/zed/types/type-typename.ts | 13 ++++-- zealot/zed/types/type-uint16.ts | 11 ++++- zealot/zed/types/type-uint32.ts | 17 +++++++ zealot/zed/types/type-uint64.ts | 11 ++++- zealot/zed/types/type-union.ts | 18 ++++++++ zealot/zed/types/types.ts | 22 +++++++++ zealot/zed/values/array.ts | 15 +++++++ zealot/zed/values/bstring.ts | 18 ++++++++ zealot/zed/values/duration.ts | 18 ++++++++ zealot/zed/values/float64.ts | 18 ++++++++ zealot/zed/values/int64.ts | 18 ++++++++ zealot/zed/values/ip.ts | 18 ++++++++ zealot/zed/values/map.ts | 22 +++++++++ zealot/zed/values/net.ts | 18 ++++++++ zealot/zed/values/null.ts | 18 ++++++++ zealot/zed/values/record.ts | 25 +++++++++++ zealot/zed/values/set.ts | 15 +++++++ zealot/zed/values/string.ts | 18 ++++++++ zealot/zed/values/time.ts | 18 ++++++++ zealot/zed/values/typename.ts | 18 ++++++++ zealot/zed/values/types.ts | 19 ++++++++ zealot/zed/values/uint16.ts | 18 ++++++++ zealot/zed/values/uint32.ts | 18 ++++++++ zealot/zed/values/uint64.ts | 18 ++++++++ zealot/zed/values/union.ts | 18 ++++++++ 41 files changed, 714 insertions(+), 43 deletions(-) create mode 100644 zealot/zed/types/type-array.ts create mode 100644 zealot/zed/types/type-float64.ts create mode 100644 zealot/zed/types/type-int64.ts create mode 100644 zealot/zed/types/type-map.ts create mode 100644 zealot/zed/types/type-net.ts create mode 100644 zealot/zed/types/type-set.ts create mode 100644 zealot/zed/types/type-uint32.ts create mode 100644 zealot/zed/types/type-union.ts create mode 100644 zealot/zed/types/types.ts create mode 100644 zealot/zed/values/array.ts create mode 100644 zealot/zed/values/bstring.ts create mode 100644 zealot/zed/values/duration.ts create mode 100644 zealot/zed/values/float64.ts create mode 100644 zealot/zed/values/int64.ts create mode 100644 zealot/zed/values/ip.ts create mode 100644 zealot/zed/values/map.ts create mode 100644 zealot/zed/values/net.ts create mode 100644 zealot/zed/values/null.ts create mode 100644 zealot/zed/values/record.ts create mode 100644 zealot/zed/values/set.ts create mode 100644 zealot/zed/values/string.ts create mode 100644 zealot/zed/values/time.ts create mode 100644 zealot/zed/values/typename.ts create mode 100644 zealot/zed/values/types.ts create mode 100644 zealot/zed/values/uint16.ts create mode 100644 zealot/zed/values/uint32.ts create mode 100644 zealot/zed/values/uint64.ts create mode 100644 zealot/zed/values/union.ts diff --git a/zealot/zed/context.ts b/zealot/zed/context.ts index a10f49af19..960258f954 100644 --- a/zealot/zed/context.ts +++ b/zealot/zed/context.ts @@ -2,6 +2,11 @@ import {TypeDef} from "./types/type-def" import {TypeRecord} from "./types/type-record" import {StreamObject, Type} from "./zjson" import primitives from "./types/type-primitives" +import {TypeArray} from "./types/type-array" +import {TypeSet} from "./types/type-set" +import {TypeUnion} from "./types/type-union" +import types from "tree-model/types" +import {TypeMap} from "./types/type-map" export class ZedContext { id: number @@ -19,11 +24,11 @@ export class ZedContext { } decode(objects: StreamObject[]) { - return objects.map((object) => this.decodeRecord(object)) + const typedefs = {} + return objects.map((object) => this.decodeRecord(object, typedefs)) } - decodeRecord({schema, types, values}: StreamObject) { - const typedefs = {} + decodeRecord({schema, types, values}: StreamObject, typedefs) { types && types.forEach((type) => this.decodeType(type, typedefs)) const type = typedefs[schema] as TypeRecord return type.create(values, typedefs) @@ -39,19 +44,32 @@ export class ZedContext { })) ) case "typedef": - var type = this.decodeType(obj.type, typedefs) - typedefs[obj.name] = type + var innerType = this.decodeType(obj.type, typedefs) + typedefs[obj.name] = innerType if (isNaN(obj.name as any)) { - type = new TypeDef(obj.name, type) - this.typedefs[obj.name] = type + var def = new TypeDef(obj.name, innerType) + this.typedefs[obj.name] = def } - return type + return innerType case "primitive": var type = primitives[obj.name] if (!type) throw `Implement primitive: ${obj.name}` return type case "typename": return typedefs[obj.name] + case "array": + return this.lookupTypeArray(this.decodeType(obj.type, typedefs)) + case "set": + return this.lookupTypeSet(this.decodeType(obj.type, typedefs)) + case "union": + return this.lookupTypeUnion( + obj.types.map((t) => this.decodeType(t, typedefs)) + ) + case "map": + return this.lookupTypeMap( + this.decodeType(obj.key_type, typedefs), + this.decodeType(obj.val_type, typedefs) + ) default: throw `Implement decoding: ${obj.kind}` } @@ -66,6 +84,42 @@ export class ZedContext { } } + lookupTypeArray(type) { + const string = TypeArray.stringify(type) + if (string in this.typeByStringValue) { + return this.typeByStringValue[string] + } else { + return this.alloc(string, new TypeArray(type)) + } + } + + lookupTypeSet(type) { + const string = TypeSet.stringify(type) + if (string in this.typeByStringValue) { + return this.typeByStringValue[string] + } else { + return this.alloc(string, new TypeSet(type)) + } + } + + lookupTypeUnion(types) { + const string = TypeUnion.stringify(types) + if (string in this.typeByStringValue) { + return this.typeByStringValue[string] + } else { + return this.alloc(string, new TypeUnion(types)) + } + } + + lookupTypeMap(keyType, valType) { + const string = TypeMap.stringify(keyType, valType) + if (string in this.typeByStringValue) { + return this.typeByStringValue[string] + } else { + return this.alloc(string, new TypeMap(keyType, valType)) + } + } + alloc(key, type) { type.id = this.id++ this.typeByStringValue[key] = type diff --git a/zealot/zed/types/type-array.ts b/zealot/zed/types/type-array.ts new file mode 100644 index 0000000000..cf790ddef7 --- /dev/null +++ b/zealot/zed/types/type-array.ts @@ -0,0 +1,23 @@ +import {ZedType} from "./types" +import {typeId} from "./utils" +import {ZedArray} from "../values/array" + +export class TypeArray { + kind = "array" + type: ZedType + + constructor(type) { + this.type = type + } + + static stringify(type: ZedType) { + return `[${typeId(type)}]` + } + + create(values, typedefs) { + return new ZedArray( + this, + values.map((value) => this.type.create(value, typedefs)) + ) + } +} diff --git a/zealot/zed/types/type-bstring.ts b/zealot/zed/types/type-bstring.ts index dab68a865f..5e1aa7fe98 100644 --- a/zealot/zed/types/type-bstring.ts +++ b/zealot/zed/types/type-bstring.ts @@ -1,10 +1,17 @@ -class TypeOfBstring { +import {BString} from "../values/bstring" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfBString implements PrimitiveTypeInterface { name = "bstring" kind = "primitive" serialize() { return {kind: this.kind, name: this.name} } + + create(value: string) { + return new BString(value) + } } -export const TypeBstring = new TypeOfBstring() +export const TypeBString = new TypeOfBString() diff --git a/zealot/zed/types/type-def.ts b/zealot/zed/types/type-def.ts index 86adf97d81..eec048bf15 100644 --- a/zealot/zed/types/type-def.ts +++ b/zealot/zed/types/type-def.ts @@ -1,7 +1,7 @@ +import {ZedType} from "./types" + export class TypeDef { - constructor(name, type) { - this.kind = "typedef" - this.name = name - this.type = type - } + kind = "typedef" + + constructor(public name: string, public type: ZedType) {} } diff --git a/zealot/zed/types/type-duration.ts b/zealot/zed/types/type-duration.ts index cb3e0e23a1..e46ee65116 100644 --- a/zealot/zed/types/type-duration.ts +++ b/zealot/zed/types/type-duration.ts @@ -1,10 +1,17 @@ -class TypeOfDuration { +import {Duration} from "../values/duration" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfDuration implements PrimitiveTypeInterface { name = "duration" kind = "primitive" serialize() { return {kind: this.kind, name: this.name} } + + create(value: string) { + return new Duration(value) + } } export const TypeDuration = new TypeOfDuration() diff --git a/zealot/zed/types/type-float64.ts b/zealot/zed/types/type-float64.ts new file mode 100644 index 0000000000..271e13d75c --- /dev/null +++ b/zealot/zed/types/type-float64.ts @@ -0,0 +1,17 @@ +import {Float64} from "../values/float64" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfFloat64 implements PrimitiveTypeInterface { + name = "float64" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } + + create(value) { + return new Float64(value) + } +} + +export const TypeFloat64 = new TypeOfFloat64() diff --git a/zealot/zed/types/type-int64.ts b/zealot/zed/types/type-int64.ts new file mode 100644 index 0000000000..94fca5b927 --- /dev/null +++ b/zealot/zed/types/type-int64.ts @@ -0,0 +1,17 @@ +import {Int64} from "../values/int64" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfInt64 implements PrimitiveTypeInterface { + name = "int64" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } + + create(value) { + return new Int64(value) + } +} + +export const TypeInt64 = new TypeOfInt64() diff --git a/zealot/zed/types/type-ip.ts b/zealot/zed/types/type-ip.ts index c88640abe5..db5e0108d3 100644 --- a/zealot/zed/types/type-ip.ts +++ b/zealot/zed/types/type-ip.ts @@ -1,10 +1,17 @@ -class TypeOfIp { +import {Ip} from "../values/ip" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfIp implements PrimitiveTypeInterface { name = "ip" kind = "primitive" serialize() { return {kind: this.kind, name: this.name} } + + create(value: string) { + return new Ip(value) + } } export const TypeIp = new TypeOfIp() diff --git a/zealot/zed/types/type-map.ts b/zealot/zed/types/type-map.ts new file mode 100644 index 0000000000..291798c1fd --- /dev/null +++ b/zealot/zed/types/type-map.ts @@ -0,0 +1,24 @@ +import {ZedType} from "./types" +import {typeId} from "./utils" +import {ZedMap} from "../values/map" +export class TypeMap { + kind = "union" + + constructor(public keyType: ZedType, public valType: ZedType) {} + + static stringify(keyType, valType) { + return `|{` + typeId(keyType) + "," + typeId(valType) + "}|" + } + + create(value, typedefs) { + return new ZedMap( + this, + new Map( + value.map((entry) => [ + this.keyType.create(entry[0], typedefs), + this.valType.create(entry[1], typedefs) + ]) + ) + ) + } +} diff --git a/zealot/zed/types/type-net.ts b/zealot/zed/types/type-net.ts new file mode 100644 index 0000000000..2e9ba0c4a0 --- /dev/null +++ b/zealot/zed/types/type-net.ts @@ -0,0 +1,17 @@ +import {Net} from "../values/net" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfNet implements PrimitiveTypeInterface { + name = "net" + kind = "primitive" + + serialize() { + return {kind: this.kind, name: this.name} + } + + create(value: string) { + return new Net(value) + } +} + +export const TypeNet = new TypeOfNet() diff --git a/zealot/zed/types/type-null.ts b/zealot/zed/types/type-null.ts index 89ce9364cf..869ce5b48d 100644 --- a/zealot/zed/types/type-null.ts +++ b/zealot/zed/types/type-null.ts @@ -1,10 +1,17 @@ -class TypeOfNull { +import {Null} from "../values/null" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfNull implements PrimitiveTypeInterface { name = "null" kind = "primitive" serialize() { return {kind: this.kind, name: this.name} } + + create() { + return null + } } export const TypeNull = new TypeOfNull() diff --git a/zealot/zed/types/type-primitives.ts b/zealot/zed/types/type-primitives.ts index 44a3b8d0df..834d3e73a2 100644 --- a/zealot/zed/types/type-primitives.ts +++ b/zealot/zed/types/type-primitives.ts @@ -1,20 +1,34 @@ -import {TypeBstring} from "./type-bstring" +import {TypeBString} from "./type-bstring" import {TypeDuration} from "./type-duration" +import {TypeFloat64} from "./type-float64" +import {TypeInt64} from "./type-int64" import {TypeIp} from "./type-ip" +import {TypeNet} from "./type-net" import {TypeNull} from "./type-null" import {TypeString} from "./type-string" import {TypeTime} from "./type-time" -import {TypeUint16} from "./type-uint16" -import {TypeUint64} from "./type-uint64" +import {TypeTypename} from "./type-typename" +import {TypeUInt16} from "./type-uint16" +import {TypeUInt32} from "./type-uint32" +import {TypeUInt64} from "./type-uint64" // These all point to a single instance of their type -export default { +const primitives = { string: TypeString, - bstring: TypeBstring, + bstring: TypeBString, time: TypeTime, ip: TypeIp, - uint16: TypeUint16, + uint16: TypeUInt16, duration: TypeDuration, - uint64: TypeUint64, - null: TypeNull -} + uint64: TypeUInt64, + uint32: TypeUInt32, + int64: TypeInt64, + null: TypeNull, + typename: TypeTypename, + net: TypeNet, + float64: TypeFloat64 +} as const + +export type PrimitiveTypes = typeof primitives[keyof typeof primitives] + +export default primitives diff --git a/zealot/zed/types/type-record.ts b/zealot/zed/types/type-record.ts index 7c07bf8280..95b8ae540d 100644 --- a/zealot/zed/types/type-record.ts +++ b/zealot/zed/types/type-record.ts @@ -1,8 +1,16 @@ -import {ZedRecord} from ".." +import {Record} from "../values/record" +import {Value} from "../zjson" +import {ZedType} from "./types" import {typeId} from "./utils" +type TypeField = { + name: string + type: ZedType +} export class TypeRecord { kind = "record" + fields: TypeField[] + constructor(fields) { this.fields = fields } @@ -18,13 +26,20 @@ export class TypeRecord { s += "}" return s } - create(values, typedefs) { - return new ZedRecord( + + create(values: Value, typedefs: object) { + return new Record( this, - this.fields.map((field, index) => ({ - name: field.name, - value: field.type.create(values[index]) - })) + this.fields.map((field, index) => { + const value = values[index] + if (!field.type.create) { + console.log(field.type) + } + return { + name: field.name, + value: field.type.create(value, typedefs) + } + }) ) } } diff --git a/zealot/zed/types/type-set.ts b/zealot/zed/types/type-set.ts new file mode 100644 index 0000000000..da44fee8a3 --- /dev/null +++ b/zealot/zed/types/type-set.ts @@ -0,0 +1,23 @@ +import {ZedType} from "./types" +import {typeId} from "./utils" +import {Set} from "../values/set" + +export class TypeSet { + kind = "set" + type: ZedType + + constructor(type) { + this.type = type + } + + static stringify(type: ZedType) { + return `|[${typeId(type)}]|` + } + + create(values, typedefs) { + return new Set( + this, + values.map((v) => this.type.create(v, typedefs)) + ) + } +} diff --git a/zealot/zed/types/type-string.ts b/zealot/zed/types/type-string.ts index cd59c8c2c9..11e680afec 100644 --- a/zealot/zed/types/type-string.ts +++ b/zealot/zed/types/type-string.ts @@ -1,10 +1,16 @@ -class TypeOfString { +import {PrimitiveTypeInterface} from "./types" +import {String} from "../values/string" +class TypeOfString implements PrimitiveTypeInterface { name = "string" kind = "primitive" serialize() { return {kind: this.kind, name: this.name} } + + create(value: string) { + return new String(value) + } } export const TypeString = new TypeOfString() diff --git a/zealot/zed/types/type-time.ts b/zealot/zed/types/type-time.ts index 6c097ade60..d01be6c734 100644 --- a/zealot/zed/types/type-time.ts +++ b/zealot/zed/types/type-time.ts @@ -1,10 +1,17 @@ -class TypeOfTime { +import {Time} from "../values/time" +import {PrimitiveTypeInterface} from "./types" + +class TypeOfTime implements PrimitiveTypeInterface
{column.name}{column.name} +
} -export function TableHeader({column}: {column: ZedField}) { +export function TableHeader({column}: {column: zed.Field}) { return } type Props = { - record: ZedRecord - field: ZedField - onRightClick?: (f: ZedField, r: ZedRecord) => void + record: zed.Record + field: zed.Field + onRightClick?: (f: zed.Field, r: zed.Record) => void } export function TableData({field, record, onRightClick}: Props) { diff --git a/src/js/components/Tables/VerticalTable.tsx b/src/js/components/Tables/VerticalTable.tsx index e0e303b229..a420fe40d6 100644 --- a/src/js/components/Tables/VerticalTable.tsx +++ b/src/js/components/Tables/VerticalTable.tsx @@ -1,12 +1,12 @@ import classNames from "classnames" import React from "react" -import {ZedField, ZedRecord} from "zealot/zed" +import {zed} from "zealot" import Table, {TableData, TableHeader} from "./Table" type Props = { - descriptor: ZedField[] - record: ZedRecord - onRightClick?: (f: ZedField, r: ZedRecord) => void + descriptor: zed.Field[] + record: zed.Record + onRightClick?: (f: zed.Field, r: zed.Record) => void light?: boolean } diff --git a/src/js/components/Viewer/Chunk.tsx b/src/js/components/Viewer/Chunk.tsx index 43dcc3e75d..9e00452830 100644 --- a/src/js/components/Viewer/Chunk.tsx +++ b/src/js/components/Viewer/Chunk.tsx @@ -1,5 +1,5 @@ import React from "react" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import TableColumns from "../../models/TableColumns" import {RowRenderer, ViewerDimens} from "../../types" import * as Styler from "./Styler" @@ -9,7 +9,7 @@ type Props = { columns: TableColumns dimens: ViewerDimens rows: number[] - logs: ZedRecord[] + logs: zed.Record[] } export default class Chunk extends React.Component { diff --git a/src/js/components/Viewer/Viewer.tsx b/src/js/components/Viewer/Viewer.tsx index ad020704e3..415dbacda0 100644 --- a/src/js/components/Viewer/Viewer.tsx +++ b/src/js/components/Viewer/Viewer.tsx @@ -1,5 +1,5 @@ import React, {useEffect, useRef, useState} from "react" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import lib from "../../lib" import ScrollHooks from "../../lib/ScrollHooks" import TableColumns from "../../models/TableColumns" @@ -16,7 +16,7 @@ type Props = { dimens: ViewerDimens tableColumns: TableColumns renderRow: RowRenderer - logs: ZedRecord[] + logs: zed.Record[] onLastChunk?: Function renderEnd: () => any scrollPos: ScrollPosition diff --git a/src/js/electron/menu/actions/detailActions.ts b/src/js/electron/menu/actions/detailActions.ts index a81c2cc63f..3f06562747 100644 --- a/src/js/electron/menu/actions/detailActions.ts +++ b/src/js/electron/menu/actions/detailActions.ts @@ -1,11 +1,5 @@ import {toZql} from "src/js/zql/toZql" -import { - SerializedZedField, - SerializedZedRecord, - ZedField, - ZedPrimitive, - ZedRecord -} from "zealot/zed" +import {ZealotContext, zed, zjson} from "zealot" import brim from "../../../brim" import {downloadPcap} from "../../../flows/downloadPcap" import {openNewSearchTab} from "../../../flows/openNewSearchWindow" @@ -31,52 +25,53 @@ function buildDetailActions() { copy: action({ name: "detail-cell-menu-copy", label: "Copy", - listener(_dispatch, data: SerializedZedField) { - const f = ZedField.deserialize(data) + listener(_dispatch, data: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(data) lib.doc.copyToClipboard(f.data.toString()) } }), countBy: action({ name: "detail-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: SerializedZedField) { + listener(dispatch, data: zjson.FieldRootRecord) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryCountBy(ZedField.deserialize(data))) + dispatch(appendQueryCountBy(ZealotContext.decodeField(data))) dispatch(openNewSearchTab()) } }), detail: action({ name: "detail-cell-menu-detail", label: "View details", - listener(dispatch, log: SerializedZedRecord) { - dispatch(viewLogDetail(ZedRecord.deserialize(log))) + listener(dispatch, log: zjson.RootRecord) { + dispatch(viewLogDetail(ZealotContext.decodeRecord(log))) } }), exclude: action({ name: "detail-cell-menu-exclude", label: "Filter != value in new search", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryExclude(ZedField.deserialize(field))) + dispatch(appendQueryExclude(ZealotContext.decodeField(field))) dispatch(openNewSearchTab()) } }), freshInclude: action({ name: "detail-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(field) dispatch(SearchBar.clearSearchBar()) - dispatch(SearchBar.changeSearchBarInput(toZql(field.data))) + dispatch(SearchBar.changeSearchBarInput(toZql(f.value))) dispatch(openNewSearchTab()) } }), fromTime: action({ name: "detail-cell-menu-from-time", label: 'Use as "start" time in new search', - listener(dispatch, fieldJSON: SerializedZedField) { - const field = ZedField.deserialize(fieldJSON) + listener(dispatch, fieldJSON: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(fieldJSON) const data = field.data - if (data instanceof ZedPrimitive && data.type === "time") { + if (data instanceof zed.Time) { dispatch(SearchBar.clearSearchBar()) dispatch(tab.setFrom(brim.time(data.toDate()).toTs())) dispatch(openNewSearchTab()) @@ -86,10 +81,10 @@ function buildDetailActions() { groupByDrillDown: action({ name: "detail-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program, log: SerializedZedRecord) { + listener(dispatch, program, log: zjson.RootRecord) { const newProgram = brim .program(program) - .drillDown(ZedRecord.deserialize(log)) + .drillDown(ZealotContext.decodeRecord(log)) .string() if (newProgram) { @@ -102,27 +97,27 @@ function buildDetailActions() { include: action({ name: "detail-cell-menu-include", label: "Filter = value in new search", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryInclude(ZedField.deserialize(field))) + dispatch(appendQueryInclude(ZealotContext.decodeField(field))) dispatch(openNewSearchTab()) } }), in: action({ name: "detail-cell-menu-in", label: "Filter in field in new search", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryIn(ZedField.deserialize(field))) + dispatch(appendQueryIn(ZealotContext.decodeField(field))) dispatch(openNewSearchTab()) } }), notIn: action({ name: "detail-cell-menu-not-in", label: "Filter not in field in new search", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { dispatch(SearchBar.clearSearchBar()) - dispatch(appendQueryNotIn(ZedField.deserialize(field))) + dispatch(appendQueryNotIn(ZealotContext.decodeField(field))) dispatch(openNewSearchTab()) } }), @@ -137,34 +132,36 @@ function buildDetailActions() { pcaps: action({ name: "detail-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, log: SerializedZedRecord) { - dispatch(downloadPcap(ZedRecord.deserialize(log))) + listener(dispatch, log: zjson.RootRecord) { + dispatch(downloadPcap(ZealotContext.decodeRecord(log))) } }), sortAsc: action({ name: "detail-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(field) dispatch(SearchBar.clearSearchBar()) - dispatch(appendQuerySortBy(field.name, "asc")) + dispatch(appendQuerySortBy(f.name, "asc")) dispatch(openNewSearchTab()) } }), sortDesc: action({ name: "detail-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, field: SerializedZedField) { + listener(dispatch, field: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(field) dispatch(SearchBar.clearSearchBar()) - dispatch(appendQuerySortBy(field.name, "desc")) + dispatch(appendQuerySortBy(f.name, "desc")) dispatch(openNewSearchTab()) } }), toTime: action({ name: "detail-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) - if (field.data instanceof ZedPrimitive && field.data.type === "time") { + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) + if (field.data instanceof zed.Time) { dispatch(SearchBar.clearSearchBar()) dispatch( tab.setTo( @@ -181,16 +178,16 @@ function buildDetailActions() { virusTotalRightclick: action({ name: "detail-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(_dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(_dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) open(virusTotal.url(field.data.toString())) } }), whoisRightclick: action({ name: "detail-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) dispatch(Modal.show("whois", {addr: field.data.toString()})) } }) diff --git a/src/js/electron/menu/actions/searchActions.ts b/src/js/electron/menu/actions/searchActions.ts index fee9fad626..153d852872 100644 --- a/src/js/electron/menu/actions/searchActions.ts +++ b/src/js/electron/menu/actions/searchActions.ts @@ -1,12 +1,6 @@ import lib from "src/js/lib" import {toZql} from "src/js/zql/toZql" -import { - SerializedZedField, - SerializedZedRecord, - ZedField, - ZedPrimitive, - ZedRecord -} from "zealot/zed" +import {ZealotContext, zed, zjson} from "zealot" import brim from "../../../brim" import {downloadPcap} from "../../../flows/downloadPcap" import scrollToLog from "../../../flows/scrollToLog" @@ -35,16 +29,16 @@ function buildSearchActions() { copy: action({ name: "search-cell-menu-copy", label: "Copy", - listener(_dispatch, data: SerializedZedField) { - const f = ZedField.deserialize(data) + listener(_dispatch, data: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(data) lib.doc.copyToClipboard(f.data.toString()) } }), countBy: action({ name: "search-cell-menu-count-by", label: "Count by field", - listener(dispatch, data: SerializedZedField) { - const f = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const f = ZealotContext.decodeField(data) dispatch(appendQueryCountBy(f)) dispatch(submitSearch()) } @@ -52,8 +46,8 @@ function buildSearchActions() { detail: action({ name: "search-cell-menu-detail", label: "Open details", - listener(dispatch, data: SerializedZedRecord) { - const record = ZedRecord.deserialize(data) + listener(dispatch, data: zjson.RootRecord) { + const record = ZealotContext.decodeRecord(data) dispatch(Layout.showRightSidebar()) dispatch(viewLogDetail(record)) } @@ -61,16 +55,16 @@ function buildSearchActions() { exclude: action({ name: "search-cell-menu-exclude", label: "Filter != value", - listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryExclude(ZedField.deserialize(data))) + listener(dispatch, data: zjson.FieldRootRecord) { + dispatch(appendQueryExclude(ZealotContext.decodeField(data))) dispatch(submitSearch()) } }), freshInclude: action({ name: "search-cell-menu-fresh-include", label: "New search with this value", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) dispatch(SearchBar.clearSearchBar()) dispatch(SearchBar.changeSearchBarInput(toZql(field.data))) dispatch(submitSearch()) @@ -79,9 +73,9 @@ function buildSearchActions() { fromTime: action({ name: "search-cell-menu-from-time", label: 'Use as "start" time', - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) - if (field.data instanceof ZedPrimitive && field.data.type === "time") { + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) + if (field.data instanceof zed.Time) { dispatch(tab.setFrom(brim.time(field.data.toDate()).toTs())) dispatch(submitSearch()) } @@ -90,8 +84,8 @@ function buildSearchActions() { groupByDrillDown: action({ name: "search-cell-menu-pivot-to-logs", label: "Pivot to logs", - listener(dispatch, program: string, data: SerializedZedRecord) { - const record = ZedRecord.deserialize(data) + listener(dispatch, program: string, data: zjson.RootRecord) { + const record = ZealotContext.decodeRecord(data) const newProgram = brim .program(program) .drillDown(record) @@ -107,16 +101,16 @@ function buildSearchActions() { include: action({ name: "search-cell-menu-include", label: "Filter = value", - listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryInclude(ZedField.deserialize(data))) + listener(dispatch, data: zjson.FieldRootRecord) { + dispatch(appendQueryInclude(ZealotContext.decodeField(data))) dispatch(submitSearch()) } }), in: action({ name: "search-cell-menu-in", label: "Filter in field", - listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryIn(ZedField.deserialize(data))) + listener(dispatch, data: zjson.FieldRootRecord) { + dispatch(appendQueryIn(ZealotContext.decodeField(data))) dispatch(submitSearch()) } }), @@ -125,12 +119,12 @@ function buildSearchActions() { label: "View in full context", listener( dispatch, - fieldData: SerializedZedField, - recordData: SerializedZedRecord + fieldData: zjson.FieldRootRecord, + recordData: zjson.RootRecord ) { - const field = ZedField.deserialize(fieldData) - const record = ZedRecord.deserialize(recordData) - if (field.data instanceof ZedPrimitive && field.data.type === "time") { + const field = ZealotContext.decodeField(fieldData) + const record = ZealotContext.decodeRecord(recordData) + if (field.data instanceof zed.Time) { const brimTime = brim.time(field.data.toDate()) dispatch(tab.setFrom(brimTime.subtract(1, "minutes").toTs())) dispatch(tab.setTo(brimTime.add(1, "minutes").toTs())) @@ -149,15 +143,15 @@ function buildSearchActions() { notIn: action({ name: "search-cell-menu-not-in", label: "Filter not in field", - listener(dispatch, data: SerializedZedField) { - dispatch(appendQueryNotIn(ZedField.deserialize(data))) + listener(dispatch, data: zjson.FieldRootRecord) { + dispatch(appendQueryNotIn(ZealotContext.decodeField(data))) dispatch(submitSearch()) } }), logResult: action({ name: "search-cell-menu-log-result", label: "Log result to console", - listener(_dispatch, field: SerializedZedField, log: SerializedZedRecord) { + listener(_dispatch, field: zjson.FieldRootRecord, log: zjson.RootRecord) { console.log(JSON.stringify(log)) console.log(JSON.stringify(field)) } @@ -165,15 +159,15 @@ function buildSearchActions() { pcaps: action({ name: "search-cell-menu-pcaps", label: "Download PCAPS", - listener(dispatch, data: SerializedZedRecord) { - dispatch(downloadPcap(ZedRecord.deserialize(data))) + listener(dispatch, data: zjson.RootRecord) { + dispatch(downloadPcap(ZealotContext.decodeRecord(data))) } }), sortAsc: action({ name: "search-cell-menu-sort-asc", label: "Sort A...Z", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) dispatch(appendQuerySortBy(field.name, "asc")) dispatch(submitSearch()) } @@ -181,8 +175,8 @@ function buildSearchActions() { sortDesc: action({ name: "search-cell-menu-sort-desc", label: "Sort Z...A", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) dispatch(appendQuerySortBy(field.name, "desc")) dispatch(submitSearch()) } @@ -190,13 +184,13 @@ function buildSearchActions() { toTime: action({ name: "search-cell-menu-to-time", label: 'Use as "end" time', - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) - if (field.data instanceof ZedPrimitive && field.data.type === "time") { + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) + if (field.data instanceof zed.Time) { dispatch( tab.setTo( brim - .time((field.data as ZedPrimitive).toDate()) + .time(field.data.toDate()) .add(1, "ms") .toTs() ) @@ -208,9 +202,9 @@ function buildSearchActions() { virusTotalRightclick: action({ name: "search-cell-menu-virus-total", label: "VirusTotal Lookup", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) - if (field.data instanceof ZedPrimitive && !field.data.isUnset()) { + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) + if (field.data instanceof zed.Primitive && !field.data.isUnset()) { open(virusTotal.url(field.data.toString() as string)) } } @@ -218,8 +212,8 @@ function buildSearchActions() { whoisRightclick: action({ name: "search-cell-menu-who-is", label: "Whois Lookup", - listener(dispatch, data: SerializedZedField) { - const field = ZedField.deserialize(data) + listener(dispatch, data: zjson.FieldRootRecord) { + const field = ZealotContext.decodeField(data) dispatch(Modal.show("whois", {addr: field.data.toString()})) } }) diff --git a/src/js/flows/downloadPcap.ts b/src/js/flows/downloadPcap.ts index 542b385a50..78abe3c2e0 100644 --- a/src/js/flows/downloadPcap.ts +++ b/src/js/flows/downloadPcap.ts @@ -1,9 +1,9 @@ -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import open from "../lib/open" import Packets from "../state/Packets" import {Thunk} from "../state/types" -export const downloadPcap = (currentLog: ZedRecord): Thunk => (dispatch) => { +export const downloadPcap = (currentLog: zed.Record): Thunk => (dispatch) => { dispatch(Packets.fetch(currentLog)).then((pcapFile) => open(pcapFile, {newWindow: true}) ) diff --git a/src/js/flows/rightclick/cellMenu.test.ts b/src/js/flows/rightclick/cellMenu.test.ts index 7629130d69..fc59d088fd 100644 --- a/src/js/flows/rightclick/cellMenu.test.ts +++ b/src/js/flows/rightclick/cellMenu.test.ts @@ -1,6 +1,6 @@ import {MenuItemConstructorOptions} from "electron" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" -import {createRecord} from "test/factories/record" +import {createRecord} from "test/factories/zed-factory" import fixtures from "../../test/fixtures" const conn = createRecord({ diff --git a/src/js/flows/scrollToLog.ts b/src/js/flows/scrollToLog.ts index 8fce624c88..539371cab9 100644 --- a/src/js/flows/scrollToLog.ts +++ b/src/js/flows/scrollToLog.ts @@ -1,9 +1,9 @@ import {isEqual} from "lodash" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {Thunk} from "../state/types" import Viewer from "../state/Viewer" -export default (log: ZedRecord): Thunk => (dispatch, getState) => { +export default (log: zed.Record): Thunk => (dispatch, getState) => { const state = getState() const logs = Viewer.getLogs(state) const index = logs.findIndex((log2) => isEqual(log2, log)) diff --git a/src/js/flows/search/handler.ts b/src/js/flows/search/handler.ts index 8b808c5999..c2ffe24da1 100644 --- a/src/js/flows/search/handler.ts +++ b/src/js/flows/search/handler.ts @@ -1,6 +1,5 @@ import {SearchResponse} from "./response" import whenIdle from "../../lib/whenIdle" -import {DecodedZJSON} from "zealot/zed/zjson" import {RecordsCallbackArgs} from "zealot/fetcher/records_callback" function abortError(e) { @@ -9,7 +8,7 @@ function abortError(e) { export function handle(request: any) { const response = new SearchResponse() - const channels = new Map() + const channels = new Map() const promise = new Promise((resolve, reject) => { function flushBuffer() { for (const [id, data] of channels) { @@ -38,8 +37,8 @@ export function handle(request: any) { response.emit("start", task_id) response.emit("status", "FETCHING") }) - .records(({channel, rows, schemas, context}: RecordsCallbackArgs) => { - channels.set(channel, {rows, schemas, context}) + .records((args: RecordsCallbackArgs) => { + channels.set(args.channel, args) flushBufferLazy() }) .end(({id, error}) => { diff --git a/src/js/flows/searchBar/actions.ts b/src/js/flows/searchBar/actions.ts index 4257417744..ea20b54ecb 100644 --- a/src/js/flows/searchBar/actions.ts +++ b/src/js/flows/searchBar/actions.ts @@ -1,4 +1,4 @@ -import {ZedField} from "zealot/zed" +import {zed} from "zealot" import brim from "../../brim" import {onlyWhitespace} from "../../lib/Str" import SearchBar from "../../state/SearchBar" @@ -8,7 +8,7 @@ import { } from "../../state/SearchBar/selectors" import {Thunk} from "../../state/types" -export function appendQueryInclude(field: ZedField): Thunk { +export function appendQueryInclude(field: zed.Field): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -21,7 +21,7 @@ export function appendQueryInclude(field: ZedField): Thunk { } } -export function appendQueryExclude(field: ZedField): Thunk { +export function appendQueryExclude(field: zed.Field): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -34,7 +34,7 @@ export function appendQueryExclude(field: ZedField): Thunk { } } -export function appendQueryCountBy(field: ZedField): Thunk { +export function appendQueryCountBy(field: zed.Field): Thunk { return function(dispatch, getState) { const {current, pinned} = getSearchBar(getState()) const query = [...pinned, current].join(" ") @@ -71,7 +71,7 @@ export function appendQuerySortBy( } } -export function appendQueryIn(field: ZedField): Thunk { +export function appendQueryIn(field: zed.Field): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( @@ -84,7 +84,7 @@ export function appendQueryIn(field: ZedField): Thunk { } } -export function appendQueryNotIn(field: ZedField): Thunk { +export function appendQueryNotIn(field: zed.Field): Thunk { return function(dispatch, getState) { dispatch( SearchBar.changeSearchBarInput( diff --git a/src/js/flows/viewLogDetail.ts b/src/js/flows/viewLogDetail.ts index 7cdd522e1e..3525850a07 100644 --- a/src/js/flows/viewLogDetail.ts +++ b/src/js/flows/viewLogDetail.ts @@ -1,13 +1,13 @@ import {isEqual} from "lodash" import {fetchCorrelation} from "ppl/detail/flows/fetch" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import ErrorFactory from "../models/ErrorFactory" import Current from "../state/Current" import LogDetails from "../state/LogDetails" import Notice from "../state/Notice" import {Thunk} from "../state/types" -export const viewLogDetail = (record: ZedRecord): Thunk => ( +export const viewLogDetail = (record: zed.Record): Thunk => ( dispatch, getState ) => { diff --git a/src/js/models/TableColumns.ts b/src/js/models/TableColumns.ts index ad99786265..361ecfc11e 100644 --- a/src/js/models/TableColumns.ts +++ b/src/js/models/TableColumns.ts @@ -1,4 +1,4 @@ -import {ZedField, ZedPrimitive, ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {createCell} from "../brim/cell" import columnOrder from "../lib/columnOrder" import {$Column} from "../state/Columns/models/column" @@ -31,28 +31,20 @@ export default class TableColumns { ) } - setWidths(logs: ZedRecord[]) { + setWidths(logs: zed.Record[]) { const MAX_WIDTH = 500 const resizeHandle = 5 const sortIcon = 11 this.cols.forEach((col) => { if (col.width) return - const colName = createCell( - new ZedField({ - name: "", - data: new ZedPrimitive({ - type: "string", - value: col.name - }) - }) - ) + const colName = createCell(new zed.Field("", new zed.String(col.name))) let max = colName.guessWidth() + resizeHandle + sortIcon logs.forEach((log) => { const data = log.try(col.name) if (data) { - const cell = createCell(new ZedField({name: col.name, data})) + const cell = createCell(new zed.Field(col.name, data)) const len = cell.guessWidth() if (len > max) max = len } diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index 62db26cdd7..228182f9e3 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -1,5 +1,5 @@ -import {createRecord} from "test/factories/record" -import {ZedPrimitive} from "zealot/zed" +import {createRecord} from "test/factories/zed-factory" +import {zed} from "zealot" import {connCorrelation} from "./programs" test("conn correlation", () => { @@ -11,10 +11,10 @@ test("conn correlation", () => { }) expect( connCorrelation( - record.get("uid") as ZedPrimitive, - record.get("community_id") as ZedPrimitive, - record.get("ts") as ZedPrimitive, - record.get("duration") as ZedPrimitive + record.get("uid") as zed.Primitive, + record.get("community_id") as zed.Primitive, + record.get("ts") as zed.Primitive, + record.get("duration") as zed.Primitive ) ).toBe( 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' diff --git a/src/js/searches/programs.ts b/src/js/searches/programs.ts index 14912f046e..25fc194e68 100644 --- a/src/js/searches/programs.ts +++ b/src/js/searches/programs.ts @@ -1,4 +1,4 @@ -import {ZedPrimitive} from "zealot/zed" +import {zed} from "zealot" import zql from "../zql" export function md5Correlation(md5: string) { @@ -17,11 +17,11 @@ export function filenameCorrelation(md5: string) { return `md5=${md5} | count() by filename, mime_type | sort -r | head 5` } -export function uidFilter(uid: string | ZedPrimitive) { +export function uidFilter(uid: string | zed.Primitive) { return zql`uid=${uid} or ${uid} in conn_uids or ${uid} in uids or referenced_file.uid=${uid}` } -export function cidFilter(cid: string | ZedPrimitive) { +export function cidFilter(cid: string | zed.Primitive) { return zql`community_id=${cid}` } @@ -42,19 +42,19 @@ export function correlationIds({uid, cid}: RelatedIds) { return [filters.join(" or "), correlationLimit()].join(" | ") } -export function uidCorrelation(uid: string | ZedPrimitive) { +export function uidCorrelation(uid: string | zed.Primitive) { return `${uidFilter(uid)} | ${correlationLimit()}` } -export function cidCorrelation(cid: string | ZedPrimitive) { +export function cidCorrelation(cid: string | zed.Primitive) { return `${cidFilter(cid)} | ${correlationLimit()}` } export function connCorrelation( - uid: ZedPrimitive, - cid: ZedPrimitive, - ts: ZedPrimitive, - duration: ZedPrimitive + uid: zed.Primitive, + cid: zed.Primitive, + ts: zed.Primitive, + duration: zed.Primitive ) { const tsDate = ts.toDate() const dur = duration.toFloat() + 90 // Add a 1.5 minute buffer for events that get logged late diff --git a/src/js/state/Chart/actions.ts b/src/js/state/Chart/actions.ts index 9c0fadd36a..f81adafbf2 100644 --- a/src/js/state/Chart/actions.ts +++ b/src/js/state/Chart/actions.ts @@ -1,4 +1,4 @@ -import {ZedPrimitive, ZedRecord} from "zealot/zed" +import {zed} from "zealot" import MergeHash from "../../models/MergeHash" import UniqArray from "../../models/UniqArray" import {SearchStatus} from "../../types/searches" @@ -10,7 +10,7 @@ export default { status, tabId }), - appendRecords: (tabId: string, records: ZedRecord[]): CHART_RECORDS => ({ + appendRecords: (tabId: string, records: zed.Record[]): CHART_RECORDS => ({ type: "CHART_RECORDS", data: histogramFormat(records), tabId @@ -18,12 +18,12 @@ export default { clear: (tabId?: string): CHART_CLEAR => ({type: "CHART_CLEAR", tabId}) } -function histogramFormat(records: ZedRecord[]): ChartData { +function histogramFormat(records: zed.Record[]): ChartData { const paths = new UniqArray() const table = new MergeHash() records.forEach((r) => { - const [ts, path, count] = r.fields.map((f) => f.data) as ZedPrimitive[] + const [ts, path, count] = r.fields.map((f) => f.data) as zed.Primitive[] try { const pathName = path.toString() diff --git a/src/js/state/Chart/test.ts b/src/js/state/Chart/test.ts index 980fe74c4f..ff761e01f8 100644 --- a/src/js/state/Chart/test.ts +++ b/src/js/state/Chart/test.ts @@ -1,4 +1,4 @@ -import {createRecord} from "test/factories/record" +import {createRecord} from "test/factories/zed-factory" import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import chart from "./" diff --git a/src/js/state/Columns/models/columnSet.ts b/src/js/state/Columns/models/columnSet.ts index a16e632295..d0324330b3 100644 --- a/src/js/state/Columns/models/columnSet.ts +++ b/src/js/state/Columns/models/columnSet.ts @@ -1,8 +1,10 @@ import {uniqBy} from "lodash" -import {TypeContext} from "zealot/zed/zjson" +import {zed} from "zealot" import {$Column, createColumn} from "./column" -export function createColumnSet(c: TypeContext) { +type Args = {[name: string]: zed.Schema} + +export function createColumnSet(c: Args) { return { getName() { const keys = Object.keys(c) @@ -17,8 +19,8 @@ export function createColumnSet(c: TypeContext) { }, getUniqColumns() { let allCols = [] - for (const typedef of Object.values(c)) { - let inner = typedef.flatten().innerType + for (const schema of Object.values(c)) { + let inner = schema.flatten().type if (inner.kind === "record") { allCols = [...allCols, ...inner.fields] } diff --git a/src/js/state/Columns/selectors.ts b/src/js/state/Columns/selectors.ts index f6d2badddc..02c38700f1 100644 --- a/src/js/state/Columns/selectors.ts +++ b/src/js/state/Columns/selectors.ts @@ -1,5 +1,5 @@ import {createSelector} from "reselect" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import TableColumns from "../../models/TableColumns" import activeTabSelect from "../Tab/activeTabSelect" import {State} from "../types" @@ -14,7 +14,7 @@ const getCurrentTableColumns = createSelector< State, ViewerColumns, ColumnsState, - ZedRecord[], + zed.Record[], TableColumns >( Viewer.getColumns, diff --git a/src/js/state/LogDetails/actions.ts b/src/js/state/LogDetails/actions.ts index 02a0718b81..6cc35e5f09 100644 --- a/src/js/state/LogDetails/actions.ts +++ b/src/js/state/LogDetails/actions.ts @@ -1,5 +1,5 @@ import {SearchStatus} from "src/js/types/searches" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import { LOG_DETAIL_BACK, LOG_DETAIL_CLEAR, @@ -9,7 +9,7 @@ import { } from "./types" export default { - push: (record: ZedRecord): LOG_DETAIL_PUSH => ({ + push: (record: zed.Record): LOG_DETAIL_PUSH => ({ type: "LOG_DETAIL_PUSH", record: record.serialize() }), @@ -22,7 +22,7 @@ export default { type: "LOG_DETAIL_FORWARD" }), - updateUidLogs: (records: ZedRecord[]): LOG_DETAIL_UPDATE => { + updateUidLogs: (records: zed.Record[]): LOG_DETAIL_UPDATE => { return { type: "LOG_DETAIL_UPDATE", updates: { diff --git a/src/js/state/LogDetails/selectors.ts b/src/js/state/LogDetails/selectors.ts index 9699364480..619ed0cb29 100644 --- a/src/js/state/LogDetails/selectors.ts +++ b/src/js/state/LogDetails/selectors.ts @@ -6,7 +6,7 @@ import {State} from "../types" import {LogDetailHistory, toHistory} from "./reducer" import {LogDetailsState} from "./types" -import {ZedRecord} from "zealot/zed" +import {ZealotContext, zed} from "zealot" const getLogDetails = activeTabSelect((state: TabState) => { return state.logDetails @@ -17,23 +17,25 @@ const getHistory = createSelector( (logDetails) => toHistory(logDetails) ) -const build = createSelector( +const build = createSelector( getHistory, (history) => { const entry = history.current() if (entry && entry.log) { - return ZedRecord.deserialize(entry.log) + return ZealotContext.decodeRecord(entry.log) } else { return null } } ) -const getUidLogs = createSelector( +const getUidLogs = createSelector( getHistory, (history) => { const entry = history.current() - return entry ? entry.uidLogs.map((data) => ZedRecord.deserialize(data)) : [] + return entry + ? entry.uidLogs.map((data) => ZealotContext.decodeRecord(data)) + : [] } ) @@ -45,7 +47,7 @@ const getUidStatus = createSelector( } ) -const getConnLog = createSelector( +const getConnLog = createSelector( getUidLogs, (uids) => { return uids.find((log) => log.try("_path")?.toString() === "conn") diff --git a/src/js/state/LogDetails/test.ts b/src/js/state/LogDetails/test.ts index 6d3ed80134..ea5fc5f922 100644 --- a/src/js/state/LogDetails/test.ts +++ b/src/js/state/LogDetails/test.ts @@ -1,4 +1,4 @@ -import {createRecord} from "test/factories/record" +import {createRecord} from "test/factories/zed-factory" import initTestStore from "../../test/initTestStore" import LogDetails from "./" diff --git a/src/js/state/LogDetails/types.ts b/src/js/state/LogDetails/types.ts index 6b9471a8f3..8e7831517d 100644 --- a/src/js/state/LogDetails/types.ts +++ b/src/js/state/LogDetails/types.ts @@ -1,4 +1,4 @@ -import {SerializedZedRecord} from "zealot/zed" +import {zjson} from "zealot" import {SearchStatus} from "../../types/searches" export type LogDetailsState = { @@ -7,8 +7,8 @@ export type LogDetailsState = { } export type LogDetails = { - log: SerializedZedRecord - uidLogs: SerializedZedRecord[] + log: zjson.RootRecord + uidLogs: zjson.RootRecord[] uidStatus: SearchStatus } @@ -21,7 +21,7 @@ export type LogDetailsAction = export type LOG_DETAIL_PUSH = { type: "LOG_DETAIL_PUSH" - record: SerializedZedRecord + record: zjson.RootRecord } export type LOG_DETAIL_UPDATE = { diff --git a/src/js/state/Packets/flows.ts b/src/js/state/Packets/flows.ts index 823a9a6f16..e82cf9d3c5 100644 --- a/src/js/state/Packets/flows.ts +++ b/src/js/state/Packets/flows.ts @@ -1,6 +1,6 @@ import {remote} from "electron" import {join} from "path" -import {ZedPrimitive, ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {getZealot} from "../../flows/getZealot" import {saveToFile} from "../../lib/response" import Current from "../Current" @@ -9,7 +9,7 @@ import {Thunk} from "../types" import View from "../View" export default { - fetch: (log: ZedRecord): Thunk> => ( + fetch: (log: zed.Record): Thunk> => ( dispatch: Function, getState: Function ) => { @@ -18,8 +18,8 @@ export default { const state = getState() const zealot = dispatch(getZealot()) const spaceId = Current.getSpaceId(state) - const ts = log.get("ts") as ZedPrimitive - const dur = log.get("duration") as ZedPrimitive + const ts = log.get("ts") as zed.Primitive + const dur = log.get("duration") as zed.Primitive const args = { ts_sec: getSec(ts), ts_ns: getNs(ts), @@ -51,13 +51,13 @@ export default { } } -function getSec(data: ZedPrimitive): number { +function getSec(data: zed.Primitive): number { if (data.isUnset()) return 0 return parseInt(data.toString().split(".")[0]) } -function getNs(data: ZedPrimitive): number { +function getNs(data: zed.Primitive): number { if (data.isUnset()) return 0 const v = data.toString().split(".") diff --git a/src/js/state/SearchBar/test.ts b/src/js/state/SearchBar/test.ts index b82459181c..aa3687aa70 100644 --- a/src/js/state/SearchBar/test.ts +++ b/src/js/state/SearchBar/test.ts @@ -1,6 +1,7 @@ import tabHistory from "app/router/tab-history" import {lakePath} from "app/router/utils/paths" import brim from "src/js/brim" +import {createField} from "test/factories/zed-factory" import {createZealotMock} from "zealot" import { appendQueryCountBy, @@ -17,7 +18,6 @@ import Url from "../Url" import Workspaces from "../Workspaces" import SearchBar from "./" import {SearchBarState} from "./types" -import {ZedPrimitive, ZedField} from "zealot/zed" let store, mock beforeEach(() => { @@ -128,16 +128,14 @@ test("search bar pin remove when out of bounds", () => { }) test("append an include field", () => { - const data = new ZedPrimitive({type: "string", value: "conn"}) - const field = new ZedField({name: "_path", data}) + const field = createField("_path", "conn") const state = store.dispatchAll([appendQueryInclude(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe('_path="conn"') }) test("append an include field when some text already exists", () => { - const data = new ZedPrimitive({type: "string", value: "conn"}) - const field = new ZedField({name: "_path", data}) + const field = createField("_path", "conn") const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), appendQueryInclude(field) @@ -146,15 +144,13 @@ test("append an include field when some text already exists", () => { }) test("append an exclude field", () => { - const data = new ZedPrimitive({type: "string", value: "conn"}) - const field = new ZedField({name: "_path", data}) + const field = createField("_path", "conn") const state = store.dispatchAll([appendQueryExclude(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe('_path!="conn"') }) test("append an exclude field when some text already exists", () => { - const data = new ZedPrimitive({type: "string", value: "conn"}) - const field = new ZedField({name: "_path", data}) + const field = createField("_path", "conn") const state = store.dispatchAll([ SearchBar.changeSearchBarInput("text"), appendQueryExclude(field) @@ -163,15 +159,13 @@ test("append an exclude field when some text already exists", () => { }) test("append a count by field", () => { - const data = new ZedPrimitive({type: "string", value: "conn"}) - const field = new ZedField({name: "_path", data}) + const field = createField("_path", "conn") const state = store.dispatchAll([appendQueryCountBy(field)]) expect(SearchBar.getSearchBarInputValue(state)).toBe("* | count() by _path") }) test("append a count to an existing query", () => { - const data = new ZedPrimitive({type: "string", value: "ho ho"}) - const field = new ZedField({name: "query", data}) + const field = createField("query", "hey") const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), appendQueryCountBy(field) @@ -180,8 +174,7 @@ test("append a count to an existing query", () => { }) test("append a count to an existing query with a pin", () => { - const data = new ZedPrimitive({type: "string", value: "heyo"}) - const field = new ZedField({name: "query", data}) + const field = createField("query", "hey") const state = store.dispatchAll([ SearchBar.changeSearchBarInput("dns"), SearchBar.pinSearchBar(), diff --git a/src/js/state/Viewer/actions.ts b/src/js/state/Viewer/actions.ts index 823e3e7e8e..87bf940e74 100644 --- a/src/js/state/Viewer/actions.ts +++ b/src/js/state/Viewer/actions.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" import { @@ -47,14 +47,14 @@ export const setEndStatus = ( export const appendRecords = ( tabId: string | null | undefined, - records: ZedRecord[] + records: zed.Record[] ): VIEWER_RECORDS => { return {type: "VIEWER_RECORDS", records, tabId} } export const setRecords = ( tabId: string | undefined, - records: ZedRecord[] + records: zed.Record[] ): VIEWER_SET_RECORDS => { return {type: "VIEWER_SET_RECORDS", records, tabId} } diff --git a/src/js/state/Viewer/selectors.ts b/src/js/state/Viewer/selectors.ts index d7ae630365..e68c264438 100644 --- a/src/js/state/Viewer/selectors.ts +++ b/src/js/state/Viewer/selectors.ts @@ -1,7 +1,7 @@ import {createSelector} from "reselect" import {ScrollPosition} from "src/js/types" import {SearchStatus} from "src/js/types/searches" -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {TabState} from "../Tab/types" import Tabs from "../Tabs" import {State} from "../types" @@ -13,10 +13,11 @@ export const getViewer = createSelector( (tab) => tab.viewer ) -export const getViewerRecords = createSelector( - getViewer, - (viewer) => viewer.records -) +export const getViewerRecords = createSelector< + State, + ViewerState, + zed.Record[] +>(getViewer, (viewer) => viewer.records) export const isFetching = (state: TabState) => state.viewer.status === "FETCHING" @@ -59,8 +60,8 @@ export const getSelection = createSelector< export const getSelectedRecords = createSelector< State, ViewerSelection, - ZedRecord[], - ZedRecord[] + zed.Record[], + zed.Record[] >(getSelection, getRecords, (selection, records) => selection.getIndices().map((index) => records[index]) ) diff --git a/src/js/state/Viewer/test.ts b/src/js/state/Viewer/test.ts index acd347ea35..11232424f7 100644 --- a/src/js/state/Viewer/test.ts +++ b/src/js/state/Viewer/test.ts @@ -1,4 +1,4 @@ -import {createRecord} from "test/factories/record" +import {createRecord} from "test/factories/zed-factory" import ZedTypeDef from "zealot/zed/type-def" import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" diff --git a/src/js/state/Viewer/types.ts b/src/js/state/Viewer/types.ts index 16465d2a8b..7f33f47ee8 100644 --- a/src/js/state/Viewer/types.ts +++ b/src/js/state/Viewer/types.ts @@ -1,4 +1,4 @@ -import {ZedRecord} from "zealot/zed" +import {zed} from "zealot" import {TypeContext} from "zealot/zed/zjson" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" @@ -13,7 +13,7 @@ export type ViewerSelectionData = { currentRange: [number, number] } export type ViewerState = { - records: ZedRecord[] + records: zed.Record[] columns: ViewerColumns endStatus: ViewerStatus status: SearchStatus @@ -42,13 +42,13 @@ export type ViewerAction = export type VIEWER_RECORDS = { type: "VIEWER_RECORDS" - records: ZedRecord[] + records: zed.Record[] tabId: string | null | undefined } export type VIEWER_SET_RECORDS = { type: "VIEWER_SET_RECORDS" - records: ZedRecord[] + records: zed.Record[] tabId?: string } diff --git a/src/js/types/index.ts b/src/js/types/index.ts index de1d3ce77d..c5fe114023 100644 --- a/src/js/types/index.ts +++ b/src/js/types/index.ts @@ -2,7 +2,7 @@ import {SpanArgs} from "../state/Search/types" import {TimeUnit} from "../lib" import AppError from "../models/AppError" import {MenuItemConstructorOptions} from "electron" -import {ZedRecord, ZedField} from "zealot/zed" +import {zed} from "zealot" export type Notification = | AppError @@ -60,12 +60,12 @@ export type LogCorrelations = { } export type RelatedLogs = { - [key: string]: ZedRecord[] + [key: string]: zed.Record[] } export type RightClickBuilder = ( - field: ZedField, - record: ZedRecord, + field: zed.Field, + record: zed.Record, compound: boolean ) => MenuItemConstructorOptions[] diff --git a/src/js/zql/toZql.ts b/src/js/zql/toZql.ts index 3390674311..4840dd8f4a 100644 --- a/src/js/zql/toZql.ts +++ b/src/js/zql/toZql.ts @@ -1,8 +1,8 @@ import isString from "lodash/isString" -import {ZedPrimitive} from "zealot/zed" +import {zed} from "zealot" export function toZql(object: unknown): string { - if (object instanceof ZedPrimitive) return toZqlZngPrimitive(object) + if (object instanceof zed.Primitive) return toZqlZngPrimitive(object) if (isString(object)) return toZqlString(object) if (object instanceof Date) return toZqlDate(object) if (typeof object === "boolean") return toZqlBool(object) @@ -29,8 +29,8 @@ function toZqlBool(bool: boolean) { return bool ? "true" : "false" } -function toZqlZngPrimitive(data: ZedPrimitive) { - if (data.type === "string" || data.type === "bstring") +function toZqlZngPrimitive(data: zed.Primitive) { + if (data instanceof zed.String || data instanceof zed.BString) return toZqlString(data.toString()) throw new Error(`Can't convert Zng Type: ${data.type} to zql`) } diff --git a/test/factories/zed-factory.ts b/test/factories/zed-factory.ts index 50e70c26ff..e138d60f3a 100644 --- a/test/factories/zed-factory.ts +++ b/test/factories/zed-factory.ts @@ -1,5 +1,5 @@ import {isDate, isInteger, isNumber, isObject, isString} from "lodash" -import * as zed from "zealot/zed" +import {zed} from "zealot" // Convert a js object into a zed record export function createRecord(object): zed.Record { diff --git a/zealot/zed/schema.ts b/zealot/zed/schema.ts index e9098ded45..106d65e30b 100644 --- a/zealot/zed/schema.ts +++ b/zealot/zed/schema.ts @@ -1,5 +1,25 @@ -import {TypeRecord} from "./types/type-record" +import {trueType} from "." +import {TypeField, TypeRecord} from "./types/type-record" export class Schema { constructor(public name: string, public type: TypeRecord) {} + + flatten(): Schema { + const inner = trueType(this.type) + + const flat = (fields: TypeField[], prefix = ""): TypeField[] => { + return fields.flatMap((field) => { + const name = prefix + field.name + const type = trueType(field.type) + if (type instanceof TypeRecord) { + return flat(type.fields || [], name + ".") + } else { + return {name, type} + } + }) + } + const fields = flat(inner.fields || []) + + return new Schema(this.name, new TypeRecord(fields)) + } } diff --git a/zealot/zed/utils.ts b/zealot/zed/utils.ts index 31d736e846..f6a7dde2e9 100644 --- a/zealot/zed/utils.ts +++ b/zealot/zed/utils.ts @@ -45,3 +45,11 @@ export function isTime(value: unknown): value is Time { export function isTypeAlias(type: ZedType): type is TypeAlias { return type instanceof TypeAlias } + +export function trueType(start: ZedType): T { + let t = start + while (isTypeAlias(t)) { + t = t.type + } + return t as T +} diff --git a/zealot/zed/values/record.ts b/zealot/zed/values/record.ts index 1a35aca8b9..4a4a9f73ca 100644 --- a/zealot/zed/values/record.ts +++ b/zealot/zed/values/record.ts @@ -1,10 +1,9 @@ import {isNull} from "lodash" -import {isTypeAlias} from ".." import {TypeAlias} from "../types/type-alias" import {TypeField, TypeRecord} from "../types/type-record" import {Field} from "./field" import {ZedValue, ZedValueInterface} from "./types" - +import {trueType} from "../utils" export class Record implements ZedValueInterface { constructor( public type: TypeRecord | TypeAlias, @@ -21,11 +20,7 @@ export class Record implements ZedValueInterface { } get trueType() { - let t = this.type - while (isTypeAlias(t)) { - t = t.type as TypeRecord | TypeAlias - } - return t as TypeRecord + return trueType(this.type) } toString() { diff --git a/zealot/zed/values/types.ts b/zealot/zed/values/types.ts index b509b1a28f..7d956c061e 100644 --- a/zealot/zed/values/types.ts +++ b/zealot/zed/values/types.ts @@ -69,8 +69,3 @@ export interface ZedValueInterface { isUnset(): boolean type: ZedType } - -export interface ZedField { - name: string - value: ZedValue -} From 26a9d42a78be3cd842eb43290aa3477b1b759c62 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 28 Apr 2021 14:19:11 -0700 Subject: [PATCH 12/41] Zero errors --- .../flows/next-page-viewer-search.test.ts | 2 +- app/search/flows/next-page-viewer-search.ts | 2 +- app/search/flows/viewer-search.ts | 4 +- .../flows/get-correlation-query.test.ts | 10 +- ppl/detail/flows/get-correlation-query.ts | 8 +- ppl/detail/models/SuricataEvent.ts | 5 +- ppl/detail/models/UnknownEvent.ts | 2 +- src/js/brim/cell.ts | 5 +- src/js/components/ConnVersation.tsx | 6 +- src/js/components/LogCell/CompoundField.tsx | 112 +++++++++--------- src/js/components/RightPane.test.tsx | 2 +- src/js/flows/search/response.ts | 4 +- src/js/searches/programs.test.ts | 8 +- src/js/searches/programs.ts | 8 +- src/js/state/Chart/actions.ts | 6 +- src/js/state/Columns/models/column.ts | 15 +-- src/js/state/Columns/selectors.ts | 4 +- src/js/state/Columns/touch.test.ts | 44 +++---- src/js/state/Columns/touch.ts | 4 +- src/js/state/Viewer/actions.ts | 6 +- src/js/state/Viewer/selectors.ts | 4 +- src/js/state/Viewer/test.ts | 35 +----- src/js/state/Viewer/types.ts | 9 +- zealot/fetcher/records_callback.ts | 13 +- 24 files changed, 140 insertions(+), 178 deletions(-) diff --git a/app/search/flows/next-page-viewer-search.test.ts b/app/search/flows/next-page-viewer-search.test.ts index 72c4d9aac7..db74eb24ba 100644 --- a/app/search/flows/next-page-viewer-search.test.ts +++ b/app/search/flows/next-page-viewer-search.test.ts @@ -51,7 +51,7 @@ test("#fetchNextPage adds 1ms to ts of last change", () => { const search = jest.spyOn(zealot.zealot, "search") store.dispatch(nextPageViewerSearch()) - const data = records[1].at(1) as zed.Primitive + const data = records[1].at(1) as zed.Time const lastChangeTs = data.toDate().getTime() expect(search).toHaveBeenCalledWith( expect.any(String), diff --git a/app/search/flows/next-page-viewer-search.ts b/app/search/flows/next-page-viewer-search.ts index d5ce6f2703..d2ea317e66 100644 --- a/app/search/flows/next-page-viewer-search.ts +++ b/app/search/flows/next-page-viewer-search.ts @@ -39,7 +39,7 @@ function nextPageArgs( if (!isEmpty(logs)) { const index = indexOfLastChange(logs, (log) => log.try("ts")?.toString()) if (index >= 0) { - const ts = logs[index].get("ts") as zed.Primitive + const ts = logs[index].get("ts") const prevTs = ts.toDate() nextSpan[1] = brim .time(prevTs) diff --git a/app/search/flows/viewer-search.ts b/app/search/flows/viewer-search.ts index ab4838da60..d6535f6ec1 100644 --- a/app/search/flows/viewer-search.ts +++ b/app/search/flows/viewer-search.ts @@ -9,8 +9,8 @@ import SearchBar from "src/js/state/SearchBar" import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import Viewer from "src/js/state/Viewer" +import {SchemaMap} from "src/js/state/Viewer/types" import {zed} from "zealot" -import {TypeContext} from "zealot/zjson" type Args = { query: string @@ -40,7 +40,7 @@ function handle( append = false ): Thunk { return function(dispatch) { - let allColumns: TypeContext = {} + let allColumns: SchemaMap = {} let allRecords: zed.Record[] = [] let count = 0 diff --git a/ppl/detail/flows/get-correlation-query.test.ts b/ppl/detail/flows/get-correlation-query.test.ts index 595c456545..d7ce577c15 100644 --- a/ppl/detail/flows/get-correlation-query.test.ts +++ b/ppl/detail/flows/get-correlation-query.test.ts @@ -29,10 +29,10 @@ test("returns conn query if ts and duration are present", () => { }) expect(getCorrelationQuery(record)).toBe( connCorrelation( - record.get("uid") as zed.Primitive, - record.get("community_id") as zed.Primitive, - record.get("ts") as zed.Primitive, - record.get("duration") as zed.Primitive + record.get("uid"), + record.get("community_id"), + record.get("ts"), + record.get("duration") ) ) }) @@ -46,7 +46,7 @@ test("returns cid query if only cid present", () => { }) expect(getCorrelationQuery(record)).toBe( - cidCorrelation(record.get("community_id") as zed.Primitive) + cidCorrelation(record.get("community_id")) ) }) diff --git a/ppl/detail/flows/get-correlation-query.ts b/ppl/detail/flows/get-correlation-query.ts index e71bcd0d55..fac14bcc32 100644 --- a/ppl/detail/flows/get-correlation-query.ts +++ b/ppl/detail/flows/get-correlation-query.ts @@ -11,10 +11,10 @@ export function getCorrelationQuery(record: zed.Record) { if (cid && uid && record.has("ts") && record.has("duration")) { return connCorrelation( - record.get("uid") as zed.Primitive, - record.get("community_id") as zed.Primitive, - record.get("ts") as zed.Primitive, - record.get("duration") as zed.Primitive + record.get("uid") as zed.String, + record.get("community_id") as zed.String, + record.get("ts") as zed.Time, + record.get("duration") as zed.Float64 ) } else if (uid) { return uidCorrelation(uid) diff --git a/ppl/detail/models/SuricataEvent.ts b/ppl/detail/models/SuricataEvent.ts index 8426d6c01d..85efcdc684 100644 --- a/ppl/detail/models/SuricataEvent.ts +++ b/ppl/detail/models/SuricataEvent.ts @@ -9,7 +9,7 @@ export class SuricataEvent implements BrimEventInterface { } getTime() { - return (this.r.get("ts") as zed.Primitive).toDate() + return this.r.get("ts").toDate() } getEndTime() { @@ -21,7 +21,6 @@ export class SuricataEvent implements BrimEventInterface { } getSeverity(): number { - const data = this.r.get("alert.severity") as zed.Primitive - return data.toInt() + return this.r.get("alert.severity").toInt() } } diff --git a/ppl/detail/models/UnknownEvent.ts b/ppl/detail/models/UnknownEvent.ts index b6b2623da9..4ced19e2c5 100644 --- a/ppl/detail/models/UnknownEvent.ts +++ b/ppl/detail/models/UnknownEvent.ts @@ -10,7 +10,7 @@ export class UnknownEvent implements BrimEventInterface { getTime() { if (this.r.has("ts")) { - return (this.r.get("ts") as zed.Primitive).toDate() + return this.r.get("ts").toDate() } else { return new Date(0) } diff --git a/src/js/brim/cell.ts b/src/js/brim/cell.ts index 3565e494cd..6f4163ba7b 100644 --- a/src/js/brim/cell.ts +++ b/src/js/brim/cell.ts @@ -1,5 +1,4 @@ import {zed} from "zealot" -import {createComplexCell} from "./complexCell" import {createPrimitiveCell} from "./primitiveCell" export interface Cell { @@ -8,13 +7,13 @@ export interface Cell { type Args = { name: string - data: ZedData + data: zed.AnyValue } export function createCell({name, data}: Args): Cell { if (data instanceof zed.Primitive) { return createPrimitiveCell({name, data}) } else { - return createComplexCell({name, data}) + // return createComplexCell({name, data}) } } diff --git a/src/js/components/ConnVersation.tsx b/src/js/components/ConnVersation.tsx index e764a0450d..673ca0c8b6 100644 --- a/src/js/components/ConnVersation.tsx +++ b/src/js/components/ConnVersation.tsx @@ -22,7 +22,11 @@ function filter(record: zed.Record, names: string[]) { if (field) fields.push(field) }) - return new zed.Record({fields}) + const type = new zed.TypeRecord( + fields.map((f) => ({name: f.name, type: f.value.type})) + ) + + return new zed.Record(type, fields) } const ConnVersation = ({record}: Props) => { diff --git a/src/js/components/LogCell/CompoundField.tsx b/src/js/components/LogCell/CompoundField.tsx index 3f12df285b..83ce626ebb 100644 --- a/src/js/components/LogCell/CompoundField.tsx +++ b/src/js/components/LogCell/CompoundField.tsx @@ -1,8 +1,5 @@ -import classNames from "classnames" import React from "react" import {zed} from "zealot" -import {createComplexCell} from "../../brim/complexCell" -import SingleField from "./SingleField" type Props = { field: zed.Field @@ -10,65 +7,66 @@ type Props = { menuBuilder: Function } -export default function CompoundField({field, log, menuBuilder}: Props) { - // @ts-ignore - const compound = createComplexCell(field) - const render = [] +export default function CompoundField(_: Props) { + return

Implement compound fields

+ // // @ts-ignore + // const compound = createComplexCell(field) + // const render = [] - for (let i = 0; i < compound.length; ++i) { - const item = new zed.Field({ - name: field.name, - // @ts-ignore - data: field.data.items && field.data.items[i] - }) - if (item) { - const menu = menuBuilder(item, log, true) - render.push() - } - if (i !== compound.length - 1) { - render.push() - } - } + // for (let i = 0; i < compound.length; ++i) { + // const item = new zed.Field({ + // name: field.name, + // // @ts-ignore + // data: field.data.items && field.data.items[i] + // }) + // if (item) { + // const menu = menuBuilder(item, log, true) + // render.push() + // } + // if (i !== compound.length - 1) { + // render.push() + // } + // } - return {render} + // return {render} } -function Comma() { - return -} +// function Comma() { +// return +// } -function Extra({value, className}: {value: string | null; className?: string}) { - return ( -
{value}
- ) -} +// function Extra({value, className}: {value: string | null; className?: string}) { +// return ( +//
{value}
+// ) +// } -type WrapperProps = { - type: string | null - children: any -} +// type WrapperProps = { +// type: string | null +// children: any +// } -function Wrapper({type, children}: WrapperProps) { - const [open, close] = getWrapper(type) - return ( - <> - - {children} - - - ) -} +// function Wrapper({type, children}: WrapperProps) { +// const [open, close] = getWrapper(type) +// return ( +// <> +// +// {children} +// +// +// ) +// } -function getWrapper(container) { - switch (container) { - case "set": - return ["{", "}"] - case "vector": - // DELETE after vector-array is merged - return ["[", "]"] - case "array": - return ["[", "]"] - default: - return [null, null] - } -} +// function getWrapper(container) { +// switch (container) { +// case "set": +// return ["{", "}"] +// case "vector": +// // DELETE after vector-array is merged +// return ["[", "]"] +// case "array": +// return ["[", "]"] +// default: +// return [null, null] +// } +// } diff --git a/src/js/components/RightPane.test.tsx b/src/js/components/RightPane.test.tsx index 96573ab8ba..2db84c8e92 100644 --- a/src/js/components/RightPane.test.tsx +++ b/src/js/components/RightPane.test.tsx @@ -13,7 +13,7 @@ test("no errors if space does not exist", async () => { store.dispatch(Layout.showRightSidebar()) store.dispatch(tabHistory.push(workspacesPath())) - store.dispatch(LogDetails.push(new zed.Record({fields: []}))) + store.dispatch(LogDetails.push(new zed.Record(null, []))) const el = provide(store, ) expect(el.html()).toBe("") }) diff --git a/src/js/flows/search/response.ts b/src/js/flows/search/response.ts index b36ae0a95c..5313f96faf 100644 --- a/src/js/flows/search/response.ts +++ b/src/js/flows/search/response.ts @@ -1,4 +1,4 @@ -import {DecodedZJSON} from "zealot/zed/zjson" +import {RecordsCallbackArgs} from "zealot/fetcher/records_callback" import {SearchStats, SearchStatus} from "../../types/searches" type EventNames = @@ -18,7 +18,7 @@ export class SearchResponse { this.callbacks = new Map() } - chan(num: number, func: (data: DecodedZJSON) => void) { + chan(num: number, func: (data: RecordsCallbackArgs) => void) { this.callbacks.set(num, func) return this } diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index 228182f9e3..84e745a288 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -11,10 +11,10 @@ test("conn correlation", () => { }) expect( connCorrelation( - record.get("uid") as zed.Primitive, - record.get("community_id") as zed.Primitive, - record.get("ts") as zed.Primitive, - record.get("duration") as zed.Primitive + record.get("uid") as zed.String, + record.get("community_id") as zed.String, + record.get("ts") as zed.Time, + record.get("duration") as zed.Float64 ) ).toBe( 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' diff --git a/src/js/searches/programs.ts b/src/js/searches/programs.ts index 25fc194e68..38a4f8cec6 100644 --- a/src/js/searches/programs.ts +++ b/src/js/searches/programs.ts @@ -51,10 +51,10 @@ export function cidCorrelation(cid: string | zed.Primitive) { } export function connCorrelation( - uid: zed.Primitive, - cid: zed.Primitive, - ts: zed.Primitive, - duration: zed.Primitive + uid: zed.String, + cid: zed.String, + ts: zed.Time, + duration: zed.Float64 ) { const tsDate = ts.toDate() const dur = duration.toFloat() + 90 // Add a 1.5 minute buffer for events that get logged late diff --git a/src/js/state/Chart/actions.ts b/src/js/state/Chart/actions.ts index f81adafbf2..546f6d0efe 100644 --- a/src/js/state/Chart/actions.ts +++ b/src/js/state/Chart/actions.ts @@ -23,7 +23,11 @@ function histogramFormat(records: zed.Record[]): ChartData { const table = new MergeHash() records.forEach((r) => { - const [ts, path, count] = r.fields.map((f) => f.data) as zed.Primitive[] + const [ts, path, count] = r.fields.map((f) => f.data) as [ + zed.Time, + zed.String, + zed.Uint64 + ] try { const pathName = path.toString() diff --git a/src/js/state/Columns/models/column.ts b/src/js/state/Columns/models/column.ts index c08a633114..7a3f15af33 100644 --- a/src/js/state/Columns/models/column.ts +++ b/src/js/state/Columns/models/column.ts @@ -1,19 +1,10 @@ -import {RecordFieldType, Type} from "zealot/zed/zjson" +import {zed} from "zealot" +import {RecordFieldType} from "zealot/zjson" export type $Column = {name: string; type: string; key: string} -function getType(type: Type) { - if (type.kind === "primitive") { - return type.name - } else if (type.kind === "typename") { - return type.name - } else if (type.kind === "typedef") { - return type.name - } -} - export function createColumn(c: RecordFieldType) { - const type = getType(c.type) + const type = zed.typeId(c.type) return { name: c.name, type, diff --git a/src/js/state/Columns/selectors.ts b/src/js/state/Columns/selectors.ts index 02c38700f1..cc39c633af 100644 --- a/src/js/state/Columns/selectors.ts +++ b/src/js/state/Columns/selectors.ts @@ -4,7 +4,7 @@ import TableColumns from "../../models/TableColumns" import activeTabSelect from "../Tab/activeTabSelect" import {State} from "../types" import Viewer from "../Viewer" -import {ViewerColumns} from "../Viewer/types" +import {SchemaMap} from "../Viewer/types" import {createColumnSet} from "./models/columnSet" import {ColumnsState} from "./types" @@ -12,7 +12,7 @@ const getColumns = activeTabSelect((tab) => tab.columns) const getCurrentTableColumns = createSelector< State, - ViewerColumns, + SchemaMap, ColumnsState, zed.Record[], TableColumns diff --git a/src/js/state/Columns/touch.test.ts b/src/js/state/Columns/touch.test.ts index 34f33f977b..4e5e023407 100644 --- a/src/js/state/Columns/touch.test.ts +++ b/src/js/state/Columns/touch.test.ts @@ -1,5 +1,5 @@ -import {INTERVAL, STRING, TIME} from "test/fixtures/zjson-types" -import ZedTypeDef from "zealot/zed/type-def" +import {INTERVAL, STRING} from "test/fixtures/zjson-types" +import {zed} from "zealot" import initTestStore from "../../test/initTestStore" import Columns from "./" import actions from "./actions" @@ -7,32 +7,20 @@ import {createColumn} from "./models/column" import touch from "./touch" const columns = { - "1": new ZedTypeDef({ - type: { - name: "1", - kind: "typedef", - type: { - kind: "record", - fields: [ - {name: "_path", type: STRING}, - {name: "duration", type: INTERVAL} - ] - } - } - }), - "2": new ZedTypeDef({ - type: { - name: "2", - kind: "typedef", - type: { - kind: "record", - fields: [ - {name: "_path", type: STRING}, - {name: "ts", type: TIME} - ] - } - } - }) + "1": new zed.Schema( + "1", + new zed.TypeRecord([ + {name: "_path", type: zed.TypeString}, + {name: "duration", type: zed.TypeDuration} + ]) + ), + "2": new zed.Schema( + "2", + new zed.TypeRecord([ + {name: "_path", type: zed.TypeString}, + {name: "ts", type: zed.TypeTime} + ]) + ) } let store diff --git a/src/js/state/Columns/touch.ts b/src/js/state/Columns/touch.ts index 4f8ec83e11..11d38a4192 100644 --- a/src/js/state/Columns/touch.ts +++ b/src/js/state/Columns/touch.ts @@ -1,11 +1,11 @@ import {Thunk} from "../types" -import {ViewerColumns} from "../Viewer/types" +import {SchemaMap} from "../Viewer/types" import {createColumnPrefs} from "./models/columnPrefs" import {createColumnSet} from "./models/columnSet" import Columns from "./" import selectors from "./selectors" -export default (columns: ViewerColumns): Thunk => (dispatch, getState) => { +export default (columns: SchemaMap): Thunk => (dispatch, getState) => { const set = createColumnSet(columns) const name = set.getName() const cols = set.getUniqColumns() diff --git a/src/js/state/Viewer/actions.ts b/src/js/state/Viewer/actions.ts index 87bf940e74..da04296f68 100644 --- a/src/js/state/Viewer/actions.ts +++ b/src/js/state/Viewer/actions.ts @@ -19,7 +19,7 @@ import { VIEWER_SET_RECORDS, VIEWER_SPLICE, VIEWER_STATUS, - ViewerColumns, + SchemaMap, ViewerStatus } from "./types" @@ -61,7 +61,7 @@ export const setRecords = ( export const updateColumns = ( tabId: string, - columns: ViewerColumns + columns: SchemaMap ): VIEWER_COLUMNS => { return { type: "VIEWER_COLUMNS", @@ -72,7 +72,7 @@ export const updateColumns = ( export const setColumns = ( tabId: string, - columns: ViewerColumns + columns: SchemaMap ): VIEWER_SET_COLUMNS => { return { type: "VIEWER_SET_COLUMNS", diff --git a/src/js/state/Viewer/selectors.ts b/src/js/state/Viewer/selectors.ts index e68c264438..52238408c0 100644 --- a/src/js/state/Viewer/selectors.ts +++ b/src/js/state/Viewer/selectors.ts @@ -6,7 +6,7 @@ import {TabState} from "../Tab/types" import Tabs from "../Tabs" import {State} from "../types" import {createSelection, ViewerSelection} from "./helpers/selection" -import {ViewerColumns, ViewerSelectionData, ViewerState} from "./types" +import {SchemaMap, ViewerSelectionData, ViewerState} from "./types" export const getViewer = createSelector( Tabs.getActiveTab, @@ -35,7 +35,7 @@ export const getEndStatus = createSelector( (viewer) => viewer.endStatus ) -export const getColumns = createSelector( +export const getColumns = createSelector( getViewer, (viewer) => viewer.columns ) diff --git a/src/js/state/Viewer/test.ts b/src/js/state/Viewer/test.ts index 11232424f7..e71841fdb2 100644 --- a/src/js/state/Viewer/test.ts +++ b/src/js/state/Viewer/test.ts @@ -1,5 +1,5 @@ import {createRecord} from "test/factories/zed-factory" -import ZedTypeDef from "zealot/zed/type-def" +import {zed} from "zealot" import initTestStore from "../../test/initTestStore" import Tabs from "../Tabs" import Viewer from "../Viewer" @@ -15,6 +15,7 @@ beforeEach(() => { const conn = createRecord({ts: new Date(1000)}) const dns = createRecord({ts: new Date(2000)}) const http = createRecord({ts: new Date(3000)}) +const type = new zed.TypeRecord([{name: "a", type: zed.TypeString}]) test("adding logs to the viewer", () => { const state = store.dispatchAll([ @@ -65,22 +66,10 @@ test("results limited", () => { test("update columns with same tds", () => { const cols1 = { - "9d14c2039a78d76760aae879c7fd2c82": new ZedTypeDef({ - type: { - kind: "typedef", - name: "hello", - type: {kind: "primitive", name: "string"} - } - }) + "9d14c2039a78d76760aae879c7fd2c82": new zed.Schema("1", type) } const cols2 = { - "71f1b421963d31952e15edf7e3957a81": new ZedTypeDef({ - type: { - kind: "typedef", - name: "hello", - type: {kind: "primitive", name: "string"} - } - }) + "71f1b421963d31952e15edf7e3957a81": new zed.Schema("1", type) } const state = store.dispatchAll([ Viewer.updateColumns(tabId, cols1), @@ -88,19 +77,7 @@ test("update columns with same tds", () => { ]) expect(Viewer.getColumns(state)).toEqual({ - "9d14c2039a78d76760aae879c7fd2c82": new ZedTypeDef({ - type: { - kind: "typedef", - name: "hello", - type: {kind: "primitive", name: "string"} - } - }), - "71f1b421963d31952e15edf7e3957a81": new ZedTypeDef({ - type: { - kind: "typedef", - name: "hello", - type: {kind: "primitive", name: "string"} - } - }) + "9d14c2039a78d76760aae879c7fd2c82": new zed.Schema("1", type), + "71f1b421963d31952e15edf7e3957a81": new zed.Schema("1", type) }) }) diff --git a/src/js/state/Viewer/types.ts b/src/js/state/Viewer/types.ts index 7f33f47ee8..a2507faec8 100644 --- a/src/js/state/Viewer/types.ts +++ b/src/js/state/Viewer/types.ts @@ -1,11 +1,10 @@ import {zed} from "zealot" -import {TypeContext} from "zealot/zed/zjson" import {ScrollPosition} from "../../types" import {SearchStatus} from "../../types/searches" export type ViewerStatus = "FETCHING" | "INCOMPLETE" | "COMPLETE" | "LIMIT" -export type ViewerColumns = TypeContext +export type SchemaMap = {[name: string]: zed.Schema} export type ViewerSelectionData = { rows: { [key: number]: boolean @@ -14,7 +13,7 @@ export type ViewerSelectionData = { } export type ViewerState = { records: zed.Record[] - columns: ViewerColumns + columns: SchemaMap endStatus: ViewerStatus status: SearchStatus scrollPos: ScrollPosition @@ -77,13 +76,13 @@ export type VIEWER_STATUS = { export type VIEWER_COLUMNS = { type: "VIEWER_COLUMNS" - columns: ViewerColumns + columns: SchemaMap tabId: string } export type VIEWER_SET_COLUMNS = { type: "VIEWER_SET_COLUMNS" - columns: ViewerColumns + columns: SchemaMap tabId: string } diff --git a/zealot/fetcher/records_callback.ts b/zealot/fetcher/records_callback.ts index 4833b1553d..2eab0bcab9 100644 --- a/zealot/fetcher/records_callback.ts +++ b/zealot/fetcher/records_callback.ts @@ -1,15 +1,17 @@ import {ZealotContext, zed} from "zealot" import * as zqd from "../zqd" +type SchemaMap = {[name: string]: zed.Schema} + export interface RecordsCallbackArgs { channel: number - typedefs: object + schemas: SchemaMap newRows: zed.Record[] rows: zed.Record[] } type ChannelMap = Map -type Channel = {rows: zed.Record[]; typedefs: object} +type Channel = {rows: zed.Record[]; schemas: SchemaMap; typedefs: object} type RecordsCallback = (args: RecordsCallbackArgs) => void type PayloadCallback = (payload: zqd.SearchRecords) => void @@ -17,6 +19,7 @@ function getChannel(id: number, channels: ChannelMap): Channel { if (!channels.has(id)) { channels.set(id, { rows: [], + schemas: {}, typedefs: {} } as Channel) } @@ -31,7 +34,7 @@ export function createRecordsCallback(cb: RecordsCallback): PayloadCallback { const next = ZealotContext.decode(records, prev.typedefs) const chan = { rows: [...prev.rows, ...next], - typedefs: prev.typedefs + schemas: prev.typedefs } channels.set(id, chan) @@ -39,7 +42,7 @@ export function createRecordsCallback(cb: RecordsCallback): PayloadCallback { channel: id, newRows: next, rows: chan.rows, - typedefs: chan.typedefs - }) + schemas: chan.schemas + } as RecordsCallbackArgs) } } From ebd916db629c3a1c921601ba4a5eccb18b15259c Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 28 Apr 2021 20:57:29 -0700 Subject: [PATCH 13/41] Fixed viewer-search test --- .../__snapshots__/viewer-search.test.ts.snap | 364 +++++++++++------- app/search/flows/viewer-search.test.ts | 211 +++++----- .../flows/get-correlation-query.test.ts | 8 +- ppl/detail/flows/get-correlation-query.ts | 8 +- ppl/detail/models/ZeekEvent.ts | 2 +- scripts/test/responses.js | 16 +- src/js/brim/program.ts | 5 +- src/js/brim/syntax.ts | 8 +- src/js/components/RightPane.test.tsx | 4 +- src/js/searches/programs.test.ts | 4 +- src/js/searches/programs.ts | 4 +- src/js/state/Columns/touch.test.ts | 17 +- src/js/state/LogDetails/actions.ts | 6 +- src/js/zql/toZql.ts | 6 +- test/debug.ts | 4 + test/factories/zed-factory.ts | 22 +- test/responses/index.ts | 10 + zealot/fetcher/records_callback.ts | 26 +- zealot/zed/index.ts | 2 +- zealot/zed/types/type-array.ts | 6 +- zealot/zed/types/type-null.ts | 2 +- zealot/zed/types/type-set.ts | 8 +- zealot/zed/values/duration.ts | 6 + zealot/zed/values/primitive.ts | 4 + zealot/zed/values/record.ts | 4 +- zealot/zjson.ts | 2 + 26 files changed, 456 insertions(+), 303 deletions(-) create mode 100644 test/debug.ts diff --git a/app/search/flows/__snapshots__/viewer-search.test.ts.snap b/app/search/flows/__snapshots__/viewer-search.test.ts.snap index 6d4c23a4de..2edb6f8324 100644 --- a/app/search/flows/__snapshots__/viewer-search.test.ts.snap +++ b/app/search/flows/__snapshots__/viewer-search.test.ts.snap @@ -2,7 +2,7 @@ exports[`a normal response sets the Columns columns 1`] = ` Object { - "7247fae6fe9d9e9087c6394e7aba3004": Object { + "28": Object { "AA:bool": Object { "isVisible": true, }, @@ -15,67 +15,67 @@ Object { "TC:bool": Object { "isVisible": true, }, - "TTLs:array": Object { + "TTLs:27": Object { "isVisible": true, }, - "Z:count": Object { + "Z:uint64": Object { "isVisible": true, }, "_path:string": Object { "isVisible": true, }, - "answers:array": Object { + "answers:26": Object { "isVisible": true, }, - "id.orig_h:addr": Object { + "id.orig_h:ip": Object { "isVisible": true, }, - "id.orig_p:port": Object { + "id.orig_p:uint16": Object { "isVisible": true, }, - "id.resp_h:addr": Object { + "id.resp_h:ip": Object { "isVisible": true, }, - "id.resp_p:port": Object { + "id.resp_p:uint16": Object { "isVisible": true, }, - "proto:enum": Object { + "proto:string": Object { "isVisible": true, }, - "qclass:count": Object { + "qclass:uint64": Object { "isVisible": true, }, - "qclass_name:string": Object { + "qclass_name:bstring": Object { "isVisible": true, }, - "qtype:count": Object { + "qtype:uint64": Object { "isVisible": true, }, - "qtype_name:string": Object { + "qtype_name:bstring": Object { "isVisible": true, }, - "query:string": Object { + "query:bstring": Object { "isVisible": true, }, - "rcode:count": Object { + "rcode:uint64": Object { "isVisible": true, }, - "rcode_name:string": Object { + "rcode_name:bstring": Object { "isVisible": true, }, "rejected:bool": Object { "isVisible": true, }, - "rtt:interval": Object { + "rtt:duration": Object { "isVisible": true, }, - "trans_id:count": Object { + "trans_id:uint64": Object { "isVisible": true, }, "ts:time": Object { "isVisible": true, }, - "uid:string": Object { + "uid:bstring": Object { "isVisible": true, }, }, @@ -84,117 +84,221 @@ Object { exports[`a normal response sets the viewer columns 1`] = ` Object { - "7247fae6fe9d9e9087c6394e7aba3004": Schema { - "columns": Array [ - Object { - "name": "_path", - "type": "string", - }, - Object { - "name": "ts", - "type": "time", - }, - Object { - "name": "uid", - "type": "string", - }, - Object { - "name": "id", - "of": Array [ - Object { - "name": "orig_h", - "type": "addr", - }, - Object { - "name": "orig_p", - "type": "port", - }, - Object { - "name": "resp_h", - "type": "addr", - }, - Object { - "name": "resp_p", - "type": "port", - }, - ], - "type": "record", - }, - Object { - "name": "proto", - "type": "enum", - }, - Object { - "name": "trans_id", - "type": "count", - }, - Object { - "name": "rtt", - "type": "interval", - }, - Object { - "name": "query", - "type": "string", - }, - Object { - "name": "qclass", - "type": "count", - }, - Object { - "name": "qclass_name", - "type": "string", - }, - Object { - "name": "qtype", - "type": "count", - }, - Object { - "name": "qtype_name", - "type": "string", - }, - Object { - "name": "rcode", - "type": "count", - }, - Object { - "name": "rcode_name", - "type": "string", - }, - Object { - "name": "AA", - "type": "bool", - }, - Object { - "name": "TC", - "type": "bool", - }, - Object { - "name": "RD", - "type": "bool", - }, - Object { - "name": "RA", - "type": "bool", - }, - Object { - "name": "Z", - "type": "count", - }, - Object { - "name": "answers", - "of": "string", - "type": "array", - }, - Object { - "name": "TTLs", - "of": "interval", - "type": "array", - }, - Object { - "name": "rejected", - "type": "bool", - }, - ], + "28": Schema { + "name": "28", + "type": TypeRecord { + "fields": Array [ + Object { + "name": "_path", + "type": TypeOfString { + "kind": "primitive", + "name": "string", + }, + }, + Object { + "name": "ts", + "type": TypeOfTime { + "kind": "primitive", + "name": "time", + }, + }, + Object { + "name": "uid", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + Object { + "name": "id", + "type": TypeRecord { + "fields": Array [ + Object { + "name": "orig_h", + "type": TypeOfIp { + "kind": "primitive", + "name": "ip", + }, + }, + Object { + "name": "orig_p", + "type": TypeAlias { + "id": 23, + "kind": "alias", + "name": "port", + "type": TypeOfUint16 { + "kind": "primitive", + "name": "uint16", + }, + }, + }, + Object { + "name": "resp_h", + "type": TypeOfIp { + "kind": "primitive", + "name": "ip", + }, + }, + Object { + "name": "resp_p", + "type": TypeAlias { + "id": 23, + "kind": "alias", + "name": "port", + "type": TypeOfUint16 { + "kind": "primitive", + "name": "uint16", + }, + }, + }, + ], + "id": 24, + "kind": "record", + }, + }, + Object { + "name": "proto", + "type": TypeAlias { + "id": 25, + "kind": "alias", + "name": "zenum", + "type": TypeOfString { + "kind": "primitive", + "name": "string", + }, + }, + }, + Object { + "name": "trans_id", + "type": TypeOfUint64 { + "kind": "primitive", + "name": "uint64", + }, + }, + Object { + "name": "rtt", + "type": TypeOfDuration { + "kind": "primitive", + "name": "duration", + }, + }, + Object { + "name": "query", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + Object { + "name": "qclass", + "type": TypeOfUint64 { + "kind": "primitive", + "name": "uint64", + }, + }, + Object { + "name": "qclass_name", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + Object { + "name": "qtype", + "type": TypeOfUint64 { + "kind": "primitive", + "name": "uint64", + }, + }, + Object { + "name": "qtype_name", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + Object { + "name": "rcode", + "type": TypeOfUint64 { + "kind": "primitive", + "name": "uint64", + }, + }, + Object { + "name": "rcode_name", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + Object { + "name": "AA", + "type": TypeOfBool { + "kind": "primitive", + "name": "bool", + }, + }, + Object { + "name": "TC", + "type": TypeOfBool { + "kind": "primitive", + "name": "bool", + }, + }, + Object { + "name": "RD", + "type": TypeOfBool { + "kind": "primitive", + "name": "bool", + }, + }, + Object { + "name": "RA", + "type": TypeOfBool { + "kind": "primitive", + "name": "bool", + }, + }, + Object { + "name": "Z", + "type": TypeOfUint64 { + "kind": "primitive", + "name": "uint64", + }, + }, + Object { + "name": "answers", + "type": TypeArray { + "id": 26, + "kind": "array", + "type": TypeOfBString { + "kind": "primitive", + "name": "bstring", + }, + }, + }, + Object { + "name": "TTLs", + "type": TypeArray { + "id": 27, + "kind": "array", + "type": TypeOfDuration { + "kind": "primitive", + "name": "duration", + }, + }, + }, + Object { + "name": "rejected", + "type": TypeOfBool { + "kind": "primitive", + "name": "bool", + }, + }, + ], + "id": 28, + "kind": "record", + }, }, } `; diff --git a/app/search/flows/viewer-search.test.ts b/app/search/flows/viewer-search.test.ts index b15566707f..97ca224f74 100644 --- a/app/search/flows/viewer-search.test.ts +++ b/app/search/flows/viewer-search.test.ts @@ -1,122 +1,117 @@ -// import tabHistory from "app/router/tab-history" -// import {lakePath} from "app/router/utils/paths" -// import Columns from "src/js/state/Columns" -// import Handlers from "src/js/state/Handlers" -// import SearchBar from "src/js/state/SearchBar" -// import Spaces from "src/js/state/Spaces" -// import Viewer from "src/js/state/Viewer" -// import Workspaces from "src/js/state/Workspaces" -// import fixtures from "src/js/test/fixtures" -// import initTestStore from "src/js/test/initTestStore" -// import responses from "src/js/test/responses" -// import {STRING} from "test/fixtures/zjson-types" -// import {createZealotMock} from "zealot" -// import {zed} from "zealot" -// import {viewerSearch} from "./viewer-search" +import tabHistory from "app/router/tab-history" +import {lakePath} from "app/router/utils/paths" +import Columns from "src/js/state/Columns" +import Handlers from "src/js/state/Handlers" +import SearchBar from "src/js/state/SearchBar" +import Spaces from "src/js/state/Spaces" +import Viewer from "src/js/state/Viewer" +import Workspaces from "src/js/state/Workspaces" +import fixtures from "src/js/test/fixtures" +import initTestStore from "src/js/test/initTestStore" +import responses from "src/js/test/responses" +import {createRecord} from "test/factories/zed-factory" +import {useResponse} from "test/responses" +import {createZealotMock} from "zealot" +import {viewerSearch} from "./viewer-search" -// const dnsResp = responses("dns.txt") -// const space = fixtures("space1") -// const warningResp = responses("search_warning.txt") +const dnsResp = useResponse("dns") +const space = fixtures("space1") +const warningResp = responses("search_warning.txt") -// let store, zealot, dispatch, select -// beforeEach(() => { -// zealot = createZealotMock() -// store = initTestStore(zealot.zealot) -// dispatch = store.dispatch -// select = (s: any) => s(store.getState()) +let store, zealot, dispatch, select +beforeEach(() => { + zealot = createZealotMock() + store = initTestStore(zealot.zealot) + dispatch = store.dispatch + select = (s: any) => s(store.getState()) -// store.dispatchAll([ -// Workspaces.add({ -// host: "testHost", -// id: "1", -// name: "testName", -// port: "9867", -// authType: "none" -// }), -// Spaces.setDetail("1", space) -// ]) -// store.dispatch(tabHistory.push(lakePath(space.id, "1"))) -// }) + store.dispatchAll([ + Workspaces.add({ + host: "testHost", + id: "1", + name: "testName", + port: "9867", + authType: "none" + }), + Spaces.setDetail("1", space) + ]) + store.dispatch(tabHistory.push(lakePath(space.id, "1"))) +}) -// const submit = () => -// dispatch( -// viewerSearch({ -// query: "dns query | head 500", -// from: new Date(), -// to: new Date(1) -// }) -// ) +const submit = () => + dispatch( + viewerSearch({ + query: "dns query | head 500", + from: new Date(), + to: new Date(1) + }) + ) -// describe("a normal response", () => { -// beforeEach(() => { -// zealot.stubStream("search", dnsResp) -// }) +describe("a normal response", () => { + beforeEach(() => { + zealot.stubStream("search", dnsResp) + }) -// test("zealot gets the request", async () => { -// await submit() -// const calls = zealot.calls("search") -// expect(calls.length).toBe(1) -// expect(calls[0].args).toEqual("dns query | head 500") -// }) + test("zealot gets the request", async () => { + await submit() + const calls = zealot.calls("search") + expect(calls.length).toBe(1) + expect(calls[0].args).toEqual("dns query | head 500") + }) -// test("the table gets populated", async () => { -// await submit() -// expect(select(Viewer.getViewerRecords).length).toBe(2) -// }) + test("the table gets populated", async () => { + await submit() + expect(select(Viewer.getViewerRecords).length).toBe(2) + }) -// test("the table gets cleared", async () => { -// dispatch( -// Viewer.setRecords(undefined, [ -// zed.Record.of([{name: "clear", type: STRING}], ["me"]) -// ]) -// ) -// await submit() -// expect(select(Viewer.getViewerRecords)[0]).not.toEqual([ -// {name: "clear", type: "string", value: "me"} -// ]) -// }) + test("the table gets cleared", async () => { + dispatch(Viewer.setRecords(undefined, [createRecord({clear: "me"})])) + await submit() + expect(select(Viewer.getViewerRecords)[0]).not.toEqual([ + {name: "clear", type: "string", value: "me"} + ]) + }) -// test("the table status updates", async () => { -// const promise = submit() -// expect(select(Viewer.getStatus)).toBe("FETCHING") -// expect(select(Viewer.getEndStatus)).toBe("FETCHING") -// await promise -// expect(select(Viewer.getStatus)).toBe("SUCCESS") -// expect(select(Viewer.getEndStatus)).toBe("COMPLETE") -// }) + test("the table status updates", async () => { + const promise = submit() + expect(select(Viewer.getStatus)).toBe("FETCHING") + expect(select(Viewer.getEndStatus)).toBe("FETCHING") + await promise + expect(select(Viewer.getStatus)).toBe("SUCCESS") + expect(select(Viewer.getEndStatus)).toBe("COMPLETE") + }) -// test("registers a table request then cleans it up", async () => { -// const promise = submit() -// expect(select(Handlers.get)["Table"]).toEqual( -// expect.objectContaining({type: "SEARCH"}) -// ) -// await promise -// expect(select(Handlers.get)["Table"]).toBe(undefined) -// }) + test("registers a table request then cleans it up", async () => { + const promise = submit() + expect(select(Handlers.get)["Table"]).toEqual( + expect.objectContaining({type: "SEARCH"}) + ) + await promise + expect(select(Handlers.get)["Table"]).toBe(undefined) + }) -// test("aborts previous table request", async () => { -// const abort = jest.fn() -// dispatch(Handlers.register("Table", {type: "SEARCH", abort})) -// await submit() -// expect(abort).toHaveBeenCalledTimes(1) -// }) + test("aborts previous table request", async () => { + const abort = jest.fn() + dispatch(Handlers.register("Table", {type: "SEARCH", abort})) + await submit() + expect(abort).toHaveBeenCalledTimes(1) + }) -// test("sets the viewer columns", async () => { -// await submit() -// expect(select(Viewer.getColumns)).toMatchSnapshot() -// }) + test("sets the viewer columns", async () => { + await submit() + expect(select(Viewer.getColumns)).toMatchSnapshot() + }) -// test("sets the Columns columns", async () => { -// await submit() -// expect(select(Columns.getColumns)).toMatchSnapshot() -// }) -// }) + test("sets the Columns columns", async () => { + await submit() + expect(select(Columns.getColumns)).toMatchSnapshot() + }) +}) -// test("a response with a warning", async () => { -// zealot.stubStream("search", warningResp) -// await submit() -// expect(select(SearchBar.getSearchBarError)).toBe( -// "Cut field boo not present in input" -// ) -// }) -test("nothing", () => {}) +test("a response with a warning", async () => { + zealot.stubStream("search", warningResp) + await submit() + expect(select(SearchBar.getSearchBarError)).toBe( + "Cut field boo not present in input" + ) +}) diff --git a/ppl/detail/flows/get-correlation-query.test.ts b/ppl/detail/flows/get-correlation-query.test.ts index d7ce577c15..a0a1dc5680 100644 --- a/ppl/detail/flows/get-correlation-query.test.ts +++ b/ppl/detail/flows/get-correlation-query.test.ts @@ -25,14 +25,14 @@ test("returns conn query if ts and duration are present", () => { uid: "CHem0e2rJqHiwjhgq7", community_id: "1:NYgcI8mLerCC20GwJVV5AftL0uY=", ts: new Date(1585852166.003543 * 1000), - duration: null + duration: new zed.Duration("0") }) expect(getCorrelationQuery(record)).toBe( connCorrelation( record.get("uid"), record.get("community_id"), record.get("ts"), - record.get("duration") + record.get("duration") ) ) }) @@ -42,7 +42,7 @@ test("returns cid query if only cid present", () => { _path: "conn", community_id: "1:NYgcI8mLerCC20GwJVV5AftL0uY=", ts: new Date(1585852166.003543 * 1000), - duration: null + duration: new zed.Duration("0") }) expect(getCorrelationQuery(record)).toBe( @@ -54,7 +54,7 @@ test("returns null if no cid or uid", () => { const record = createRecord({ _path: "conn", ts: new Date(1585852166.003543 * 1000), - duration: null + duration: new zed.Duration("0") }) expect(getCorrelationQuery(record)).toBe(null) diff --git a/ppl/detail/flows/get-correlation-query.ts b/ppl/detail/flows/get-correlation-query.ts index fac14bcc32..01ec4b495a 100644 --- a/ppl/detail/flows/get-correlation-query.ts +++ b/ppl/detail/flows/get-correlation-query.ts @@ -8,13 +8,15 @@ import {Correlation} from "../models/Correlation" export function getCorrelationQuery(record: zed.Record) { const {uid, cid} = new Correlation(record).getIds() + const ts = record.try("ts") + const dur = record.try("duration") - if (cid && uid && record.has("ts") && record.has("duration")) { + if (cid && uid && ts && ts.isSet() && dur && dur.isSet()) { return connCorrelation( record.get("uid") as zed.String, record.get("community_id") as zed.String, - record.get("ts") as zed.Time, - record.get("duration") as zed.Float64 + ts, + dur ) } else if (uid) { return uidCorrelation(uid) diff --git a/ppl/detail/models/ZeekEvent.ts b/ppl/detail/models/ZeekEvent.ts index d389decc5f..610c1eb08e 100644 --- a/ppl/detail/models/ZeekEvent.ts +++ b/ppl/detail/models/ZeekEvent.ts @@ -14,7 +14,7 @@ export class ZeekEvent implements BrimEventInterface { getEndTime() { if (this.r.get("_path").toString() !== "conn") return null - const dur = this.r.get("duration").toFloat() + const dur = this.r.get("duration").asSeconds() if (!dur) return const ts = this.r.get("ts").toDate() return new Date(ts.getTime() + dur * 1000) diff --git a/scripts/test/responses.js b/scripts/test/responses.js index 99b93229b0..dd93210310 100644 --- a/scripts/test/responses.js +++ b/scripts/test/responses.js @@ -1,9 +1,23 @@ +/** + * This will spin up a new instance of zqd for each of the responses + * saved in the config file, ingest the input file, run the query, + * and save the response in the output file. It uses Deno to manage + * the zqd process and the zealot calls. (search.deno.ts) + * + * It will use the zqd bundled in zdeps. + * + * Before running this, it would be good to: + * 1. npm install # to install the desired version of zqd + * 2. npx rollup -c --silent # to bundle the current version of zealot + * + * Then run this script: node scripts/test/responses.js + */ + const {spawn} = require("child_process") const fs = require("fs-extra") const glob = require("glob") const config = require("../../test/responses/config") -// In order for this to work, rollup the zealot module first function saveResponse(input, output, query) { const deno = spawn( "deno", diff --git a/src/js/brim/program.ts b/src/js/brim/program.ts index 8810f6a96a..1cbbad4551 100644 --- a/src/js/brim/program.ts +++ b/src/js/brim/program.ts @@ -1,11 +1,9 @@ import {isEqual} from "lodash" -import {parse} from "zealot" -import {zed} from "zealot" +import {parse, zed} from "zealot" import {trim} from "../lib/Str" import stdlib from "../stdlib" import brim from "./" import {EVERYTHING_FILTER, FILTER_PROC, TUPLE_PROCS} from "./ast" -import {createCell} from "./cell" export default function(p = "", pins: string[] = []) { p = concatPins(p, pins) @@ -45,7 +43,6 @@ export default function(p = "", pins: string[] = []) { .groupByKeys() .map((n) => log.tryField(n)) .filter((f) => !!f) - .map(createCell) .map(brim.syntax.include) .join(" ") diff --git a/src/js/brim/syntax.ts b/src/js/brim/syntax.ts index 4771859417..e58d209e96 100644 --- a/src/js/brim/syntax.ts +++ b/src/js/brim/syntax.ts @@ -3,16 +3,16 @@ import {toZql} from "../zql/toZql" export default { exclude(field: zed.Field) { - return `${field.name}!=${toZql(field.data)}` + return `${field.name}!=${toZql(field.value)}` }, include(field: zed.Field) { - return `${field.name}=${toZql(field.data)}` + return `${field.name}=${toZql(field.value)}` }, in(field: zed.Field) { - return `${toZql(field.data)} in ${field.name}` + return `${toZql(field.value)} in ${field.name}` }, notIn(field: zed.Field) { - return `!${toZql(field.data)} in ${field.name}` + return `!${toZql(field.value)} in ${field.name}` }, countBy(field: zed.Field) { return `count() by ${field.name}` diff --git a/src/js/components/RightPane.test.tsx b/src/js/components/RightPane.test.tsx index 2db84c8e92..766ac3bb77 100644 --- a/src/js/components/RightPane.test.tsx +++ b/src/js/components/RightPane.test.tsx @@ -1,7 +1,7 @@ import tabHistory from "app/router/tab-history" import {workspacesPath} from "app/router/utils/paths" import React from "react" -import {zed} from "zealot" +import {createRecord} from "test/factories/zed-factory" import Layout from "../state/Layout" import LogDetails from "../state/LogDetails" import loginTo from "../test/helpers/loginTo" @@ -13,7 +13,7 @@ test("no errors if space does not exist", async () => { store.dispatch(Layout.showRightSidebar()) store.dispatch(tabHistory.push(workspacesPath())) - store.dispatch(LogDetails.push(new zed.Record(null, []))) + store.dispatch(LogDetails.push(createRecord({}))) const el = provide(store, ) expect(el.html()).toBe("") }) diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index 84e745a288..425cd09a1f 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -6,7 +6,7 @@ test("conn correlation", () => { const record = createRecord({ ts: new Date(1425568032.998178 * 1000), uid: "CbOjYpkXn9LfqV51c", - duration: 0.70995, + duration: new zed.Duration("0.70995"), community_id: "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" }) expect( @@ -14,7 +14,7 @@ test("conn correlation", () => { record.get("uid") as zed.String, record.get("community_id") as zed.String, record.get("ts") as zed.Time, - record.get("duration") as zed.Float64 + record.get("duration") as zed.Duration ) ).toBe( 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" and ts >= 1425568032.998 and ts < 1425568123.707) | head 100' diff --git a/src/js/searches/programs.ts b/src/js/searches/programs.ts index 38a4f8cec6..4318767e3a 100644 --- a/src/js/searches/programs.ts +++ b/src/js/searches/programs.ts @@ -54,10 +54,10 @@ export function connCorrelation( uid: zed.String, cid: zed.String, ts: zed.Time, - duration: zed.Float64 + duration: zed.Duration ) { const tsDate = ts.toDate() - const dur = duration.toFloat() + 90 // Add a 1.5 minute buffer for events that get logged late + const dur = duration.asSeconds() + 90 // Add a 1.5 minute buffer for events that get logged late const endTsDate = new Date(new Date(tsDate).getTime() + dur * 1000) const cidFilter = zql`community_id = ${cid} and ts >= ${tsDate} and ts < ${endTsDate}` return `${uidFilter(uid)} or (${cidFilter}) | ${correlationLimit()}` diff --git a/src/js/state/Columns/touch.test.ts b/src/js/state/Columns/touch.test.ts index 4e5e023407..6480c8753e 100644 --- a/src/js/state/Columns/touch.test.ts +++ b/src/js/state/Columns/touch.test.ts @@ -1,4 +1,3 @@ -import {INTERVAL, STRING} from "test/fixtures/zjson-types" import {zed} from "zealot" import initTestStore from "../../test/initTestStore" import Columns from "./" @@ -30,7 +29,7 @@ beforeEach(() => { test("visibility false when at least one is hidden", () => { const prefName = "temp" - const col = createColumn({name: "_path", type: STRING}) + const col = createColumn({name: "_path", type: zed.TypeString.serialize()}) const update = {[col.key]: {isVisible: false}} store.dispatch(actions.updateColumns(prefName, update)) @@ -39,7 +38,7 @@ test("visibility false when at least one is hidden", () => { expect(prefs).toEqual({ "_path:string": {isVisible: false}, - "duration:interval": {isVisible: false}, + "duration:duration": {isVisible: false}, "ts:time": {isVisible: false} }) }) @@ -52,15 +51,19 @@ test("visibility true when no preferences exist", () => { expect(prefs).toEqual({ "_path:string": {isVisible: true}, - "duration:interval": {isVisible: true}, + "duration:duration": {isVisible: true}, "ts:time": {isVisible: true} }) }) test("visibility true when all are visible", () => { const prefName = "temp" - const col = createColumn({name: "_path", type: STRING}) - const col2 = createColumn({name: "duration", type: INTERVAL}) + + const col = createColumn({name: "_path", type: zed.TypeString.serialize()}) + const col2 = createColumn({ + name: "duration", + type: zed.TypeDuration.serialize() + }) const update = { [col.key]: {isVisible: true}, [col2.key]: {isVisible: true} @@ -72,7 +75,7 @@ test("visibility true when all are visible", () => { expect(prefs).toEqual({ "_path:string": {isVisible: true}, - "duration:interval": {isVisible: true}, + "duration:duration": {isVisible: true}, "ts:time": {isVisible: true} }) }) diff --git a/src/js/state/LogDetails/actions.ts b/src/js/state/LogDetails/actions.ts index 6cc35e5f09..20319d630a 100644 --- a/src/js/state/LogDetails/actions.ts +++ b/src/js/state/LogDetails/actions.ts @@ -1,5 +1,5 @@ import {SearchStatus} from "src/js/types/searches" -import {zed} from "zealot" +import {ZealotContext, zed} from "zealot" import { LOG_DETAIL_BACK, LOG_DETAIL_CLEAR, @@ -11,7 +11,7 @@ import { export default { push: (record: zed.Record): LOG_DETAIL_PUSH => ({ type: "LOG_DETAIL_PUSH", - record: record.serialize() + record: ZealotContext.encodeRecord(record) }), back: (): LOG_DETAIL_BACK => ({ @@ -26,7 +26,7 @@ export default { return { type: "LOG_DETAIL_UPDATE", updates: { - uidLogs: records.map((r) => r.serialize()) + uidLogs: records.map((r) => ZealotContext.encodeRecord(r)) } } }, diff --git a/src/js/zql/toZql.ts b/src/js/zql/toZql.ts index 4840dd8f4a..b7b530eccc 100644 --- a/src/js/zql/toZql.ts +++ b/src/js/zql/toZql.ts @@ -30,7 +30,9 @@ function toZqlBool(bool: boolean) { } function toZqlZngPrimitive(data: zed.Primitive) { - if (data instanceof zed.String || data instanceof zed.BString) + if (data instanceof zed.Ip) { + return data.toString() + } else { return toZqlString(data.toString()) - throw new Error(`Can't convert Zng Type: ${data.type} to zql`) + } } diff --git a/test/debug.ts b/test/debug.ts new file mode 100644 index 0000000000..cfb8f9ddf0 --- /dev/null +++ b/test/debug.ts @@ -0,0 +1,4 @@ +export function puts(obj) { + console.log(obj) + return obj +} diff --git a/test/factories/zed-factory.ts b/test/factories/zed-factory.ts index e138d60f3a..08eaf416f6 100644 --- a/test/factories/zed-factory.ts +++ b/test/factories/zed-factory.ts @@ -1,21 +1,31 @@ import {isDate, isInteger, isNumber, isObject, isString} from "lodash" -import {zed} from "zealot" +import {ZealotContext, zed} from "zealot" // Convert a js object into a zed record + export function createRecord(object): zed.Record { - let fields = [] + let fields: zed.Field[] = [] for (let name in object) { fields.push(createField(name, object[name])) } - // If the tests need the type, create it. - return new zed.Record(null, fields) + const typeFields: zed.TypeField[] = fields.map((f) => ({ + name: f.name, + type: f.value.type + })) + + const type = ZealotContext.lookupTypeRecord(typeFields) + return new zed.Record(type, fields) } -export function createField(name, value) { +export function createField(name, value): zed.Field { return new zed.Field(name, createData(value)) } -export function createData(value) { +export function createData(value): zed.AnyValue { + if (value instanceof zed.Primitive) { + return value as zed.AnyValue + } + if (value === null) { return new zed.Null() } diff --git a/test/responses/index.ts b/test/responses/index.ts index 5234b52bcc..0a8e5be78a 100644 --- a/test/responses/index.ts +++ b/test/responses/index.ts @@ -4,6 +4,16 @@ import * as config from "test/responses/config" const cache = {} +/** + * + * @param name string key of the object in test/responses/config.ts + * @returns An array of server responses from zqd as parsed json + * + * Example: const response = useResponse("dns") + * + * To recreate all the responses saved in the config file: + * node scripts/test/responses.js + */ export function useResponse(name: string) { if (name in cache) return cache[name] diff --git a/zealot/fetcher/records_callback.ts b/zealot/fetcher/records_callback.ts index 2eab0bcab9..a45ca6de9e 100644 --- a/zealot/fetcher/records_callback.ts +++ b/zealot/fetcher/records_callback.ts @@ -29,20 +29,18 @@ function getChannel(id: number, channels: ChannelMap): Channel { export function createRecordsCallback(cb: RecordsCallback): PayloadCallback { let channels = new Map() - return ({channel_id: id, records}: zqd.SearchRecords) => { - const prev = getChannel(id, channels) - const next = ZealotContext.decode(records, prev.typedefs) - const chan = { - rows: [...prev.rows, ...next], - schemas: prev.typedefs - } - channels.set(id, chan) + return ({channel_id: channel, records}: zqd.SearchRecords) => { + const {typedefs, schemas, rows: prevRows} = getChannel(channel, channels) + const newRows = records.map((zjson) => { + const rec = ZealotContext.decodeRecord(zjson, typedefs) + const name = zjson.schema + const type = typedefs[name] + schemas[name] = new zed.Schema(name, type) + return rec + }) + const rows = prevRows.concat(newRows) + channels.set(channel, {rows, typedefs, schemas}) - cb({ - channel: id, - newRows: next, - rows: chan.rows, - schemas: chan.schemas - } as RecordsCallbackArgs) + cb({channel, rows, newRows, schemas} as RecordsCallbackArgs) } } diff --git a/zealot/zed/index.ts b/zealot/zed/index.ts index 60823e47fd..bf3565d1df 100644 --- a/zealot/zed/index.ts +++ b/zealot/zed/index.ts @@ -16,7 +16,7 @@ export {TypeIp} from "./types/type-ip" export {TypeMap} from "./types/type-map" export {TypeNet} from "./types/type-net" export {TypeNull} from "./types/type-null" -export {TypeRecord} from "./types/type-record" +export {TypeRecord, TypeField} from "./types/type-record" export {TypeSet} from "./types/type-set" export {TypeString} from "./types/type-string" export {TypeTime} from "./types/type-time" diff --git a/zealot/zed/types/type-array.ts b/zealot/zed/types/type-array.ts index 040e93f32a..a6936a1b84 100644 --- a/zealot/zed/types/type-array.ts +++ b/zealot/zed/types/type-array.ts @@ -18,10 +18,12 @@ export class TypeArray implements ContainerTypeInterface { return `[${typeId(type)}]` } - create(values, typedefs) { + create(values: zjson.ArrayValue, typedefs) { return new Array( this, - values.map((value) => this.type.create(value, typedefs)) + isNull(values) + ? null + : values.map((value) => this.type.create(value, typedefs)) ) } diff --git a/zealot/zed/types/type-null.ts b/zealot/zed/types/type-null.ts index 930fa6aa99..8498381e70 100644 --- a/zealot/zed/types/type-null.ts +++ b/zealot/zed/types/type-null.ts @@ -10,7 +10,7 @@ class TypeOfNull implements PrimitiveTypeInterface { return {kind: "primitive", name: this.name} } - create() { + create(_value: any) { return new Null() } } diff --git a/zealot/zed/types/type-set.ts b/zealot/zed/types/type-set.ts index ddc84eef3d..0ea7e2521f 100644 --- a/zealot/zed/types/type-set.ts +++ b/zealot/zed/types/type-set.ts @@ -1,9 +1,9 @@ import {isNull} from "lodash" -import {Value} from "zealot/zjson" +import {SetValue, Value} from "zealot/zjson" import {ZedContext} from "../context" +import {typeId} from "../utils" import {Set} from "../values/set" import {ContainerTypeInterface, ZedType} from "./types" -import {typeId} from "../utils" export class TypeSet implements ContainerTypeInterface { kind = "set" @@ -17,10 +17,10 @@ export class TypeSet implements ContainerTypeInterface { return `|[${typeId(type)}]|` } - create(values, typedefs) { + create(values: SetValue, typedefs) { return new Set( this, - values.map((v) => this.type.create(v, typedefs)) + isNull(values) ? null : values.map((v) => this.type.create(v, typedefs)) ) } diff --git a/zealot/zed/values/duration.ts b/zealot/zed/values/duration.ts index 3cf4cbb113..6102c287cd 100644 --- a/zealot/zed/values/duration.ts +++ b/zealot/zed/values/duration.ts @@ -1,6 +1,12 @@ +import {isNull} from "lodash" import {TypeDuration} from "../types/type-duration" import {Primitive} from "./primitive" export class Duration extends Primitive { type = TypeDuration + + asSeconds() { + if (isNull(this.value)) return null + return parseFloat(this.value) + } } diff --git a/zealot/zed/values/primitive.ts b/zealot/zed/values/primitive.ts index 8cac1c8dde..c899246d52 100644 --- a/zealot/zed/values/primitive.ts +++ b/zealot/zed/values/primitive.ts @@ -11,6 +11,10 @@ export class Primitive implements ZedValueInterface { return isNull(this.value) } + isSet() { + return !this.isUnset() + } + toString() { if (isNull(this.value)) return "null" return this.value.toString() diff --git a/zealot/zed/values/record.ts b/zealot/zed/values/record.ts index 4a4a9f73ca..ba3e77cca3 100644 --- a/zealot/zed/values/record.ts +++ b/zealot/zed/values/record.ts @@ -68,9 +68,9 @@ export class Record implements ZedValueInterface { }, new Field("", this)) } - try(name: string) { + try(name: string): T | null { try { - return this.get(name) + return this.get(name) as T } catch { return null } diff --git a/zealot/zjson.ts b/zealot/zjson.ts index 055bd40dcd..8adebe51ae 100644 --- a/zealot/zjson.ts +++ b/zealot/zjson.ts @@ -77,4 +77,6 @@ export type Value = string | null | Value[] export type ArrayValue = Value[] | null +export type SetValue = Value[] | null + export type TypeContext = object From 7c6b08934e057926e7c739e74f83f9ed6941f9d1 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 29 Apr 2021 09:04:50 -0700 Subject: [PATCH 14/41] Need to rebase --- .../histogram-search.test.ts.snap | 44 +----- app/search/flows/histogram-search.test.ts | 137 +++++++++--------- ppl/detail/flows/fetch.test.ts | 2 +- scripts/test/responses.js | 4 +- src/js/searches/programs.test.ts | 2 +- test/responses/config.js | 5 + .../correlation-uid-community-id.response | 4 +- test/responses/correlation-uid.response | 4 +- test/responses/count-by-path.response | 4 +- test/responses/count.response | 2 +- test/responses/dns.response | 4 +- test/responses/every-count-by-path.response | 17 +++ .../no-community-id-in-conn.response | 4 +- test/responses/only-alerts.response | 4 +- test/responses/sample.response | 4 +- zealot/zed/context.ts | 3 +- zealot/zed/types/type-array.ts | 2 +- zealot/zed/types/type-map.ts | 2 +- zealot/zed/types/type-record.ts | 2 +- zealot/zed/types/type-set.ts | 2 +- zealot/zed/types/type-type.ts | 2 +- zealot/zed/types/type-union.ts | 2 +- zealot/zed/utils.ts | 4 + zealot/zed/values/array.ts | 2 +- zealot/zed/values/duration.ts | 14 +- zealot/zed/values/float64.ts | 2 +- zealot/zed/values/int16.ts | 2 +- zealot/zed/values/int32.ts | 2 +- zealot/zed/values/int64.ts | 2 +- zealot/zed/values/int8.ts | 2 +- zealot/zed/values/map.ts | 2 +- zealot/zed/values/primitive.ts | 2 +- zealot/zed/values/record.ts | 2 +- zealot/zed/values/set.ts | 2 +- zealot/zed/values/time.ts | 15 +- zealot/zed/values/type.ts | 2 +- zealot/zed/values/uint16.ts | 2 +- zealot/zed/values/uint32.ts | 2 +- zealot/zed/values/uint64.ts | 2 +- zealot/zed/values/uint8.ts | 2 +- zealot/zed/values/union.ts | 2 +- 41 files changed, 167 insertions(+), 154 deletions(-) create mode 100644 test/responses/every-count-by-path.response diff --git a/app/search/flows/__snapshots__/histogram-search.test.ts.snap b/app/search/flows/__snapshots__/histogram-search.test.ts.snap index 5d983502e1..6d8714466d 100644 --- a/app/search/flows/__snapshots__/histogram-search.test.ts.snap +++ b/app/search/flows/__snapshots__/histogram-search.test.ts.snap @@ -5,56 +5,24 @@ Object { "keys": Array [ "capture_loss", "conn", - "files", "x509", "ssl", + "files", "dns", "weird", "stats", ], "table": Object { - "1582646585000": Object { - "conn": 1, - "stats": 1, - }, - "1582646586000": Object { - "conn": 1, - "weird": 1, - }, - "1582646587000": Object { - "conn": 2, - }, - "1582646588000": Object { - "conn": 5, - }, - "1582646589000": Object { - "conn": 1, - }, - "1582646590000": Object { - "conn": 2, - }, - "1582646591000": Object { - "conn": 1, - "dns": 1, - }, - "1582646592000": Object { - "conn": 1, - }, - "1582646593000": Object { - "conn": 2, + "NaN": Object { + "capture_loss": 1, + "conn": 3, "dns": 1, - }, - "1582646594000": Object { "files": 4, "ssl": 1, + "stats": 1, + "weird": 1, "x509": 1, }, - "1582646595000": Object { - "conn": 3, - }, - "1582646597000": Object { - "capture_loss": 1, - }, }, } `; diff --git a/app/search/flows/histogram-search.test.ts b/app/search/flows/histogram-search.test.ts index 8f95328f2a..432d3f8dc4 100644 --- a/app/search/flows/histogram-search.test.ts +++ b/app/search/flows/histogram-search.test.ts @@ -1,78 +1,77 @@ -// import tabHistory from "app/router/tab-history" -// import {lakePath} from "app/router/utils/paths" -// import Chart from "src/js/state/Chart" -// import Handlers from "src/js/state/Handlers" -// import Spaces from "src/js/state/Spaces" -// import Workspaces from "src/js/state/Workspaces" -// import fixtures from "src/js/test/fixtures" -// import initTestStore from "src/js/test/initTestStore" -// import responses from "src/js/test/responses" -// import {createZealotMock} from "zealot" -// import {histogramSearch} from "./histogram-search" +import tabHistory from "app/router/tab-history" +import {lakePath} from "app/router/utils/paths" +import Chart from "src/js/state/Chart" +import Handlers from "src/js/state/Handlers" +import Spaces from "src/js/state/Spaces" +import Workspaces from "src/js/state/Workspaces" +import fixtures from "src/js/test/fixtures" +import initTestStore from "src/js/test/initTestStore" +import {useResponse} from "test/responses" +import {createZealotMock} from "zealot" +import {histogramSearch} from "./histogram-search" -// const countByPathResp = responses("count_by_path.txt") -// const space = fixtures("space1") +const countByPathResp = useResponse("everyCountByPath") +const space = fixtures("space1") -// let store, zealot, dispatch, select -// beforeEach(() => { -// zealot = createZealotMock() -// store = initTestStore(zealot.zealot) -// dispatch = store.dispatch -// select = (s: any) => s(store.getState()) +let store, zealot, dispatch, select +beforeEach(() => { + zealot = createZealotMock() + store = initTestStore(zealot.zealot) + dispatch = store.dispatch + select = (s: any) => s(store.getState()) -// store.dispatchAll([ -// Workspaces.add({ -// host: "testHost", -// id: "1", -// name: "testName", -// port: "9867", -// authType: "none" -// }), -// Spaces.setDetail("1", space) -// ]) -// store.dispatch(tabHistory.push(lakePath(space.id, "1"))) -// zealot.stubStream("search", countByPathResp) -// }) + store.dispatchAll([ + Workspaces.add({ + host: "testHost", + id: "1", + name: "testName", + port: "9867", + authType: "none" + }), + Spaces.setDetail("1", space) + ]) + store.dispatch(tabHistory.push(lakePath(space.id, "1"))) + zealot.stubStream("search", countByPathResp) +}) -// const submit = () => dispatch(histogramSearch()) +const submit = () => dispatch(histogramSearch()) -// test("zealot gets the request", async () => { -// await submit() -// const calls = zealot.calls("search") -// expect(calls.length).toBe(1) -// expect(calls[0].args).toEqual("* | every 12h count() by _path") -// }) +test("zealot gets the request", async () => { + await submit() + const calls = zealot.calls("search") + expect(calls.length).toBe(1) + expect(calls[0].args).toEqual("* | every 12h count() by _path") +}) -// test("the chart status updates", async () => { -// const promise = submit() -// expect(select(Chart.getStatus)).toBe("FETCHING") -// await promise -// expect(select(Chart.getStatus)).toBe("SUCCESS") -// }) +test("the chart status updates", async () => { + const promise = submit() + expect(select(Chart.getStatus)).toBe("FETCHING") + await promise + expect(select(Chart.getStatus)).toBe("SUCCESS") +}) -// test("registers historgram request then cleans it up", async (done) => { -// const promise = submit() -// expect(select(Handlers.get)["Histogram"]).toEqual( -// expect.objectContaining({type: "SEARCH"}) -// ) -// await promise -// // The promise only waits for the table, might be good to return two -// // promises so people can decide what they want to wait for. -// setTimeout(() => { -// expect(select(Handlers.get)["Histogram"]).toBe(undefined) -// done() -// }) -// }) +test("registers historgram request then cleans it up", async (done) => { + const promise = submit() + expect(select(Handlers.get)["Histogram"]).toEqual( + expect.objectContaining({type: "SEARCH"}) + ) + await promise + // The promise only waits for the table, might be good to return two + // promises so people can decide what they want to wait for. + setTimeout(() => { + expect(select(Handlers.get)["Histogram"]).toBe(undefined) + done() + }) +}) -// test("aborts previous histogram request", async () => { -// const abort = jest.fn() -// dispatch(Handlers.register("Histogram", {type: "SEARCH", abort})) -// await submit() -// expect(abort).toHaveBeenCalledTimes(1) -// }) +test("aborts previous histogram request", async () => { + const abort = jest.fn() + dispatch(Handlers.register("Histogram", {type: "SEARCH", abort})) + await submit() + expect(abort).toHaveBeenCalledTimes(1) +}) -// test("populates the chart", async () => { -// await submit() -// expect(select(Chart.getData)).toMatchSnapshot() -// }) -test("nothing", () => {}) +test("populates the chart", async () => { + await submit() + expect(select(Chart.getData)).toMatchSnapshot() +}) diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index d4f9758a2b..83e5715d1c 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -46,7 +46,7 @@ describe("zeek log when community_id is found", () => { expect(zealot.calls("search")).toHaveLength(before + 2) }) - test("executes uid first, then cid", async () => { + test.only("executes uid first, then cid", async () => { const {store, zealot} = setup await store.dispatch(fetchCorrelation(zeek)) const searches = zealot.calls("search") diff --git a/scripts/test/responses.js b/scripts/test/responses.js index dd93210310..7e888c6ad5 100644 --- a/scripts/test/responses.js +++ b/scripts/test/responses.js @@ -6,7 +6,7 @@ * * It will use the zqd bundled in zdeps. * - * Before running this, it would be good to: + * Before running this, run these commands once: * 1. npm install # to install the desired version of zqd * 2. npx rollup -c --silent # to bundle the current version of zealot * @@ -26,7 +26,7 @@ function saveResponse(input, output, query) { ) const out = fs.createWriteStream(output) deno.stdout.pipe(out) - + deno.stdout.pipe(process.stdout) return new Promise((resolve) => { deno.on("close", () => { resolve() diff --git a/src/js/searches/programs.test.ts b/src/js/searches/programs.test.ts index 425cd09a1f..02d97294c4 100644 --- a/src/js/searches/programs.test.ts +++ b/src/js/searches/programs.test.ts @@ -6,7 +6,7 @@ test("conn correlation", () => { const record = createRecord({ ts: new Date(1425568032.998178 * 1000), uid: "CbOjYpkXn9LfqV51c", - duration: new zed.Duration("0.70995"), + duration: new zed.Duration("0.70995s"), community_id: "1:h09VUfAoDYfBA0xGKuKCQ7nOxqU=" }) expect( diff --git a/test/responses/config.js b/test/responses/config.js index 6941573b24..da6d39f7b9 100644 --- a/test/responses/config.js +++ b/test/responses/config.js @@ -17,6 +17,11 @@ module.exports = { input: "sample.tsv", output: "count-by-path.response" }, + everyCountByPath: { + query: "every 1s count() by _path", + input: "sample.tsv", + output: "every-count-by-path.response" + }, sample: { query: "*", input: "sample.tsv", diff --git a/test/responses/correlation-uid-community-id.response b/test/responses/correlation-uid-community-id.response index 8dd01f21ea..795e665845 100644 --- a/test/responses/correlation-uid-community-id.response +++ b/test/responses/correlation-uid-community-id.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","16.907ms","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","16.907ms","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["209.216.230.240"],["35s"],"false"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961329,"ns":341415000},"update_time":{"sec":1618961329,"ns":342521000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619669702,"ns":490465000},"update_time":{"sec":1619669702,"ns":491148000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/correlation-uid.response b/test/responses/correlation-uid.response index 68dbc4a505..9f460aca6f 100644 --- a/test/responses/correlation-uid.response +++ b/test/responses/correlation-uid.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"community_id","type":{"kind":"primitive","name":"string"}}]}}],"values":["conn","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","16.907ms","38","54","SF",null,null,"0","Dd","1","66","1","82",null,"1:N7YGmWjwTmMKNhsZHBR618n3ReA="]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","16.907ms","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["209.216.230.240"],["35s"],"false"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961329,"ns":92713000},"update_time":{"sec":1618961329,"ns":93862000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619669702,"ns":236500000},"update_time":{"sec":1619669702,"ns":237221000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count-by-path.response b/test/responses/count-by-path.response index 76ed673f1e..018d8819e2 100644 --- a/test/responses/count-by-path.response +++ b/test/responses/count-by-path.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["dns","2"]},{"schema":"23","values":["weird","1"]},{"schema":"23","values":["stats","1"]},{"schema":"23","values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["weird","1"]},{"schema":"23","values":["stats","1"]},{"schema":"23","values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]},{"schema":"23","values":["dns","2"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961328,"ns":598653000},"update_time":{"sec":1618961328,"ns":600533000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619669701,"ns":473479000},"update_time":{"sec":1619669701,"ns":474646000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count.response b/test/responses/count.response index 4fae8e84e5..2a14c179a0 100644 --- a/test/responses/count.response +++ b/test/responses/count.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961328,"ns":357440000},"update_time":{"sec":1618961328,"ns":360568000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619669701,"ns":219273000},"update_time":{"sec":1619669701,"ns":221022000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/dns.response b/test/responses/dns.response index 65084e7e0d..38ab786755 100644 --- a/test/responses/dns.response +++ b/test/responses/dns.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]},{"schema":"28","values":["dns","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","0.017946","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["173.194.201.189"],["213"],"F"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","2020-02-25T16:03:13.978298Z","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","16.907ms","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["209.216.230.240"],["35s"],"false"]},{"schema":"28","values":["dns","2020-02-25T16:03:11.27555Z","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","17.946ms","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["173.194.201.189"],["3m33s"],"false"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961328,"ns":111091000},"update_time":{"sec":1618961328,"ns":113691000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619669700,"ns":960752000},"update_time":{"sec":1619669700,"ns":962439000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/every-count-by-path.response b/test/responses/every-count-by-path.response new file mode 100644 index 0000000000..8132a71a38 --- /dev/null +++ b/test/responses/every-count-by-path.response @@ -0,0 +1,17 @@ +{"type":"TaskStart","task_id":0} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["2020-02-25T16:03:17Z","capture_loss","1"]},{"schema":"23","values":["2020-02-25T16:03:15Z","conn","3"]},{"schema":"23","values":["2020-02-25T16:03:14Z","x509","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","ssl","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","files","4"]},{"schema":"23","values":["2020-02-25T16:03:13Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:13Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:12Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:10Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:09Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:08Z","conn","5"]},{"schema":"23","values":["2020-02-25T16:03:07Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:06Z","weird","1"]},{"schema":"23","values":["2020-02-25T16:03:06Z","conn","1"]}]} + + +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","values":["2020-02-25T16:03:05Z","stats","1"]},{"schema":"23","values":["2020-02-25T16:03:05Z","conn","1"]}]} + + +{"type":"SearchEnd","channel_id":0,"reason":"eof"} + + +{"type":"SearchStats","start_time":{"sec":1619669701,"ns":722861000},"update_time":{"sec":1619669701,"ns":724641000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} + + +{"type":"TaskEnd","task_id":0} + diff --git a/test/responses/no-community-id-in-conn.response b/test/responses/no-community-id-in-conn.response index 4a4af75912..65c6809579 100644 --- a/test/responses/no-community-id-in-conn.response +++ b/test/responses/no-community-id-in-conn.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"27","types":[{"kind":"typedef","name":"27","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","16.907ms","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","2020-02-25T16:03:13.978298Z","CbOjYpkXn9LfqV51c",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","16.907ms","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["209.216.230.240"],["35s"],"false"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961329,"ns":585595000},"update_time":{"sec":1618961329,"ns":586740000},"bytes_read":224,"bytes_matched":224,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619669702,"ns":741767000},"update_time":{"sec":1619669702,"ns":742575000},"bytes_read":224,"bytes_matched":224,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/only-alerts.response b/test/responses/only-alerts.response index 9666d4fb15..09e1cc8e82 100644 --- a/test/responses/only-alerts.response +++ b/test/responses/only-alerts.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"event_type","type":{"kind":"primitive","name":"bstring"}},{"name":"src_ip","type":{"kind":"primitive","name":"ip"}},{"name":"src_port","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"dest_ip","type":{"kind":"primitive","name":"ip"}},{"name":"dest_port","type":{"kind":"typename","name":"port"}},{"name":"vlan","type":{"kind":"array","type":{"kind":"primitive","name":"uint16"}}},{"name":"proto","type":{"kind":"primitive","name":"bstring"}},{"name":"app_proto","type":{"kind":"primitive","name":"bstring"}},{"name":"alert","type":{"kind":"record","fields":[{"name":"severity","type":{"kind":"primitive","name":"uint16"}},{"name":"signature","type":{"kind":"primitive","name":"bstring"}},{"name":"category","type":{"kind":"primitive","name":"bstring"}},{"name":"action","type":{"kind":"primitive","name":"bstring"}},{"name":"signature_id","type":{"kind":"primitive","name":"uint64"}},{"name":"gid","type":{"kind":"primitive","name":"uint64"}},{"name":"rev","type":{"kind":"primitive","name":"uint64"}},{"name":"metadata","type":{"kind":"record","fields":[{"name":"signature_severity","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"former_category","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"attack_target","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"deployment","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"affected_product","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"created_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"performance_impact","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"updated_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"malware_family","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"tag","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}}]}}]}},{"name":"flow_id","type":{"kind":"primitive","name":"uint64"}},{"name":"pcap_cnt","type":{"kind":"primitive","name":"uint64"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"tx_id","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_code","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_type","type":{"kind":"primitive","name":"uint64"}},{"name":"community_id","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668996500830","9025085","1428696616.318954","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668255955402","8015332","1428390396.903542","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394670027691253","7847662","1428286359.587171","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669830007389","7174571","1427922895.913599","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668388559068","6196400","1427802619.776149","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668479470212","5719123","1427509072.835646","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668951075328","5413510","1427319659.24276","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669628743811","5030650","1427100624.107701","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669695329403","4864841","1427003335.253496","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669771541024","4703729","1426906194.903837","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668624120943","2766977","1426102254.635397","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"event_type","type":{"kind":"primitive","name":"bstring"}},{"name":"src_ip","type":{"kind":"primitive","name":"ip"}},{"name":"src_port","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"dest_ip","type":{"kind":"primitive","name":"ip"}},{"name":"dest_port","type":{"kind":"typename","name":"port"}},{"name":"vlan","type":{"kind":"array","type":{"kind":"primitive","name":"uint16"}}},{"name":"proto","type":{"kind":"primitive","name":"bstring"}},{"name":"app_proto","type":{"kind":"primitive","name":"bstring"}},{"name":"alert","type":{"kind":"record","fields":[{"name":"severity","type":{"kind":"primitive","name":"uint16"}},{"name":"signature","type":{"kind":"primitive","name":"bstring"}},{"name":"category","type":{"kind":"primitive","name":"bstring"}},{"name":"action","type":{"kind":"primitive","name":"bstring"}},{"name":"signature_id","type":{"kind":"primitive","name":"uint64"}},{"name":"gid","type":{"kind":"primitive","name":"uint64"}},{"name":"rev","type":{"kind":"primitive","name":"uint64"}},{"name":"metadata","type":{"kind":"record","fields":[{"name":"signature_severity","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"former_category","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"attack_target","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"deployment","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"affected_product","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"created_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"performance_impact","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"updated_at","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"malware_family","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"tag","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}}]}}]}},{"name":"flow_id","type":{"kind":"primitive","name":"uint64"}},{"name":"pcap_cnt","type":{"kind":"primitive","name":"uint64"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"tx_id","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_code","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_type","type":{"kind":"primitive","name":"uint64"}},{"name":"community_id","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668996500830","9025085","2015-04-10T20:10:16.318954Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668255955402","8015332","2015-04-07T07:06:36.903542Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394670027691253","7847662","2015-04-06T02:12:39.587171Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669830007389","7174571","2015-04-01T21:14:55.913599Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668388559068","6196400","2015-03-31T11:50:19.776149Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668479470212","5719123","2015-03-28T02:17:52.835646Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668951075328","5413510","2015-03-25T21:40:59.24276Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669628743811","5030650","2015-03-23T08:50:24.107701Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669695329403","4864841","2015-03-22T05:48:55.253496Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394669771541024","4703729","2015-03-21T02:49:54.903837Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]},{"schema":"28","values":["alert","61.240.144.67","60000","192.168.0.2","80",null,"TCP","http",["3","ET SCAN NETWORK Incoming Masscan detected","Detection of a Network Scan","allowed","2017616","1","6",[null,null,null,null,null,["2013_10_18"],null,["2020_04_27"],null,null]],"1394668624120943","2766977","2015-03-11T19:30:54.635397Z","0",null,null,"1:+3I+tIoxlnE/NIV4M9Wh277sSxI="]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961329,"ns":835445000},"update_time":{"sec":1618961329,"ns":836198000},"bytes_read":2299,"bytes_matched":2299,"records_read":11,"records_matched":11} +{"type":"SearchStats","start_time":{"sec":1619669702,"ns":985238000},"update_time":{"sec":1619669702,"ns":985938000},"bytes_read":2299,"bytes_matched":2299,"records_read":11,"records_matched":11} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/sample.response b/test/responses/sample.response index 7f1d112212..20e6f2b2b1 100644 --- a/test/responses/sample.response +++ b/test/responses/sample.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"ts_delta","type":{"kind":"primitive","name":"duration"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"gaps","type":{"kind":"primitive","name":"uint64"}},{"name":"acks","type":{"kind":"primitive","name":"uint64"}},{"name":"percent_lost","type":{"kind":"primitive","name":"float64"}}]}}],"values":["capture_loss","1582646597.838527","11.854892","zeek","0","6","0"]},{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","1582646595.986756","Cl4Rbf2czFCAu7bc23",["192.168.1.110","57540","172.217.1.138","443"],"tcp",null,"0.060027","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","1582646595.784014","C2J5XS1KgpVaWCJpRg",["192.168.1.110","57572","172.217.6.138","443"],"tcp",null,"0.053917","63","0","SF",null,null,"0","DFafA","4","271","3","156",null]},{"schema":"28","values":["conn","1582646595.481957","C1dB0oBEPvJUas0dg",["192.168.1.179","51524","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"tx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"rx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"conn_uids","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"source","type":{"kind":"primitive","name":"bstring"}},{"name":"depth","type":{"kind":"primitive","name":"uint64"}},{"name":"analyzers","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"mime_type","type":{"kind":"primitive","name":"bstring"}},{"name":"filename","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"is_orig","type":{"kind":"primitive","name":"bool"}},{"name":"seen_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"total_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"missing_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"overflow_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"timedout","type":{"kind":"primitive","name":"bool"}},{"name":"parent_fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"md5","type":{"kind":"primitive","name":"bstring"}},{"name":"sha1","type":{"kind":"primitive","name":"bstring"}},{"name":"sha256","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted_cutoff","type":{"kind":"primitive","name":"bool"}},{"name":"extracted_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["files","1582646594.050188","FMa5RD1f8J80UyJKQe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1"],"application/ocsp-response",null,"0",null,"F","471",null,"0","0","F",null,"fca4341c673e74dc3e330e1640e9b7d4","570304f7776ed6ca29464404554fb927395328e6",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","FXkLzR25cwYgeI5yI7",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-user-cert",null,"0",null,"F","1711",null,"0","0","F",null,"d16bf2ff6647cb8bccce2594e9237e50","987050ffb905cad3a79a8596c2120db97c03a165",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","Frbo5s1CRsV02JUoSe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0",null,"F","1176",null,"0","0","F",null,"345eff15b7a49add451b65a7f4bdc6ae","1fb86b1168ec743154062e8c9cc5b171a4b7ccb4",null,null,null,null]},{"schema":"30","values":["files","1582646594.050187","F1VUER3CattO8PiHWc",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0",null,"F","947",null,"0","0","F",null,"79e4a9840d7d3a96d7c04fe2434c892e","a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436",null,null,null,null]},{"schema":"36","types":[{"kind":"typedef","name":"36","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"id","type":{"kind":"primitive","name":"bstring"}},{"name":"certificate","type":{"kind":"record","fields":[{"name":"version","type":{"kind":"primitive","name":"uint64"}},{"name":"serial","type":{"kind":"primitive","name":"bstring"}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"not_valid_before","type":{"kind":"primitive","name":"time"}},{"name":"not_valid_after","type":{"kind":"primitive","name":"time"}},{"name":"key_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"sig_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"key_type","type":{"kind":"primitive","name":"bstring"}},{"name":"key_length","type":{"kind":"primitive","name":"uint64"}},{"name":"exponent","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}}]}},{"name":"san","type":{"kind":"record","fields":[{"name":"dns","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"uri","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"email","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"ip","type":{"kind":"array","type":{"kind":"primitive","name":"ip"}}}]}},{"name":"basic_constraints","type":{"kind":"record","fields":[{"name":"ca","type":{"kind":"primitive","name":"bool"}},{"name":"path_len","type":{"kind":"primitive","name":"uint64"}}]}}]}}],"values":["x509","1582646594.050187","FXkLzR25cwYgeI5yI7",["3","074FE902C6B7D619884E1CA1854D9178","CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","1562569200","1631300400","rsaEncryption","sha256WithRSAEncryption","rsa","2048","65537",null],[["news.ycombinator.com"],null,null,null],["F",null]]},{"schema":"37","types":[{"kind":"typedef","name":"37","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"version","type":{"kind":"primitive","name":"bstring"}},{"name":"cipher","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}},{"name":"server_name","type":{"kind":"primitive","name":"bstring"}},{"name":"resumed","type":{"kind":"primitive","name":"bool"}},{"name":"last_alert","type":{"kind":"primitive","name":"bstring"}},{"name":"next_protocol","type":{"kind":"primitive","name":"bstring"}},{"name":"established","type":{"kind":"primitive","name":"bool"}},{"name":"cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"client_cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"client_subject","type":{"kind":"primitive","name":"bstring"}},{"name":"client_issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"validation_status","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["ssl","1582646594.021637","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"TLSv12","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","secp256r1","news.ycombinator.com","F",null,"http/1.1","T",["FXkLzR25cwYgeI5yI7","Frbo5s1CRsV02JUoSe","F1VUER3CattO8PiHWc"],[],"CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US",null,null,"ok"]},{"schema":"28","values":["conn","1582646593.996366","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"tcp","ssl","0.104626","1088","6425","S1",null,null,"0","ShADda","10","1620","8","6849",null]},{"schema":"28","values":["conn","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","0.016907","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"39","types":[{"kind":"typedef","name":"39","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","1582646593.978298","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","0.016907","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["209.216.230.240"],["35"],"F"]},{"schema":"28","values":["conn","1582646592.860963","CK9UVX1h8kJ6Wrmju6",["192.168.1.110","57635","17.125.252.5","443"],"tcp",null,"0.593878","699","305","OTH",null,null,"0","DadA","5","899","5","505",null]},{"schema":"28","values":["conn","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","dns","0.017946","46","62","SF",null,null,"0","Dd","1","74","1","90",null]},{"schema":"39","values":["dns","1582646591.27555","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","0.017946","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","F","F","T","T","0",["173.194.201.189"],["213"],"F"]},{"schema":"28","values":["conn","1582646590.938093","CFH9bc1tw1tM6UNPjl",["192.168.1.110","55351","18.205.93.211","443"],"tcp",null,"0.088679","215","193","OTH",null,null,"0","DadA","2","319","2","297",null]},{"schema":"28","values":["conn","1582646590.264455","C7sBkQ1LVS7gbqDtzk",["192.168.1.110","57332","64.233.179.189","443"],"tcp",null,"1.079187","363","382","OTH",null,null,"0","^dADa","7","727","7","746",null]},{"schema":"28","values":["conn","1582646589.440467","CcIQuP2iS3J8Gl3td7",["192.168.1.179","47783","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"28","values":["conn","1582646588.807682","CkqJmu2oNuiB3zZ0ta",["192.168.1.110","55354","52.37.243.173","443"],"tcp",null,"0.761817","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","1582646588.4727","C6aipo1N64FtzIpSqc",["192.168.1.110","57487","192.30.253.125","443"],"tcp",null,"0.077944","28","24","OTH",null,null,"0","^dADa","2","132","2","128",null]},{"schema":"28","values":["conn","1582646588.449312","CpH9k34gcifnZmAH3h",["192.168.1.110","57326","173.194.201.189","443"],"tcp",null,"1.152367","403","380","OTH",null,null,"0","^dADa","7","767","7","744",null]},{"schema":"28","values":["conn","1582646588.334796","C1VePj3MBeuyPB31wi",["192.168.1.110","55346","52.37.243.173","443"],"tcp",null,"1.230297","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","1582646588.210507","CLuXz9mjBQJrNe822",["192.168.1.110","55344","52.37.243.173","443"],"tcp",null,"0.034261","56","56","OTH",null,null,"0","DadA","2","160","2","160",null]},{"schema":"28","values":["conn","1582646587.715839","Cy3dlK3PhYKCwG227k",["192.168.1.110","55747","13.52.5.22","443"],"tcp",null,"0.017643","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","1582646587.715728","CFwwj51CZ17Px2g2rl",["192.168.1.110","55635","18.246.31.137","443"],"tcp",null,"0.040702","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"40","types":[{"kind":"typedef","name":"40","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"name","type":{"kind":"primitive","name":"bstring"}},{"name":"addl","type":{"kind":"primitive","name":"bstring"}},{"name":"notice","type":{"kind":"primitive","name":"bool"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["weird","1582646586.880512",null,[null,null,null,null],"unknown_protocol","2","F","zeek"]},{"schema":"28","values":["conn","1582646586.154443","Cki8nOlkkWyYzyqx2",["192.168.1.110","57591","172.217.9.142","443"],"tcp",null,"0.216412","2511","3263","OTH",null,null,"0","DadA","11","3083","9","3731",null]},{"schema":"41","types":[{"kind":"typedef","name":"41","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"mem","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"bytes_recv","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_dropped","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_link","type":{"kind":"primitive","name":"uint64"}},{"name":"pkt_lag","type":{"kind":"primitive","name":"duration"}},{"name":"events_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"events_queued","type":{"kind":"primitive","name":"uint64"}},{"name":"active_tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"timers","type":{"kind":"primitive","name":"uint64"}},{"name":"active_timers","type":{"kind":"primitive","name":"uint64"}},{"name":"files","type":{"kind":"primitive","name":"uint64"}},{"name":"active_files","type":{"kind":"primitive","name":"uint64"}},{"name":"dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"active_dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_tcp_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_file_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_frag_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_unknown_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["stats","1582646585.983635","zeek","71","1","1001",null,null,null,"403","12","1","0","0","1","0","0","38","34","0","0","0","0","1008","0","0","0"]},{"schema":"28","values":["conn","1582646585.983635","CMa2ZP3LnKsjzJCDy7",["192.168.1.110","56625","172.217.9.142","443"],"tcp",null,"11.317514","2801","1306","OTH",null,null,"0","DadA","16","3633","16","2138",null]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"ts_delta","type":{"kind":"primitive","name":"duration"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"gaps","type":{"kind":"primitive","name":"uint64"}},{"name":"acks","type":{"kind":"primitive","name":"uint64"}},{"name":"percent_lost","type":{"kind":"primitive","name":"float64"}}]}}],"values":["capture_loss","2020-02-25T16:03:17.838527Z","11.854892s","zeek","0","6","0."]},{"schema":"28","types":[{"kind":"typedef","name":"28","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typedef","name":"port","type":{"kind":"primitive","name":"uint16"}}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typedef","name":"zenum","type":{"kind":"primitive","name":"string"}}},{"name":"service","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"orig_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"conn_state","type":{"kind":"primitive","name":"bstring"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"local_resp","type":{"kind":"primitive","name":"bool"}},{"name":"missed_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"history","type":{"kind":"primitive","name":"bstring"}},{"name":"orig_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"orig_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_pkts","type":{"kind":"primitive","name":"uint64"}},{"name":"resp_ip_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"tunnel_parents","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}}]}}],"values":["conn","2020-02-25T16:03:15.986756Z","Cl4Rbf2czFCAu7bc23",["192.168.1.110","57540","172.217.1.138","443"],"tcp",null,"60.027ms","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","2020-02-25T16:03:15.784014Z","C2J5XS1KgpVaWCJpRg",["192.168.1.110","57572","172.217.6.138","443"],"tcp",null,"53.917ms","63","0","SF",null,null,"0","DFafA","4","271","3","156",null]},{"schema":"28","values":["conn","2020-02-25T16:03:15.481957Z","C1dB0oBEPvJUas0dg",["192.168.1.179","51524","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"30","types":[{"kind":"typedef","name":"30","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"tx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"rx_hosts","type":{"kind":"set","type":{"kind":"primitive","name":"ip"}}},{"name":"conn_uids","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"source","type":{"kind":"primitive","name":"bstring"}},{"name":"depth","type":{"kind":"primitive","name":"uint64"}},{"name":"analyzers","type":{"kind":"set","type":{"kind":"primitive","name":"bstring"}}},{"name":"mime_type","type":{"kind":"primitive","name":"bstring"}},{"name":"filename","type":{"kind":"primitive","name":"bstring"}},{"name":"duration","type":{"kind":"primitive","name":"duration"}},{"name":"local_orig","type":{"kind":"primitive","name":"bool"}},{"name":"is_orig","type":{"kind":"primitive","name":"bool"}},{"name":"seen_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"total_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"missing_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"overflow_bytes","type":{"kind":"primitive","name":"uint64"}},{"name":"timedout","type":{"kind":"primitive","name":"bool"}},{"name":"parent_fuid","type":{"kind":"primitive","name":"bstring"}},{"name":"md5","type":{"kind":"primitive","name":"bstring"}},{"name":"sha1","type":{"kind":"primitive","name":"bstring"}},{"name":"sha256","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted","type":{"kind":"primitive","name":"bstring"}},{"name":"extracted_cutoff","type":{"kind":"primitive","name":"bool"}},{"name":"extracted_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["files","2020-02-25T16:03:14.050188Z","FMa5RD1f8J80UyJKQe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1"],"application/ocsp-response",null,"0s",null,"false","471",null,"0","0","false",null,"fca4341c673e74dc3e330e1640e9b7d4","570304f7776ed6ca29464404554fb927395328e6",null,null,null,null]},{"schema":"30","values":["files","2020-02-25T16:03:14.050187Z","FXkLzR25cwYgeI5yI7",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-user-cert",null,"0s",null,"false","1711",null,"0","0","false",null,"d16bf2ff6647cb8bccce2594e9237e50","987050ffb905cad3a79a8596c2120db97c03a165",null,null,null,null]},{"schema":"30","values":["files","2020-02-25T16:03:14.050187Z","Frbo5s1CRsV02JUoSe",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0s",null,"false","1176",null,"0","0","false",null,"345eff15b7a49add451b65a7f4bdc6ae","1fb86b1168ec743154062e8c9cc5b171a4b7ccb4",null,null,null,null]},{"schema":"30","values":["files","2020-02-25T16:03:14.050187Z","F1VUER3CattO8PiHWc",["209.216.230.240"],["192.168.1.110"],["CeYm4A4XtVPVfF9wo6"],"SSL","0",["MD5","SHA1","X509"],"application/x-x509-ca-cert",null,"0s",null,"false","947",null,"0","0","false",null,"79e4a9840d7d3a96d7c04fe2434c892e","a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436",null,null,null,null]},{"schema":"36","types":[{"kind":"typedef","name":"36","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"id","type":{"kind":"primitive","name":"bstring"}},{"name":"certificate","type":{"kind":"record","fields":[{"name":"version","type":{"kind":"primitive","name":"uint64"}},{"name":"serial","type":{"kind":"primitive","name":"bstring"}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"not_valid_before","type":{"kind":"primitive","name":"time"}},{"name":"not_valid_after","type":{"kind":"primitive","name":"time"}},{"name":"key_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"sig_alg","type":{"kind":"primitive","name":"bstring"}},{"name":"key_type","type":{"kind":"primitive","name":"bstring"}},{"name":"key_length","type":{"kind":"primitive","name":"uint64"}},{"name":"exponent","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}}]}},{"name":"san","type":{"kind":"record","fields":[{"name":"dns","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"uri","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"email","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"ip","type":{"kind":"array","type":{"kind":"primitive","name":"ip"}}}]}},{"name":"basic_constraints","type":{"kind":"record","fields":[{"name":"ca","type":{"kind":"primitive","name":"bool"}},{"name":"path_len","type":{"kind":"primitive","name":"uint64"}}]}}]}}],"values":["x509","2020-02-25T16:03:14.050187Z","FXkLzR25cwYgeI5yI7",["3","074FE902C6B7D619884E1CA1854D9178","CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US","2019-07-08T07:00:00Z","2021-09-10T19:00:00Z","rsaEncryption","sha256WithRSAEncryption","rsa","2048","65537",null],[["news.ycombinator.com"],null,null,null],["false",null]]},{"schema":"37","types":[{"kind":"typedef","name":"37","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"version","type":{"kind":"primitive","name":"bstring"}},{"name":"cipher","type":{"kind":"primitive","name":"bstring"}},{"name":"curve","type":{"kind":"primitive","name":"bstring"}},{"name":"server_name","type":{"kind":"primitive","name":"bstring"}},{"name":"resumed","type":{"kind":"primitive","name":"bool"}},{"name":"last_alert","type":{"kind":"primitive","name":"bstring"}},{"name":"next_protocol","type":{"kind":"primitive","name":"bstring"}},{"name":"established","type":{"kind":"primitive","name":"bool"}},{"name":"cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"client_cert_chain_fuids","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"subject","type":{"kind":"primitive","name":"bstring"}},{"name":"issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"client_subject","type":{"kind":"primitive","name":"bstring"}},{"name":"client_issuer","type":{"kind":"primitive","name":"bstring"}},{"name":"validation_status","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["ssl","2020-02-25T16:03:14.021637Z","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"TLSv12","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","secp256r1","news.ycombinator.com","false",null,"http/1.1","true",["FXkLzR25cwYgeI5yI7","Frbo5s1CRsV02JUoSe","F1VUER3CattO8PiHWc"],[],"CN=news.ycombinator.com,O=Y Combinator\\, Inc.,L=San Francisco,ST=California,C=US","CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US",null,null,"ok"]},{"schema":"28","values":["conn","2020-02-25T16:03:13.996366Z","CeYm4A4XtVPVfF9wo6",["192.168.1.110","57640","209.216.230.240","443"],"tcp","ssl","104.626ms","1088","6425","S1",null,null,"0","ShADda","10","1620","8","6849",null]},{"schema":"28","values":["conn","2020-02-25T16:03:13.978298Z","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","dns","16.907ms","38","54","SF",null,null,"0","Dd","1","66","1","82",null]},{"schema":"39","types":[{"kind":"typedef","name":"39","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"proto","type":{"kind":"typename","name":"zenum"}},{"name":"trans_id","type":{"kind":"primitive","name":"uint64"}},{"name":"rtt","type":{"kind":"primitive","name":"duration"}},{"name":"query","type":{"kind":"primitive","name":"bstring"}},{"name":"qclass","type":{"kind":"primitive","name":"uint64"}},{"name":"qclass_name","type":{"kind":"primitive","name":"bstring"}},{"name":"qtype","type":{"kind":"primitive","name":"uint64"}},{"name":"qtype_name","type":{"kind":"primitive","name":"bstring"}},{"name":"rcode","type":{"kind":"primitive","name":"uint64"}},{"name":"rcode_name","type":{"kind":"primitive","name":"bstring"}},{"name":"AA","type":{"kind":"primitive","name":"bool"}},{"name":"TC","type":{"kind":"primitive","name":"bool"}},{"name":"RD","type":{"kind":"primitive","name":"bool"}},{"name":"RA","type":{"kind":"primitive","name":"bool"}},{"name":"Z","type":{"kind":"primitive","name":"uint64"}},{"name":"answers","type":{"kind":"array","type":{"kind":"primitive","name":"bstring"}}},{"name":"TTLs","type":{"kind":"array","type":{"kind":"primitive","name":"duration"}}},{"name":"rejected","type":{"kind":"primitive","name":"bool"}}]}}],"values":["dns","2020-02-25T16:03:13.978298Z","CmOKuI3h5QDmQBsGDf",["192.168.1.110","51848","192.168.1.254","53"],"udp","47856","16.907ms","news.ycombinator.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["209.216.230.240"],["35s"],"false"]},{"schema":"28","values":["conn","2020-02-25T16:03:12.860963Z","CK9UVX1h8kJ6Wrmju6",["192.168.1.110","57635","17.125.252.5","443"],"tcp",null,"593.878ms","699","305","OTH",null,null,"0","DadA","5","899","5","505",null]},{"schema":"28","values":["conn","2020-02-25T16:03:11.27555Z","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","dns","17.946ms","46","62","SF",null,null,"0","Dd","1","74","1","90",null]},{"schema":"39","values":["dns","2020-02-25T16:03:11.27555Z","CIOfElOG3g9JicgQ5",["192.168.1.110","54375","192.168.1.254","53"],"udp","28084","17.946ms","11.client-channel.google.com","1","C_INTERNET","1","A","0","NOERROR","false","false","true","true","0",["173.194.201.189"],["3m33s"],"false"]},{"schema":"28","values":["conn","2020-02-25T16:03:10.938093Z","CFH9bc1tw1tM6UNPjl",["192.168.1.110","55351","18.205.93.211","443"],"tcp",null,"88.679ms","215","193","OTH",null,null,"0","DadA","2","319","2","297",null]},{"schema":"28","values":["conn","2020-02-25T16:03:10.264455Z","C7sBkQ1LVS7gbqDtzk",["192.168.1.110","57332","64.233.179.189","443"],"tcp",null,"1.079187s","363","382","OTH",null,null,"0","^dADa","7","727","7","746",null]},{"schema":"28","values":["conn","2020-02-25T16:03:09.440467Z","CcIQuP2iS3J8Gl3td7",["192.168.1.179","47783","192.168.1.255","15600"],"udp",null,null,null,null,"S0",null,null,"0","D","1","63","0","0",null]},{"schema":"28","values":["conn","2020-02-25T16:03:08.807682Z","CkqJmu2oNuiB3zZ0ta",["192.168.1.110","55354","52.37.243.173","443"],"tcp",null,"761.817ms","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","2020-02-25T16:03:08.4727Z","C6aipo1N64FtzIpSqc",["192.168.1.110","57487","192.30.253.125","443"],"tcp",null,"77.944ms","28","24","OTH",null,null,"0","^dADa","2","132","2","128",null]},{"schema":"28","values":["conn","2020-02-25T16:03:08.449312Z","CpH9k34gcifnZmAH3h",["192.168.1.110","57326","173.194.201.189","443"],"tcp",null,"1.152367s","403","380","OTH",null,null,"0","^dADa","7","767","7","744",null]},{"schema":"28","values":["conn","2020-02-25T16:03:08.334796Z","C1VePj3MBeuyPB31wi",["192.168.1.110","55346","52.37.243.173","443"],"tcp",null,"1.230297s","114","56","OTH",null,null,"0","DdAa","3","270","2","160",null]},{"schema":"28","values":["conn","2020-02-25T16:03:08.210507Z","CLuXz9mjBQJrNe822",["192.168.1.110","55344","52.37.243.173","443"],"tcp",null,"34.261ms","56","56","OTH",null,null,"0","DadA","2","160","2","160",null]},{"schema":"28","values":["conn","2020-02-25T16:03:07.715839Z","Cy3dlK3PhYKCwG227k",["192.168.1.110","55747","13.52.5.22","443"],"tcp",null,"17.643ms","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"28","values":["conn","2020-02-25T16:03:07.715728Z","CFwwj51CZ17Px2g2rl",["192.168.1.110","55635","18.246.31.137","443"],"tcp",null,"40.702ms","39","39","OTH",null,null,"0","DdA","2","143","1","91",null]},{"schema":"40","types":[{"kind":"typedef","name":"40","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"uid","type":{"kind":"primitive","name":"bstring"}},{"name":"id","type":{"kind":"record","fields":[{"name":"orig_h","type":{"kind":"primitive","name":"ip"}},{"name":"orig_p","type":{"kind":"typename","name":"port"}},{"name":"resp_h","type":{"kind":"primitive","name":"ip"}},{"name":"resp_p","type":{"kind":"typename","name":"port"}}]}},{"name":"name","type":{"kind":"primitive","name":"bstring"}},{"name":"addl","type":{"kind":"primitive","name":"bstring"}},{"name":"notice","type":{"kind":"primitive","name":"bool"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}}]}}],"values":["weird","2020-02-25T16:03:06.880512Z",null,[null,null,null,null],"unknown_protocol","2","false","zeek"]},{"schema":"28","values":["conn","2020-02-25T16:03:06.154443Z","Cki8nOlkkWyYzyqx2",["192.168.1.110","57591","172.217.9.142","443"],"tcp",null,"216.412ms","2511","3263","OTH",null,null,"0","DadA","11","3083","9","3731",null]},{"schema":"41","types":[{"kind":"typedef","name":"41","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"peer","type":{"kind":"primitive","name":"bstring"}},{"name":"mem","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"bytes_recv","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_dropped","type":{"kind":"primitive","name":"uint64"}},{"name":"pkts_link","type":{"kind":"primitive","name":"uint64"}},{"name":"pkt_lag","type":{"kind":"primitive","name":"duration"}},{"name":"events_proc","type":{"kind":"primitive","name":"uint64"}},{"name":"events_queued","type":{"kind":"primitive","name":"uint64"}},{"name":"active_tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"active_icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"tcp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"udp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"icmp_conns","type":{"kind":"primitive","name":"uint64"}},{"name":"timers","type":{"kind":"primitive","name":"uint64"}},{"name":"active_timers","type":{"kind":"primitive","name":"uint64"}},{"name":"files","type":{"kind":"primitive","name":"uint64"}},{"name":"active_files","type":{"kind":"primitive","name":"uint64"}},{"name":"dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"active_dns_requests","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_tcp_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_file_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_frag_size","type":{"kind":"primitive","name":"uint64"}},{"name":"reassem_unknown_size","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["stats","2020-02-25T16:03:05.983635Z","zeek","71","1","1001",null,null,null,"403","12","1","0","0","1","0","0","38","34","0","0","0","0","1008","0","0","0"]},{"schema":"28","values":["conn","2020-02-25T16:03:05.983635Z","CMa2ZP3LnKsjzJCDy7",["192.168.1.110","56625","172.217.9.142","443"],"tcp",null,"11.317514s","2801","1306","OTH",null,null,"0","DadA","16","3633","16","2138",null]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1618961328,"ns":846342000},"update_time":{"sec":1618961328,"ns":848980000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619669701,"ns":972898000},"update_time":{"sec":1619669701,"ns":975503000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/zealot/zed/context.ts b/zealot/zed/context.ts index 71bde5d1c9..acb60454fb 100644 --- a/zealot/zed/context.ts +++ b/zealot/zed/context.ts @@ -1,4 +1,3 @@ -import {isEmpty} from "lodash" import {TypeAlias} from "./types/type-alias" import {TypeArray} from "./types/type-array" import {TypeMap} from "./types/type-map" @@ -175,7 +174,7 @@ export class ZedContext { this.encodeTypeDef(typeName, typedefs, types) }) - return isEmpty(types) ? {schema, values} : {schema, types, values} + return types.length === 0 ? {schema, values} : {schema, types, values} } encodeTypeDef(name, typedefs, types) { diff --git a/zealot/zed/types/type-array.ts b/zealot/zed/types/type-array.ts index a6936a1b84..168d0a8bc9 100644 --- a/zealot/zed/types/type-array.ts +++ b/zealot/zed/types/type-array.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-array-constructor */ -import {isNull} from "lodash" +import {isNull} from "../utils" import {ZedContext} from "../context" import {Array} from "../values/array" import {ContainerTypeInterface, ZedType} from "./types" diff --git a/zealot/zed/types/type-map.ts b/zealot/zed/types/type-map.ts index b806ef36c3..b3a2b94c52 100644 --- a/zealot/zed/types/type-map.ts +++ b/zealot/zed/types/type-map.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {Value} from "zealot/zjson" import {ZedContext} from "../context" import {ZedMap} from "../values/map" diff --git a/zealot/zed/types/type-record.ts b/zealot/zed/types/type-record.ts index 035182faa9..0d57942e44 100644 --- a/zealot/zed/types/type-record.ts +++ b/zealot/zed/types/type-record.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {RecordType, Value} from "../../zjson" import {ZedContext} from "../context" import {typeId} from "../utils" diff --git a/zealot/zed/types/type-set.ts b/zealot/zed/types/type-set.ts index 0ea7e2521f..9349b9a840 100644 --- a/zealot/zed/types/type-set.ts +++ b/zealot/zed/types/type-set.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {SetValue, Value} from "zealot/zjson" import {ZedContext} from "../context" import {typeId} from "../utils" diff --git a/zealot/zed/types/type-type.ts b/zealot/zed/types/type-type.ts index d918ace9a4..290995f1ec 100644 --- a/zealot/zed/types/type-type.ts +++ b/zealot/zed/types/type-type.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeValue} from "../values/type" import {PrimitiveType} from "../../zjson" import {PrimitiveTypeInterface} from "./types" diff --git a/zealot/zed/types/type-union.ts b/zealot/zed/types/type-union.ts index 1198663370..35c87fe96b 100644 --- a/zealot/zed/types/type-union.ts +++ b/zealot/zed/types/type-union.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {Value} from "zealot/zjson" import {Union} from "../values/union" import {TypeNull} from "./type-null" diff --git a/zealot/zed/utils.ts b/zealot/zed/utils.ts index f6a7dde2e9..e60710b87b 100644 --- a/zealot/zed/utils.ts +++ b/zealot/zed/utils.ts @@ -53,3 +53,7 @@ export function trueType(start: ZedType): T { } return t as T } + +export function isNull(value): value is null { + return value === null +} diff --git a/zealot/zed/values/array.ts b/zealot/zed/values/array.ts index 2f83fade59..a7818007f9 100644 --- a/zealot/zed/values/array.ts +++ b/zealot/zed/values/array.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeArray} from "../types/type-array" import {ZedValue, ZedValueInterface} from "./types" diff --git a/zealot/zed/values/duration.ts b/zealot/zed/values/duration.ts index 6102c287cd..33c5c0d1c2 100644 --- a/zealot/zed/values/duration.ts +++ b/zealot/zed/values/duration.ts @@ -1,12 +1,22 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeDuration} from "../types/type-duration" import {Primitive} from "./primitive" export class Duration extends Primitive { type = TypeDuration + parse() { + let unit = this.value!.replace(/[\d.]*/, "") + if (unit.trim().length === 0) unit = "s" + const number = parseFloat(this.value!.replace(unit, "")) + return {unit, number} + } + asSeconds() { if (isNull(this.value)) return null - return parseFloat(this.value) + const {unit, number} = this.parse() + if (unit === "ms") return number / 1000 + if (unit === "s") return number + throw new Error("Implement duration unit: " + unit) } } diff --git a/zealot/zed/values/float64.ts b/zealot/zed/values/float64.ts index c76c84826c..55fd19bbb8 100644 --- a/zealot/zed/values/float64.ts +++ b/zealot/zed/values/float64.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeFloat64} from "../types/type-float64" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/int16.ts b/zealot/zed/values/int16.ts index 4f3110cc19..dd4d41c2f7 100644 --- a/zealot/zed/values/int16.ts +++ b/zealot/zed/values/int16.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeInt16} from "../types/type-int16" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/int32.ts b/zealot/zed/values/int32.ts index 38fa99464a..de07e52ebd 100644 --- a/zealot/zed/values/int32.ts +++ b/zealot/zed/values/int32.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeInt32} from "../types/type-int32" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/int64.ts b/zealot/zed/values/int64.ts index 9ad5f176bf..537648d9c1 100644 --- a/zealot/zed/values/int64.ts +++ b/zealot/zed/values/int64.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeInt64} from "../types/type-int64" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/int8.ts b/zealot/zed/values/int8.ts index 9db6b26120..538e190eed 100644 --- a/zealot/zed/values/int8.ts +++ b/zealot/zed/values/int8.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeInt8} from "../types/type-int8" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/map.ts b/zealot/zed/values/map.ts index 4c5748827a..eab1d219e1 100644 --- a/zealot/zed/values/map.ts +++ b/zealot/zed/values/map.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeMap} from "../types/type-map" import {ZedValue, ZedValueInterface} from "./types" diff --git a/zealot/zed/values/primitive.ts b/zealot/zed/values/primitive.ts index c899246d52..719937a25c 100644 --- a/zealot/zed/values/primitive.ts +++ b/zealot/zed/values/primitive.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {ZedType} from "../types/types" import {ZedValueInterface} from "./types" diff --git a/zealot/zed/values/record.ts b/zealot/zed/values/record.ts index ba3e77cca3..55d4ad37f4 100644 --- a/zealot/zed/values/record.ts +++ b/zealot/zed/values/record.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeAlias} from "../types/type-alias" import {TypeField, TypeRecord} from "../types/type-record" import {Field} from "./field" diff --git a/zealot/zed/values/set.ts b/zealot/zed/values/set.ts index 2666085103..ef986aaabb 100644 --- a/zealot/zed/values/set.ts +++ b/zealot/zed/values/set.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeSet} from "../types/type-set" import {ZedValue, ZedValueInterface} from "./types" diff --git a/zealot/zed/values/time.ts b/zealot/zed/values/time.ts index 36b52c207a..ffc1a15bea 100644 --- a/zealot/zed/values/time.ts +++ b/zealot/zed/values/time.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeTime} from "../types/type-time" import {Primitive} from "./primitive" @@ -7,6 +7,17 @@ export class Time extends Primitive { toDate() { if (isNull(this.value)) return null - return new Date(+this.value * 1000) + // Need to parse dates special now... + let d + if (this.value.match(/[\d.]*/)) { + // Epoch Seconds + d = new Date(+this.value * 1000) + } else { + // ISO Date String + d = new Date(Date.parse(this.value)) + } + if (isNaN(d)) { + throw new Error(`Unkown Time Value: ${this.value}`) + } } } diff --git a/zealot/zed/values/type.ts b/zealot/zed/values/type.ts index 0778ddf3f4..69887b215c 100644 --- a/zealot/zed/values/type.ts +++ b/zealot/zed/values/type.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeType} from "../types/type-type" import {ZedType} from "../types/types" import {typeId} from "../utils" diff --git a/zealot/zed/values/uint16.ts b/zealot/zed/values/uint16.ts index f5f995a694..8e2ebf3b91 100644 --- a/zealot/zed/values/uint16.ts +++ b/zealot/zed/values/uint16.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeUint16} from "../types/type-uint16" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/uint32.ts b/zealot/zed/values/uint32.ts index 3629ae0857..60365987ba 100644 --- a/zealot/zed/values/uint32.ts +++ b/zealot/zed/values/uint32.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeUint32} from "../types/type-uint32" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/uint64.ts b/zealot/zed/values/uint64.ts index f6611edd2e..45b9ad68c4 100644 --- a/zealot/zed/values/uint64.ts +++ b/zealot/zed/values/uint64.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeUint64} from "../types/type-uint64" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/uint8.ts b/zealot/zed/values/uint8.ts index 22b95886f9..f8017bd9c2 100644 --- a/zealot/zed/values/uint8.ts +++ b/zealot/zed/values/uint8.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeUint8} from "../types/type-uint8" import {Primitive} from "./primitive" diff --git a/zealot/zed/values/union.ts b/zealot/zed/values/union.ts index 4615207316..19bdd83dc4 100644 --- a/zealot/zed/values/union.ts +++ b/zealot/zed/values/union.ts @@ -1,4 +1,4 @@ -import {isNull} from "lodash" +import {isNull} from "../utils" import {TypeUnion} from "../types/type-union" import {ZedType} from "../types/types" import {ZedValue, ZedValueInterface} from "./types" From 03783097c1b2e60309a1db4706050d98c3236a1d Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 29 Apr 2021 09:46:48 -0700 Subject: [PATCH 15/41] TS Errors Fixed --- package-lock.json | 135 ------------------------------ plugins/brimcap/brimcap-plugin.ts | 68 +++++---------- src/js/flows/downloadPcap.ts | 10 --- src/js/state/Packets/flows.ts | 71 ---------------- zealot/enhancers/zngToZeek.ts | 74 ---------------- 5 files changed, 20 insertions(+), 338 deletions(-) delete mode 100644 src/js/flows/downloadPcap.ts delete mode 100644 src/js/state/Packets/flows.ts delete mode 100644 zealot/enhancers/zngToZeek.ts diff --git a/package-lock.json b/package-lock.json index 0d8c88eb92..1159813f38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25385,141 +25385,6 @@ "version": "git+https://github.com/brimdata/zed.git#3c17abe55921b3b63023f3cfd361899793b1cf85", "from": "git+https://github.com/brimdata/zed.git#3c17abe55921b3b63023f3cfd361899793b1cf85" }, - "zip-folder": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/zip-folder/-/zip-folder-1.0.0.tgz", - "integrity": "sha1-cKd0T9F4mi/rQa00GbMun9h5V7I=", - "requires": { - "archiver": "^0.11.0" - }, - "dependencies": { - "archiver": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/archiver/-/archiver-0.11.0.tgz", - "integrity": "sha1-mBd9p6bAGSt/J5jzDNbquKvXZpA=", - "requires": { - "async": "~0.9.0", - "buffer-crc32": "~0.2.1", - "glob": "~3.2.6", - "lazystream": "~0.1.0", - "lodash": "~2.4.1", - "readable-stream": "~1.0.26", - "tar-stream": "~0.4.0", - "zip-stream": "~0.4.0" - } - }, - "async": { - "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", - "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=" - }, - "bl": { - "version": "0.9.5", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz", - "integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=", - "requires": { - "readable-stream": "~1.0.26" - } - }, - "compress-commons": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-0.1.6.tgz", - "integrity": "sha1-DHQIcP3ljLpRbwrAyCLjOguF36M=", - "requires": { - "buffer-crc32": "~0.2.1", - "crc32-stream": "~0.3.1", - "readable-stream": "~1.0.26" - } - }, - "crc32-stream": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-0.3.4.tgz", - "integrity": "sha1-c7wltF+sHbZjIjGnv86JJ+nwZVI=", - "requires": { - "buffer-crc32": "~0.2.1", - "readable-stream": "~1.0.24" - } - }, - "glob": { - "version": "3.2.11", - "resolved": "https://registry.npmjs.org/glob/-/glob-3.2.11.tgz", - "integrity": "sha1-Spc/Y1uRkPcV0QmH1cAP0oFevj0=", - "requires": { - "inherits": "2", - "minimatch": "0.3" - } - }, - "lazystream": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-0.1.0.tgz", - "integrity": "sha1-GyXWPHcqTCDwpe0KnXf0hLbhaSA=", - "requires": { - "readable-stream": "~1.0.2" - } - }, - "lodash": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", - "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=" - }, - "lru-cache": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", - "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" - }, - "minimatch": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-0.3.0.tgz", - "integrity": "sha1-J12O2qxPG7MyZHIInnlJyDlGmd0=", - "requires": { - "lru-cache": "2", - "sigmund": "~1.0.0" - } - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "0.0.1", - "string_decoder": "~0.10.x" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, - "tar-stream": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-0.4.7.tgz", - "integrity": "sha1-Hx0s6evHtCdlJDyg6PG3v9oKrc0=", - "requires": { - "bl": "^0.9.0", - "end-of-stream": "^1.0.0", - "readable-stream": "^1.0.27-1", - "xtend": "^4.0.0" - } - }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - }, - "zip-stream": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-0.4.1.tgz", - "integrity": "sha1-TqeVqM4Z6fq0mjHR0IdyFBWfA6M=", - "requires": { - "compress-commons": "~0.1.0", - "lodash": "~2.4.1", - "readable-stream": "~1.0.26" - } - } - } - }, "zip-stream": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.0.2.tgz", diff --git a/plugins/brimcap/brimcap-plugin.ts b/plugins/brimcap/brimcap-plugin.ts index abdfb7c16c..9571625986 100644 --- a/plugins/brimcap/brimcap-plugin.ts +++ b/plugins/brimcap/brimcap-plugin.ts @@ -1,16 +1,14 @@ -import {DateTimeFormatter, LocalDateTime, ZoneOffset} from "@js-joda/core" +import fsExtra, {pathExistsSync} from "fs-extra" import path, {join} from "path" +import errors from "src/js/errors" +import {ZealotContext, zed} from "zealot" import {fetchCorrelation} from "../../ppl/detail/flows/fetch" -import open from "../../src/js/lib/open" -import {AppDispatch} from "../../src/js/state/types" -import {zng} from "../../zealot" -import {Record} from "../../zealot/zng" -import BrimcapCLI, {searchOptions} from "./brimcap-cli" import BrimApi from "../../src/js/api" import {IngestParams} from "../../src/js/brim/ingest/getParams" -import fsExtra, {pathExistsSync} from "fs-extra" -import errors from "src/js/errors" +import open from "../../src/js/lib/open" +import {AppDispatch} from "../../src/js/state/types" import {reactElementProps} from "../../src/js/test/integration" +import BrimcapCLI, {searchOptions} from "./brimcap-cli" export default class BrimcapPlugin { private cli: BrimcapCLI @@ -66,7 +64,7 @@ export default class BrimcapPlugin { // TODO: handle contextMenu items, and detail pane/window correlation UI } - private async tryConn(detail: zng.Record, eventId: string) { + private async tryConn(detail: zed.Record, eventId: string) { // TODO: dispatch is only temporarily public to plugins, so this won't always be needed const dispatch = this.api.dispatch as AppDispatch const uidRecords = await dispatch(fetchCorrelation(detail, eventId)) @@ -105,8 +103,8 @@ export default class BrimcapPlugin { const updateButtonStatus = ( toolbarId: string, buttonId: string, - data: zng.Record, - setConn: (conn: zng.Record) => {} + data: zed.Record, + setConn: (conn: zed.Record) => {} ) => { this.tryConn(data, buttonId) .then((conn) => { @@ -148,7 +146,7 @@ export default class BrimcapPlugin { // the detail window's packets button will operate off of the 'current' record this.api.commands.add("data-detail:current", ([record]) => { if (!record) return - const data = Record.deserialize(record) + const data = ZealotContext.decodeRecord(record) updateButtonStatus( "detail", @@ -171,27 +169,21 @@ export default class BrimcapPlugin { ) } - private logToSearchOpts(log: zng.Record): searchOptions { - const ts = log.get("ts") as zng.Primitive + private logToSearchOpts(log: zed.Record): searchOptions { + const ts = log.get("ts") as zed.Time // RFC3999nano format with zero timezone offset - const formatter = DateTimeFormatter.ofPattern( - "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'" - ) - const tsString = LocalDateTime.ofEpochSecond( - getSec(ts), - getNs(ts), - ZoneOffset.UTC - ) - .format(formatter) - .toString() + // const formatter = DateTimeFormatter.ofPattern( + // "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'" + // ) - const dur = log.get("duration") as zng.Primitive + const tsString = ts.toString() + const dur = log.get("duration") as zed.Duration const dest = join(this.api.getTempDir(), `packets-${ts.toString()}.pcap`) return { dstIp: log.get("id.resp_h").toString(), dstPort: log.get("id.resp_p").toString(), - duration: `${dur.toFloat()}s`, + duration: dur.toString(), proto: log.get("proto").toString(), root: this.brimcapDataRoot, srcIp: log.get("id.orig_h").toString(), @@ -201,7 +193,7 @@ export default class BrimcapPlugin { } } - private async downloadPcap(log: zng.Record) { + private async downloadPcap(log: zed.Record) { const searchOpts = this.logToSearchOpts(log) const searchAndOpen = async () => { @@ -264,7 +256,7 @@ export default class BrimcapPlugin { }) // wait for process to end - await new Promise((res) => { + await new Promise((res) => { p.on("close", async () => { res() }) @@ -294,23 +286,3 @@ function statusToPercent(status): number { if (status.pcap_total_size === 0) return 1 else return status.pcap_read_size / status.pcap_total_size || 0 } - -function getSec(data: zng.Primitive): number { - if (data.isSet()) { - return parseInt(data.getValue().split(".")[0]) - } else { - return 0 - } -} - -function getNs(data: zng.Primitive): number { - if (data.isSet()) { - const v = data.getValue().split(".") - if (v.length === 2) { - const frac = v[1] - const digits = frac.length - return parseInt(frac) * Math.pow(10, 9 - digits) - } - } - return 0 -} diff --git a/src/js/flows/downloadPcap.ts b/src/js/flows/downloadPcap.ts deleted file mode 100644 index 78abe3c2e0..0000000000 --- a/src/js/flows/downloadPcap.ts +++ /dev/null @@ -1,10 +0,0 @@ -import {zed} from "zealot" -import open from "../lib/open" -import Packets from "../state/Packets" -import {Thunk} from "../state/types" - -export const downloadPcap = (currentLog: zed.Record): Thunk => (dispatch) => { - dispatch(Packets.fetch(currentLog)).then((pcapFile) => - open(pcapFile, {newWindow: true}) - ) -} diff --git a/src/js/state/Packets/flows.ts b/src/js/state/Packets/flows.ts deleted file mode 100644 index e82cf9d3c5..0000000000 --- a/src/js/state/Packets/flows.ts +++ /dev/null @@ -1,71 +0,0 @@ -import {remote} from "electron" -import {join} from "path" -import {zed} from "zealot" -import {getZealot} from "../../flows/getZealot" -import {saveToFile} from "../../lib/response" -import Current from "../Current" -import Packets from "../Packets" -import {Thunk} from "../types" -import View from "../View" - -export default { - fetch: (log: zed.Record): Thunk> => ( - dispatch: Function, - getState: Function - ) => { - dispatch(Packets.request(log["uid"].toString())) - dispatch(View.showDownloads()) - const state = getState() - const zealot = dispatch(getZealot()) - const spaceId = Current.getSpaceId(state) - const ts = log.get("ts") as zed.Primitive - const dur = log.get("duration") as zed.Primitive - const args = { - ts_sec: getSec(ts), - ts_ns: getNs(ts), - duration_sec: getSec(dur), - duration_ns: getNs(dur), - proto: log.get("proto").toString(), - src_host: log.get("id.orig_h").toString(), - src_port: log.get("id.orig_p").toString(), - dst_host: log.get("id.resp_h").toString(), - dst_port: log.get("id.resp_p").toString(), - spaceId - } - const destDir = remote.app.getPath("temp") - const dest = join(destDir, `packets-${args.ts_sec + args.ts_ns / 1e9}.pcap`) - return zealot.pcaps - .get(args) - .then((resp) => saveToFile(resp, dest)) - .then((file) => { - dispatch(Packets.receive(log.get("uid").toString(), file)) - return file - }) - .catch((error) => { - dispatch(Packets.error(log.get("uid").toString(), error)) - throw error - }) - .finally(() => { - setTimeout(() => dispatch(View.hideDownloads()), 5000) - }) - } -} - -function getSec(data: zed.Primitive): number { - if (data.isUnset()) return 0 - - return parseInt(data.toString().split(".")[0]) -} - -function getNs(data: zed.Primitive): number { - if (data.isUnset()) return 0 - - const v = data.toString().split(".") - if (v.length === 2) { - const frac = v[1] - const digits = frac.length - return parseInt(frac) * Math.pow(10, 9 - digits) - } else { - return 0 - } -} diff --git a/zealot/enhancers/zngToZeek.ts b/zealot/enhancers/zngToZeek.ts deleted file mode 100644 index 609ebe2661..0000000000 --- a/zealot/enhancers/zngToZeek.ts +++ /dev/null @@ -1,74 +0,0 @@ -import {ZealotPayload} from "../types" -import * as zjson from "../zjson" - -function fail(t: any): never { - throw new Error("Unknown zjson Type: " + JSON.stringify(t)) -} - -function getZeekPrimitive(type: zjson.Primitive): zjson.Primitive { - switch (type) { - case "byte": - case "int16": - case "int32": - case "int64": - case "uint16": - case "uint32": - return "int" - case "uint64": - return "count" - case "float64": - return "double" - case "ip": - return "addr" - case "net": - return "subnet" - case "duration": - return "interval" - case "bstring": - return "string" - case "zenum": - return "enum" - default: - return type - } -} - -function replaceColumn(c: zjson.Column): zjson.Column { - if (c.type == "array" || c.type == "set") { - return {...c, of: replaceTypes(c.of)} - } else if (c.type == "union") { - return {...c, of: (c.of || []).map(replaceTypes)} - } else if (c.type == "record") { - return {...c, of: (c.of || []).map(replaceColumn)} - } else { - return {...c, type: getZeekPrimitive(c.type)} - } -} - -export function replaceTypes(t: zjson.Type): zjson.Type { - if (typeof t == "string") { - return getZeekPrimitive(t) - } else if (t.type == "array" || t.type == "set") { - return {...t, of: t.of = replaceTypes(t.of)} - } else if (t.type == "union") { - return {...t, of: t.of.map(replaceTypes)} - } else if (t.type == "record") { - return {...t, of: t.of.map(replaceColumn)} - } - fail(t) -} - -function replaceSchema(r: zjson.Item) { - if (r.schema) r.schema.of = r.schema.of.map(replaceColumn) - return r -} - -export function zngToZeek() { - return (p: ZealotPayload) => { - if (p.type == "SearchRecords") { - return {...p, records: p.records.map(replaceSchema)} - } else { - return p - } - } -} From 8658ad1771b325198aea5d08ddf13c7b83e1e235 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 29 Apr 2021 15:09:00 -0700 Subject: [PATCH 16/41] All tests pass --- .../histogram-search.test.ts.snap | 42 +++++++++++++-- ppl/detail/flows/fetch.test.ts | 4 +- zealot/zed/values/time.test.ts | 21 ++++++++ zealot/zed/values/time.ts | 53 ++++++++++++++----- 4 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 zealot/zed/values/time.test.ts diff --git a/app/search/flows/__snapshots__/histogram-search.test.ts.snap b/app/search/flows/__snapshots__/histogram-search.test.ts.snap index 6d8714466d..706b0407cf 100644 --- a/app/search/flows/__snapshots__/histogram-search.test.ts.snap +++ b/app/search/flows/__snapshots__/histogram-search.test.ts.snap @@ -13,16 +13,48 @@ Object { "stats", ], "table": Object { - "NaN": Object { - "capture_loss": 1, - "conn": 3, + "1582675385000": Object { + "conn": 1, + "stats": 1, + }, + "1582675386000": Object { + "conn": 1, + "weird": 1, + }, + "1582675387000": Object { + "conn": 2, + }, + "1582675388000": Object { + "conn": 5, + }, + "1582675389000": Object { + "conn": 1, + }, + "1582675390000": Object { + "conn": 2, + }, + "1582675391000": Object { + "conn": 1, "dns": 1, + }, + "1582675392000": Object { + "conn": 1, + }, + "1582675393000": Object { + "conn": 2, + "dns": 1, + }, + "1582675394000": Object { "files": 4, "ssl": 1, - "stats": 1, - "weird": 1, "x509": 1, }, + "1582675395000": Object { + "conn": 3, + }, + "1582675397000": Object { + "capture_loss": 1, + }, }, } `; diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index 83e5715d1c..13013f61cb 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -17,7 +17,7 @@ const suricata = createRecord({ }) const uidOrCommunityIdZql = - 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" and ts >= 1582646593.978 and ts < 1582646683.994) | head 100' + 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" and ts >= 1582675393 and ts < 1582675483.016) | head 100' const uidZql = 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" | head 100' @@ -46,7 +46,7 @@ describe("zeek log when community_id is found", () => { expect(zealot.calls("search")).toHaveLength(before + 2) }) - test.only("executes uid first, then cid", async () => { + test("executes uid first, then cid", async () => { const {store, zealot} = setup await store.dispatch(fetchCorrelation(zeek)) const searches = zealot.calls("search") diff --git a/zealot/zed/values/time.test.ts b/zealot/zed/values/time.test.ts new file mode 100644 index 0000000000..b044e7d23d --- /dev/null +++ b/zealot/zed/values/time.test.ts @@ -0,0 +1,21 @@ +import {Time} from "./time" +import {createData} from "test/factories/zed-factory" +import {zed} from "zealot" + +test("toDate()", () => { + new Time("2020-02-25T16:03:13.987654321Z").toDate() + new Time("2020-02-25T16:03:13.87654321Z").toDate() + new Time("2020-02-25T16:03:13.7654321Z").toDate() + new Time("2020-02-25T16:03:13.654321Z").toDate() + new Time("2020-02-25T16:03:13.54321Z").toDate() + new Time("2020-02-25T16:03:13.4321Z").toDate() + new Time("2020-02-25T16:03:13.321Z").toDate() + new Time("2020-02-25T16:03:13.21Z").toDate() + new Time("2020-02-25T16:03:13.1Z").toDate() + new Time("2020-02-25T16:03:13Z").toDate() +}) + +test("create record with time field", () => { + const t = createData(new Date(0)) as zed.Time + expect(t.toDate()).toEqual(new Date(0)) +}) diff --git a/zealot/zed/values/time.ts b/zealot/zed/values/time.ts index ffc1a15bea..580cbadbcc 100644 --- a/zealot/zed/values/time.ts +++ b/zealot/zed/values/time.ts @@ -1,23 +1,48 @@ -import {isNull} from "../utils" +import { + convert, + DateTimeFormatter, + LocalDateTime, + nativeJs +} from "@js-joda/core" import {TypeTime} from "../types/type-time" +import {isNull} from "../utils" import {Primitive} from "./primitive" export class Time extends Primitive { type = TypeTime + _time: LocalDateTime | null - toDate() { - if (isNull(this.value)) return null - // Need to parse dates special now... - let d - if (this.value.match(/[\d.]*/)) { - // Epoch Seconds - d = new Date(+this.value * 1000) - } else { - // ISO Date String - d = new Date(Date.parse(this.value)) - } - if (isNaN(d)) { - throw new Error(`Unkown Time Value: ${this.value}`) + static parse(value: string) { + let time + for (const parse of PARSERS) { + try { + time = parse(value) + break + } catch (e) { + continue + } } + if (!time) throw new Error("zed.Time couldn't parse: " + value) + return time + } + + constructor(value) { + super(value) + this._time = isNull(value) ? null : Time.parse(value) + } + + toDate() { + if (isNull(this._time)) return null + return convert(this._time).toDate() } } + +const parseEpochSec = (v) => { + const d = new Date(+v * 1000) + if (isNaN(d as any)) throw new Error("Not Epoch Seconds: " + v) + return LocalDateTime.from(nativeJs(d)) +} +const NanoFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss[.n]'Z'") +const parseNano = (v) => LocalDateTime.parse(v, NanoFormat) + +const PARSERS = [parseNano, parseEpochSec] From 068f037a5c63283916bb35cb70455aade350b339 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 29 Apr 2021 15:29:49 -0700 Subject: [PATCH 17/41] Fixed api tests --- package-lock.json | 47 +++++++++++++++++++ package.json | 1 + rollup.config.js | 2 + scripts/test/responses.js | 1 + test/api/helper/mod.ts | 2 +- test/api/ingest_test.ts | 5 +- test/api/search_test.ts | 18 ++----- .../correlation-uid-community-id.response | 2 +- test/responses/correlation-uid.response | 2 +- test/responses/count-by-path.response | 4 +- test/responses/count.response | 2 +- test/responses/dns.response | 2 +- test/responses/every-count-by-path.response | 6 +-- .../no-community-id-in-conn.response | 2 +- test/responses/only-alerts.response | 2 +- test/responses/sample.response | 2 +- 16 files changed, 71 insertions(+), 29 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1159813f38..7890b1b512 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4800,6 +4800,38 @@ } } }, + "@rollup/plugin-node-resolve": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz", + "integrity": "sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^3.1.0", + "@types/resolve": "1.17.1", + "builtin-modules": "^3.1.0", + "deepmerge": "^4.2.2", + "is-module": "^1.0.0", + "resolve": "^1.19.0" + }, + "dependencies": { + "builtin-modules": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.2.0.tgz", + "integrity": "sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA==", + "dev": true + }, + "resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "dev": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + } + } + }, "@rollup/plugin-typescript": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-typescript/-/plugin-typescript-8.1.0.tgz", @@ -5589,6 +5621,15 @@ "@types/react": "*" } }, + "@types/resolve": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", + "integrity": "sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -14378,6 +14419,12 @@ "is-path-inside": "^3.0.2" } }, + "is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=", + "dev": true + }, "is-npm": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-5.0.0.tgz", diff --git a/package.json b/package.json index 2dc4a53a84..d04e45420f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@babel/preset-react": "^7.10.4", "@babel/preset-typescript": "^7.10.4", "@rollup/plugin-commonjs": "^17.0.0", + "@rollup/plugin-node-resolve": "^11.2.1", "@types/animejs": "^3.1.2", "@types/classnames": "^2.2.10", "@types/d3": "^5.7.2", diff --git a/rollup.config.js b/rollup.config.js index c13539363a..9eb9afeb0d 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,5 +1,6 @@ import typescript from "@rollup/plugin-typescript" import commonjs from "@rollup/plugin-commonjs" +import nodeResolve from "@rollup/plugin-node-resolve" export default { input: "zealot/index.ts", @@ -12,6 +13,7 @@ export default { ], plugins: [ typescript({module: "ES2020"}), + nodeResolve(), commonjs({extensions: [".js", ".ts"]}) ] } diff --git a/scripts/test/responses.js b/scripts/test/responses.js index 7e888c6ad5..cae55c9e1e 100644 --- a/scripts/test/responses.js +++ b/scripts/test/responses.js @@ -27,6 +27,7 @@ function saveResponse(input, output, query) { const out = fs.createWriteStream(output) deno.stdout.pipe(out) deno.stdout.pipe(process.stdout) + deno.stderr.pipe(process.stderr) return new Promise((resolve) => { deno.on("close", () => { resolve() diff --git a/test/api/helper/mod.ts b/test/api/helper/mod.ts index 4c77af4ec5..645c39b92a 100644 --- a/test/api/helper/mod.ts +++ b/test/api/helper/mod.ts @@ -15,7 +15,7 @@ export function test(name: string, fn: () => void | Promise) { } export function testFile(name: string) { - return join(Deno.cwd(), "data", name) + return join(Deno.cwd(), "..", "data", name) } export function uniq(things: any[]) { diff --git a/test/api/ingest_test.ts b/test/api/ingest_test.ts index 21b2f937fa..9dc90e2405 100644 --- a/test/api/ingest_test.ts +++ b/test/api/ingest_test.ts @@ -1,9 +1,10 @@ import {join} from "https://deno.land/std@0.70.0/path/mod.ts" +import {testFile} from "./helper/mod.ts" import {testApi, assertEquals, uniq} from "./helper/mod.ts" testApi("ingest log", async (zealot) => { const space = await zealot.spaces.create({name: "space1"}) - const log = join(Deno.cwd(), "data/sample.tsv") + const log = testFile("sample.tsv") const resp = await zealot.logs.postPaths({paths: [log], spaceId: space.id}) const messages = await resp.array() @@ -16,7 +17,7 @@ testApi("ingest log", async (zealot) => { testApi("ingest ndjson log", async (zealot) => { const space = await zealot.spaces.create({name: "space1"}) - const log = join(Deno.cwd(), "data/custom-sample.ndjson") + const log = testFile("custom-sample.ndjson") const resp = await zealot.logs.postPaths({ paths: [log], spaceId: space.id diff --git a/test/api/search_test.ts b/test/api/search_test.ts index 7f276ea9f0..39d0e26c2a 100644 --- a/test/api/search_test.ts +++ b/test/api/search_test.ts @@ -27,11 +27,6 @@ testApi("search#records", async (zealot) => { const results = await resp.records() assertEquals(results.length, 30) - assertEquals(results[0].type.splice(0, 2), [ - {name: "_path", type: "string"}, - {name: "ts", type: "time"} - ]) - assertEquals(results[0].value.splice(0, 2), ["stats", "1582646585.983635"]) }) testApi("search#iterator", async (zealot) => { @@ -87,16 +82,11 @@ testApi("search#callbacks record", async (zealot: any) => { }) const args = records.calls[0].args[0] - assertEquals(Object.keys(args), [ - "channel", - "schemas", - "newRecords", - "allRecords" - ]) + assertEquals(Object.keys(args), ["channel", "rows", "newRows", "schemas"]) assertEquals(args.channel, 0) - assertEquals(args.schemas.size, 1) - assertEquals(args.newRecords.length, 1) - assertEquals(args.allRecords.length, 1) + assertEquals(Object.keys(args.schemas).length, 1) + assertEquals(args.newRows.length, 1) + assertEquals(args.rows.length, 1) }) testApi("search#originResponse format=zng", async (zealot: any) => { diff --git a/test/responses/correlation-uid-community-id.response b/test/responses/correlation-uid-community-id.response index 795e665845..294b17d60d 100644 --- a/test/responses/correlation-uid-community-id.response +++ b/test/responses/correlation-uid-community-id.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669702,"ns":490465000},"update_time":{"sec":1619669702,"ns":491148000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619734774,"ns":238431000},"update_time":{"sec":1619734774,"ns":239211000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/correlation-uid.response b/test/responses/correlation-uid.response index 9f460aca6f..7904f9ea34 100644 --- a/test/responses/correlation-uid.response +++ b/test/responses/correlation-uid.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669702,"ns":236500000},"update_time":{"sec":1619669702,"ns":237221000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619734773,"ns":900179000},"update_time":{"sec":1619734773,"ns":900933000},"bytes_read":255,"bytes_matched":255,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count-by-path.response b/test/responses/count-by-path.response index 018d8819e2..95152fe3a4 100644 --- a/test/responses/count-by-path.response +++ b/test/responses/count-by-path.response @@ -1,13 +1,13 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["weird","1"]},{"schema":"23","values":["stats","1"]},{"schema":"23","values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]},{"schema":"23","values":["ssl","1"]},{"schema":"23","values":["dns","2"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["ssl","1"]},{"schema":"23","values":["dns","2"]},{"schema":"23","values":["weird","1"]},{"schema":"23","values":["stats","1"]},{"schema":"23","values":["capture_loss","1"]},{"schema":"23","values":["conn","19"]},{"schema":"23","values":["files","4"]},{"schema":"23","values":["x509","1"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669701,"ns":473479000},"update_time":{"sec":1619669701,"ns":474646000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619734772,"ns":888694000},"update_time":{"sec":1619734772,"ns":890039000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/count.response b/test/responses/count.response index 2a14c179a0..4fd45abd09 100644 --- a/test/responses/count.response +++ b/test/responses/count.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669701,"ns":219273000},"update_time":{"sec":1619669701,"ns":221022000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619734772,"ns":541066000},"update_time":{"sec":1619734772,"ns":542095000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/dns.response b/test/responses/dns.response index 38ab786755..2d48c23ba0 100644 --- a/test/responses/dns.response +++ b/test/responses/dns.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669700,"ns":960752000},"update_time":{"sec":1619669700,"ns":962439000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619734772,"ns":182532000},"update_time":{"sec":1619734772,"ns":183596000},"bytes_read":3519,"bytes_matched":291,"records_read":30,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/every-count-by-path.response b/test/responses/every-count-by-path.response index 8132a71a38..3233ecfbd4 100644 --- a/test/responses/every-count-by-path.response +++ b/test/responses/every-count-by-path.response @@ -1,16 +1,16 @@ {"type":"TaskStart","task_id":0} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["2020-02-25T16:03:17Z","capture_loss","1"]},{"schema":"23","values":["2020-02-25T16:03:15Z","conn","3"]},{"schema":"23","values":["2020-02-25T16:03:14Z","x509","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","ssl","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","files","4"]},{"schema":"23","values":["2020-02-25T16:03:13Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:13Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:12Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:10Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:09Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:08Z","conn","5"]},{"schema":"23","values":["2020-02-25T16:03:07Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:06Z","weird","1"]},{"schema":"23","values":["2020-02-25T16:03:06Z","conn","1"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","types":[{"kind":"typedef","name":"23","type":{"kind":"record","fields":[{"name":"ts","type":{"kind":"primitive","name":"time"}},{"name":"_path","type":{"kind":"primitive","name":"string"}},{"name":"count","type":{"kind":"primitive","name":"uint64"}}]}}],"values":["2020-02-25T16:03:17Z","capture_loss","1"]},{"schema":"23","values":["2020-02-25T16:03:15Z","conn","3"]},{"schema":"23","values":["2020-02-25T16:03:14Z","x509","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","ssl","1"]},{"schema":"23","values":["2020-02-25T16:03:14Z","files","4"]},{"schema":"23","values":["2020-02-25T16:03:13Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:13Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:12Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","dns","1"]},{"schema":"23","values":["2020-02-25T16:03:11Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:10Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:09Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:08Z","conn","5"]},{"schema":"23","values":["2020-02-25T16:03:07Z","conn","2"]},{"schema":"23","values":["2020-02-25T16:03:06Z","weird","1"]},{"schema":"23","values":["2020-02-25T16:03:06Z","conn","1"]}]} -{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","values":["2020-02-25T16:03:05Z","stats","1"]},{"schema":"23","values":["2020-02-25T16:03:05Z","conn","1"]}]} +{"type":"SearchRecords","channel_id":0,"records":[{"schema":"23","values":["2020-02-25T16:03:05Z","conn","1"]},{"schema":"23","values":["2020-02-25T16:03:05Z","stats","1"]}]} {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669701,"ns":722861000},"update_time":{"sec":1619669701,"ns":724641000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619734773,"ns":221437000},"update_time":{"sec":1619734773,"ns":223407000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/no-community-id-in-conn.response b/test/responses/no-community-id-in-conn.response index 65c6809579..e1b704b1e0 100644 --- a/test/responses/no-community-id-in-conn.response +++ b/test/responses/no-community-id-in-conn.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669702,"ns":741767000},"update_time":{"sec":1619669702,"ns":742575000},"bytes_read":224,"bytes_matched":224,"records_read":2,"records_matched":2} +{"type":"SearchStats","start_time":{"sec":1619734774,"ns":570457000},"update_time":{"sec":1619734774,"ns":571148000},"bytes_read":224,"bytes_matched":224,"records_read":2,"records_matched":2} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/only-alerts.response b/test/responses/only-alerts.response index 09e1cc8e82..120e61717a 100644 --- a/test/responses/only-alerts.response +++ b/test/responses/only-alerts.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669702,"ns":985238000},"update_time":{"sec":1619669702,"ns":985938000},"bytes_read":2299,"bytes_matched":2299,"records_read":11,"records_matched":11} +{"type":"SearchStats","start_time":{"sec":1619734774,"ns":900640000},"update_time":{"sec":1619734774,"ns":901332000},"bytes_read":2299,"bytes_matched":2299,"records_read":11,"records_matched":11} {"type":"TaskEnd","task_id":0} diff --git a/test/responses/sample.response b/test/responses/sample.response index 20e6f2b2b1..201bc9beba 100644 --- a/test/responses/sample.response +++ b/test/responses/sample.response @@ -7,7 +7,7 @@ {"type":"SearchEnd","channel_id":0,"reason":"eof"} -{"type":"SearchStats","start_time":{"sec":1619669701,"ns":972898000},"update_time":{"sec":1619669701,"ns":975503000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} +{"type":"SearchStats","start_time":{"sec":1619734773,"ns":563106000},"update_time":{"sec":1619734773,"ns":565874000},"bytes_read":3519,"bytes_matched":3519,"records_read":30,"records_matched":30} {"type":"TaskEnd","task_id":0} From 6d8d19610006f2dd4265d9b60de25353167483a4 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Thu, 29 Apr 2021 16:07:13 -0700 Subject: [PATCH 18/41] Guessing widths again --- app/core/formatters/format-zed.ts | 2 +- app/viewer/measure.ts | 23 +++++++++++++++++++++++ src/js/brim/primitiveCell.ts | 5 +---- src/js/models/TableColumns.ts | 27 +++++++++------------------ zealot/zed/context.ts | 16 +++++++++------- zealot/zed/schema.ts | 2 +- zealot/zed/types/type-record.ts | 30 ++++++++++++++++++++---------- zealot/zed/utils.ts | 2 +- zealot/zjson.ts | 2 +- 9 files changed, 66 insertions(+), 43 deletions(-) create mode 100644 app/viewer/measure.ts diff --git a/app/core/formatters/format-zed.ts b/app/core/formatters/format-zed.ts index 8f0d061660..e8464a9ee0 100644 --- a/app/core/formatters/format-zed.ts +++ b/app/core/formatters/format-zed.ts @@ -5,6 +5,6 @@ import {zed} from "zealot" export function formatPrimitive(data: zed.Primitive) { if (data.isUnset()) return "⦻" if (zed.isInt(data)) return withCommas(data.toString()) - if (zed.isTime(data)) return brim.time(this.toDate()).format() + if (zed.isTime(data)) return brim.time(data.toDate()).format() return data.toString() } diff --git a/app/viewer/measure.ts b/app/viewer/measure.ts new file mode 100644 index 0000000000..cba9a0384a --- /dev/null +++ b/app/viewer/measure.ts @@ -0,0 +1,23 @@ +import {formatPrimitive} from "app/core/formatters/format-zed" +import {zed} from "zealot" + +const ONE_CHAR = 7.39 +const FIELD_PAD = 14 + +const MAX_WIDTH = 500 +const MIN_WIDTH = 10 +const resizeHandle = 5 +const sortIcon = 11 + +export function estimateHeaderWidth(name: string) { + let width = Math.ceil(name.length * ONE_CHAR + resizeHandle + sortIcon) + return Math.min(MAX_WIDTH, width) +} + +export function estimateCellWidth(value: zed.AnyValue) { + let width = MIN_WIDTH + if (value instanceof zed.Primitive) { + width = Math.ceil(formatPrimitive(value).length * ONE_CHAR + FIELD_PAD) + } + return Math.min(MAX_WIDTH, width) +} diff --git a/src/js/brim/primitiveCell.ts b/src/js/brim/primitiveCell.ts index 461fc7bfa5..a846ad67d6 100644 --- a/src/js/brim/primitiveCell.ts +++ b/src/js/brim/primitiveCell.ts @@ -1,9 +1,6 @@ +import {ONE_CHAR, FIELD_PAD, PATH_PAD} from "app/viewer/measure" import {zed} from "zealot" -export const ONE_CHAR = 7.39 -export const FIELD_PAD = 14 -export const PATH_PAD = 12 - interface PrimitiveField { name: string data: zed.Primitive diff --git a/src/js/models/TableColumns.ts b/src/js/models/TableColumns.ts index 361ecfc11e..f3a4c21018 100644 --- a/src/js/models/TableColumns.ts +++ b/src/js/models/TableColumns.ts @@ -1,5 +1,5 @@ +import {estimateCellWidth, estimateHeaderWidth} from "app/viewer/measure" import {zed} from "zealot" -import {createCell} from "../brim/cell" import columnOrder from "../lib/columnOrder" import {$Column} from "../state/Columns/models/column" import {ColumnSettingsMap, TableColumn} from "../state/Columns/types" @@ -31,26 +31,17 @@ export default class TableColumns { ) } - setWidths(logs: zed.Record[]) { - const MAX_WIDTH = 500 - const resizeHandle = 5 - const sortIcon = 11 - + setWidths(records: zed.Record[]) { this.cols.forEach((col) => { if (col.width) return - const colName = createCell(new zed.Field("", new zed.String(col.name))) - - let max = colName.guessWidth() + resizeHandle + sortIcon - logs.forEach((log) => { - const data = log.try(col.name) - if (data) { - const cell = createCell(new zed.Field(col.name, data)) - const len = cell.guessWidth() - if (len > max) max = len - } + let max = estimateHeaderWidth(col.name) + records.forEach((r) => { + const data = r.try(col.name) + if (!data) return + const width = estimateCellWidth(data) + if (width > max) max = width }) - - col.width = Math.min(max, MAX_WIDTH) + col.width = max }) } diff --git a/zealot/zed/context.ts b/zealot/zed/context.ts index acb60454fb..aa47c7d628 100644 --- a/zealot/zed/context.ts +++ b/zealot/zed/context.ts @@ -6,7 +6,7 @@ import {TypeRecord} from "./types/type-record" import {TypeSet} from "./types/type-set" import {TypeUnion} from "./types/type-union" import {ContainerTypeInterface, ZedType} from "./types/types" -import {isAlias, typeId} from "./utils" +import {isAlias, isNull, typeId} from "./utils" import {Record} from "./values/record" import * as zjson from "../zjson" import {Field} from "./values/field" @@ -61,12 +61,14 @@ export class ZedContext { // Containers case "record": return this.lookupTypeRecord( - obj.fields.map(({name, type}) => { - return { - name, - type: this.decodeType(type, typedefs) - } - }) + isNull(obj.fields) + ? null + : obj.fields.map(({name, type}) => { + return { + name, + type: this.decodeType(type, typedefs) + } + }) ) case "array": return this.lookupTypeArray(this.decodeType(obj.type, typedefs)) diff --git a/zealot/zed/schema.ts b/zealot/zed/schema.ts index 106d65e30b..da423ecc70 100644 --- a/zealot/zed/schema.ts +++ b/zealot/zed/schema.ts @@ -1,4 +1,4 @@ -import {trueType} from "." +import {trueType} from "./index" import {TypeField, TypeRecord} from "./types/type-record" export class Schema { diff --git a/zealot/zed/types/type-record.ts b/zealot/zed/types/type-record.ts index 0d57942e44..af5313f23a 100644 --- a/zealot/zed/types/type-record.ts +++ b/zealot/zed/types/type-record.ts @@ -12,13 +12,14 @@ export type TypeField = { } export class TypeRecord implements ContainerTypeInterface { kind = "record" - fields: TypeField[] + fields: TypeField[] | null constructor(fields: TypeField[]) { this.fields = fields } static stringify(fields) { + if (isNull(fields)) return "null" let s = "{" let sep = "" fields.forEach((f) => { @@ -34,30 +35,39 @@ export class TypeRecord implements ContainerTypeInterface { if (values === null) return new Record(this, null) return new Record( this, - this.fields.map((field, index) => { - return new Field(field.name, field.type.create(values[index], typedefs)) - }) + isNull(this.fields) + ? null + : this.fields.map((field, index) => { + return new Field( + field.name, + field.type.create(values[index], typedefs) + ) + }) ) } serialize(typedefs): RecordType { return { kind: "record", - fields: this.fields.map((f) => { - return { - name: f.name, - type: f.type.serialize(typedefs) - } - }) + fields: isNull(this.fields) + ? null + : this.fields.map((f) => { + return { + name: f.name, + type: f.type.serialize(typedefs) + } + }) } } hasTypeType(ctx: ZedContext) { + if (isNull(this.fields)) return false return this.fields.some((f) => ctx.hasTypeType(f.type)) } walkTypeValues(ctx: ZedContext, values, visit) { if (isNull(values)) return + if (isNull(this.fields)) return this.fields.forEach((f, i) => { ctx.walkTypeValues(f.type, values[i], visit) diff --git a/zealot/zed/utils.ts b/zealot/zed/utils.ts index e60710b87b..33662b1458 100644 --- a/zealot/zed/utils.ts +++ b/zealot/zed/utils.ts @@ -1,4 +1,4 @@ -import {TypeAlias, Uint16, Uint32, Uint64, Uint8} from "." +import {TypeAlias, Uint16, Uint32, Uint64, Uint8} from "./index" import {ZedType} from "./types/types" import {Int16} from "./values/int16" import {Int32} from "./values/int32" diff --git a/zealot/zjson.ts b/zealot/zjson.ts index 8adebe51ae..f9bd446367 100644 --- a/zealot/zjson.ts +++ b/zealot/zjson.ts @@ -12,7 +12,7 @@ export type PrimitiveType = { export type RecordType = { kind: "record" - fields: RecordFieldType[] + fields: RecordFieldType[] | null } export type ArrayType = { From ba738764c51599e390dc0dda539d83525bdfdb0d Mon Sep 17 00:00:00 2001 From: James Kerr Date: Fri, 30 Apr 2021 15:06:29 -0700 Subject: [PATCH 19/41] Working on itests --- app/core/hooks/useStoreExport.ts | 4 +- app/core/utils/type-class-names.ts | 2 +- app/detail/Pane.tsx | 2 +- app/detail/flows/contextMenu.ts | 21 ++--- app/viewer/cell.tsx | 43 +++++++++ app/viewer/measure.ts | 6 +- app/viewer/value.tsx | 93 +++++++++++++++++++ ppl/menus/detailFieldContextMenu.ts | 45 ++++----- ppl/menus/searchFieldContextMenu.ts | 47 +++++----- src/css/_field-cell.scss | 23 ----- src/css/_log-viewer.scss | 21 ++++- src/css/main.scss | 1 - src/css/shared/_type-colors.scss | 4 + src/js/brim/cell.ts | 19 ---- src/js/brim/complexCell.ts | 51 ---------- src/js/brim/primitiveCell.ts | 40 -------- src/js/components/LogCell/CompoundField.tsx | 72 -------------- src/js/components/LogCell/SingleField.tsx | 52 ----------- src/js/components/LogCell/index.tsx | 67 ------------- src/js/components/LogRow.tsx | 33 +++---- .../components/SearchResults/ResultsTable.tsx | 5 - src/js/flows/rightclick/cellMenu.test.ts | 80 ---------------- 22 files changed, 233 insertions(+), 498 deletions(-) create mode 100644 app/viewer/cell.tsx create mode 100644 app/viewer/value.tsx delete mode 100644 src/css/_field-cell.scss delete mode 100644 src/js/brim/cell.ts delete mode 100644 src/js/brim/complexCell.ts delete mode 100644 src/js/brim/primitiveCell.ts delete mode 100644 src/js/components/LogCell/CompoundField.tsx delete mode 100644 src/js/components/LogCell/SingleField.tsx delete mode 100644 src/js/components/LogCell/index.tsx delete mode 100644 src/js/flows/rightclick/cellMenu.test.ts diff --git a/app/core/hooks/useStoreExport.ts b/app/core/hooks/useStoreExport.ts index 54b8173d05..9b4bfa66ab 100644 --- a/app/core/hooks/useStoreExport.ts +++ b/app/core/hooks/useStoreExport.ts @@ -5,14 +5,16 @@ import {useEffect} from "react" import {useDispatch, useSelector} from "react-redux" import LogDetails from "src/js/state/LogDetails" import Viewer from "src/js/state/Viewer" +import {ZealotContext} from "zealot" import {executeCommand} from "../../../src/js/flows/executeCommand" const useStoreExport = () => { const currentData = useSelector(LogDetails.build) const dispatch = useDispatch() + const zjson = currentData ? ZealotContext.encodeRecord(currentData) : null useEffect(() => { - dispatch(executeCommand("data-detail:current", currentData?.serialize())) + dispatch(executeCommand("data-detail:current", zjson)) }, [currentData]) const selectedData = useSelector(Viewer.getSelectedRecords) diff --git a/app/core/utils/type-class-names.ts b/app/core/utils/type-class-names.ts index 7050c90385..8899637c45 100644 --- a/app/core/utils/type-class-names.ts +++ b/app/core/utils/type-class-names.ts @@ -3,7 +3,7 @@ import {zed} from "zealot" export function typeClassNames(data: zed.AnyValue) { const classNames = [] if (data instanceof zed.Primitive) { - classNames.push(data.type.toString()) + classNames.push(data.type.name) } if (data.isUnset()) classNames.push("null") return classNames.join(" ") diff --git a/app/detail/Pane.tsx b/app/detail/Pane.tsx index 82da61ba6e..4b8ac813a2 100644 --- a/app/detail/Pane.tsx +++ b/app/detail/Pane.tsx @@ -32,7 +32,7 @@ const Content = memo(function Content({record}) { const isZeek = event instanceof ZeekEvent const isSuricata = event instanceof SuricataEvent const {uid, cid} = new Correlation(record).getIds() - const isConn = isZeek && record["_path"].toString() === "conn" + const isConn = isZeek && record.try("_path")?.toString() === "conn" const hasMd5 = isZeek && record.has("md5") return ( diff --git a/app/detail/flows/contextMenu.ts b/app/detail/flows/contextMenu.ts index ce8bc1db87..7ea2cc4f88 100644 --- a/app/detail/flows/contextMenu.ts +++ b/app/detail/flows/contextMenu.ts @@ -1,22 +1,13 @@ import detailFieldContextMenu from "ppl/menus/detailFieldContextMenu" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" -import {showContextMenu} from "src/js/lib/System" -import Columns from "src/js/state/Columns" -import Current from "src/js/state/Current" -import SearchBar from "src/js/state/SearchBar" import {zed} from "zealot" -const contextMenu = (field: zed.Field, record: zed.Record) => (_, getState) => { - const space = Current.mustGetSpace(getState()) - const program = SearchBar.getSearchProgram(getState()) - const tableColumns = Columns.getCurrentTableColumns(getState()) - const columns = tableColumns.getColumns().map((c) => c.name) - const builder = - global.windowName === "detail" - ? detailFieldContextMenu - : searchFieldContextMenu - - showContextMenu(builder(program, columns)(field, record, false)) +const contextMenu = (field: zed.Field, record: zed.Record) => (dispatch) => { + if (global.windowName === "detail") { + dispatch(detailFieldContextMenu({field, record, value: field.value})) + } else { + dispatch(searchFieldContextMenu({field, record, value: field.value})) + } } export default contextMenu diff --git a/app/viewer/cell.tsx b/app/viewer/cell.tsx new file mode 100644 index 0000000000..8d1fa9b226 --- /dev/null +++ b/app/viewer/cell.tsx @@ -0,0 +1,43 @@ +import React, {useState} from "react" +import Tooltip from "src/js/components/Tooltip" +import styled from "styled-components" + +const BG = styled.div` + overflow: hidden; +` + +const getTooltipStyle = (el: Element) => { + if (!el) return {} + const {top, left} = el.getBoundingClientRect() + return {top: top - 21, left: left + 4} +} + +export default function Cell({width, children, name}) { + const [hover, setHover] = useState(false) + const [tooltipStyle, setTooltipStyle] = useState({}) + + function handleMouseEnter(e) { + setHover(true) + setTooltipStyle(getTooltipStyle(e.currentTarget)) + } + + function handleMouseLeave() { + setHover(false) + } + + return ( + + {children} + {hover && ( + + {name} + + )} + + ) +} diff --git a/app/viewer/measure.ts b/app/viewer/measure.ts index cba9a0384a..7be50e8fe2 100644 --- a/app/viewer/measure.ts +++ b/app/viewer/measure.ts @@ -2,7 +2,7 @@ import {formatPrimitive} from "app/core/formatters/format-zed" import {zed} from "zealot" const ONE_CHAR = 7.39 -const FIELD_PAD = 14 +const CELL_PAD = ONE_CHAR * 2 const MAX_WIDTH = 500 const MIN_WIDTH = 10 @@ -17,7 +17,9 @@ export function estimateHeaderWidth(name: string) { export function estimateCellWidth(value: zed.AnyValue) { let width = MIN_WIDTH if (value instanceof zed.Primitive) { - width = Math.ceil(formatPrimitive(value).length * ONE_CHAR + FIELD_PAD) + width = Math.ceil(formatPrimitive(value).length * ONE_CHAR + CELL_PAD) + } else { + width = Math.ceil(value.toString().length * ONE_CHAR + CELL_PAD) } return Math.min(MAX_WIDTH, width) } diff --git a/app/viewer/value.tsx b/app/viewer/value.tsx new file mode 100644 index 0000000000..540e6c7ca6 --- /dev/null +++ b/app/viewer/value.tsx @@ -0,0 +1,93 @@ +import {formatPrimitive} from "app/core/formatters/format-zed" +import {typeClassNames} from "app/core/utils/type-class-names" +import {cssVar, transparentize} from "polished" +import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" +import React from "react" +import {useDispatch} from "react-redux" +import styled from "styled-components" +import {zed} from "zealot" + +const havelock = cssVar("--havelock") +const transHavelock = transparentize(0.75, havelock as string) + +const BG = styled.span` + cursor: default; + display: inline-block; + &:hover { + background: ${transHavelock}; + border-radius: 4px; + } +` +const Syntax = styled.span` + color: var(--cloudy); +` + +type ValueProps = { + field: zed.Field + value: zed.AnyValue + record: zed.Record + padBefore?: boolean + padAfter?: boolean +} + +const pad = (bool) => (bool ? <> : null) + +export default function Value(props: ValueProps) { + if (props.value.isUnset()) { + return + } else if (props.value instanceof zed.Set) { + return + } else { + return + } +} + +export function PrimitiveValue(props: ValueProps) { + const dispatch = useDispatch() + const fillCell = props.field.value === props.value // This is the only value in the cell + return ( + + dispatch( + searchFieldContextMenu({ + field: props.field, + record: props.record, + value: props.value + }) + ) + } + > + {pad(props.padBefore)} + {formatPrimitive(props.value as zed.Primitive)} + {pad(props.padAfter)} + + ) +} + +export function SetValue(props: ValueProps) { + const set = props.value as zed.Set + const lastItem = (i) => i === set.items.length - 1 + const firstItem = (i) => i === 0 + return ( + <> + {pad(props.padBefore)} + |[ + {set.items.map((v, i) => ( + <> + + {lastItem(i) ? null : ,} + + ))} + ]| + {pad(props.padBefore)} + + ) +} diff --git a/ppl/menus/detailFieldContextMenu.ts b/ppl/menus/detailFieldContextMenu.ts index fa0897bb1a..3e8be5f843 100644 --- a/ppl/menus/detailFieldContextMenu.ts +++ b/ppl/menus/detailFieldContextMenu.ts @@ -1,24 +1,25 @@ -import {MenuItemConstructorOptions} from "electron/main" import {isEqual} from "lodash" import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" -import {ZealotContext} from "zealot" -import {zed} from "zealot" +import {showContextMenu} from "src/js/lib/System" +import Columns from "src/js/state/Columns" +import SearchBar from "src/js/state/SearchBar" +import {ZealotContext, zed} from "zealot" -export default function detailFieldContextMenu( - program: string, - columns: string[] -) { - return function( - field: zed.Field, - log: zed.Record, - compound: boolean - ): MenuItemConstructorOptions[] { - const isTime = field.data instanceof zed.Time +export default function detailFieldContextMenu({field, record, value}) { + return (_, getState) => { + const columns = Columns.getCurrentTableColumns(getState()) + .getColumns() + .map((c) => c.name) + const program = SearchBar.getSearchProgram(getState()) + const isTime = value instanceof zed.Time const isGroupBy = hasGroupByProc(program) - const isIp = field.data instanceof zed.Ip + const isIp = value instanceof zed.Ip const hasCol = columns.includes(field.name) - const sameCols = isEqual(log.columns.sort(), columns.sort()) + const sameCols = isEqual(record.columns.sort(), columns.sort()) + const isPrimitive = field.value instanceof zed.Primitive + const isArrayish = + field.value instanceof zed.Array || field.value instanceof zed.Set const virusTotal = [ "hassh", "host", @@ -34,22 +35,22 @@ export default function detailFieldContextMenu( const detailMenuActions = menu.actions.detail const fieldData = ZealotContext.encodeField(field) - const recordData = ZealotContext.encodeRecord(log) + const recordData = ZealotContext.encodeRecord(record) - return [ + return showContextMenu([ detailMenuActions.include.menuItem([fieldData], { enabled: hasCol, - visible: !compound + visible: isPrimitive }), detailMenuActions.exclude.menuItem([fieldData], { enabled: hasCol, - visible: !compound + visible: isPrimitive }), detailMenuActions.in.menuItem([fieldData], { - visible: !!compound + visible: isArrayish }), detailMenuActions.notIn.menuItem([fieldData], { - visible: !!compound + visible: isArrayish }), detailMenuActions.freshInclude.menuItem([fieldData], {enabled: true}), menu.separator(), @@ -76,6 +77,6 @@ export default function detailFieldContextMenu( detailMenuActions.logResult.menuItem([fieldData, recordData], { enabled: true }) - ] + ]) } } diff --git a/ppl/menus/searchFieldContextMenu.ts b/ppl/menus/searchFieldContextMenu.ts index daec1095c6..ba306fcf7f 100644 --- a/ppl/menus/searchFieldContextMenu.ts +++ b/ppl/menus/searchFieldContextMenu.ts @@ -1,25 +1,27 @@ -import {MenuItemConstructorOptions} from "electron" import {isEqual} from "lodash" import menu from "src/js/electron/menu" import {hasGroupByProc} from "src/js/lib/Program" -import {RightClickBuilder} from "src/js/types" +import {showContextMenu} from "src/js/lib/System" +import Columns from "src/js/state/Columns" +import SearchBar from "src/js/state/SearchBar" import {ZealotContext, zed} from "zealot" -export default function searchFieldContextMenu( - program: string, - columns: string[] -): RightClickBuilder { - return function( - field: zed.Field, - log: zed.Record, - compound: boolean - ): MenuItemConstructorOptions[] { - const isTime = field.data instanceof zed.Time +export default function searchFieldContextMenu({field, record, value}) { + return (_, getState) => { + const columns = Columns.getCurrentTableColumns(getState()) + .getColumns() + .map((c) => c.name) + const program = SearchBar.getSearchProgram(getState()) + const isTime = value instanceof zed.Time const isGroupBy = hasGroupByProc(program) - const isIp = field.data instanceof zed.Ip + const isIp = value instanceof zed.Ip const hasCol = columns.includes(field.name) - const flatColNames = log.flatten().columns + const flatColNames = record.flatten().columns const sameCols = isEqual(flatColNames.sort(), columns.sort()) + const isPrimitive = field.value instanceof zed.Primitive + const isArrayish = + field.value instanceof zed.Array || field.value instanceof zed.Set + const virusTotal = [ "hassh", "host", @@ -33,24 +35,25 @@ export default function searchFieldContextMenu( ].includes(field.name) const searchMenuActions = menu.actions.search - + // A bit of a hack + field.value = value const fieldData = ZealotContext.encodeField(field) - const recordData = ZealotContext.encodeRecord(log) + const recordData = ZealotContext.encodeRecord(record) - return [ + showContextMenu([ searchMenuActions.include.menuItem([fieldData], { enabled: hasCol, - visible: !compound + visible: isPrimitive }), searchMenuActions.exclude.menuItem([fieldData], { enabled: hasCol, - visible: !compound + visible: isPrimitive }), searchMenuActions.in.menuItem([fieldData], { - visible: !!compound + visible: isArrayish }), searchMenuActions.notIn.menuItem([fieldData], { - visible: !!compound + visible: isArrayish }), searchMenuActions.freshInclude.menuItem([fieldData], {enabled: true}), menu.separator(), @@ -80,6 +83,6 @@ export default function searchFieldContextMenu( searchMenuActions.logResult.menuItem([fieldData, recordData], { enabled: true }) - ] + ]) } } diff --git a/src/css/_field-cell.scss b/src/css/_field-cell.scss deleted file mode 100644 index 8704362300..0000000000 --- a/src/css/_field-cell.scss +++ /dev/null @@ -1,23 +0,0 @@ -.field-cell { - @include type-colors; - cursor: default; - user-select: all; - - &.time { - span { - padding: 0 6px; - &:last-child { - padding-right: 0; - } - - &:first-child { - padding-left: 0; - } - } - } - - &._path, - &.event_type { - @include path-tag; - } -} diff --git a/src/css/_log-viewer.scss b/src/css/_log-viewer.scss index 62a6fb59ae..ce79846053 100644 --- a/src/css/_log-viewer.scss +++ b/src/css/_log-viewer.scss @@ -8,6 +8,10 @@ .using-keyboard &:focus { outline: none; } + + * { + @include type-colors; + } } .viewer .view { @@ -45,7 +49,7 @@ &.highlight { background: var(--havelock); - .field-cell { + span { color: white; } } @@ -58,9 +62,18 @@ line-height: 100px; } -.viewer .field-cell { - &.count, - &.duration { +.viewer { + .count, + .duration, + .uint8, + .uint16, + .uint32, + .uint64, + .int8, + .int16, + .int32, + .int64 + { text-align: right; } } diff --git a/src/css/main.scss b/src/css/main.scss index 6e29cea8f5..88cc72a9a1 100644 --- a/src/css/main.scss +++ b/src/css/main.scss @@ -76,7 +76,6 @@ @import "debug-modal"; @import "click-feedback"; @import "header-cell"; -@import "field-cell"; @import "time-span-pickers"; @import "input-suggestions"; @import "tab-bar"; diff --git a/src/css/shared/_type-colors.scss b/src/css/shared/_type-colors.scss index 30ab149686..2a8c37bbea 100644 --- a/src/css/shared/_type-colors.scss +++ b/src/css/shared/_type-colors.scss @@ -37,4 +37,8 @@ &.null { color: var(--cloudy); } + + &.time { + color: var(--interval) + } } diff --git a/src/js/brim/cell.ts b/src/js/brim/cell.ts deleted file mode 100644 index 6f4163ba7b..0000000000 --- a/src/js/brim/cell.ts +++ /dev/null @@ -1,19 +0,0 @@ -import {zed} from "zealot" -import {createPrimitiveCell} from "./primitiveCell" - -export interface Cell { - guessWidth: () => number -} - -type Args = { - name: string - data: zed.AnyValue -} - -export function createCell({name, data}: Args): Cell { - if (data instanceof zed.Primitive) { - return createPrimitiveCell({name, data}) - } else { - // return createComplexCell({name, data}) - } -} diff --git a/src/js/brim/complexCell.ts b/src/js/brim/complexCell.ts deleted file mode 100644 index d14ac3721c..0000000000 --- a/src/js/brim/complexCell.ts +++ /dev/null @@ -1,51 +0,0 @@ -// import { -// ZedArray, -// ZedEnum, -// ZedMap, -// zed.Record, -// ZedSet, -// ZedUnion -// } from "zealot/zed" -// import {createCell} from "./cell" -// import {ONE_CHAR} from "./primitiveCell" - -// export const COMPOUND_FIELD_RGX = /^(set|array|union|record)$/ - -// export type ComplexCell = ReturnType - -// type Args = { -// name: string -// data: zed.Record | ZedArray | ZedSet | ZedUnion | ZedEnum | ZedMap -// } - -// export function createComplexCell({name, data}: Args) { -// const items = -// "items" in data -// ? data.items.map((data, i) => createCell({name: `${name}.${i}`, data})) -// : [] - -// return { -// name, -// container: data.constructor.name, -// length: items.length, -// item: (index: number) => items[index], - -// stringValue() { -// return data.toString() -// }, -// compound() { -// return true -// }, -// guessWidth() { -// const comma = ONE_CHAR -// const wrap = 2 * ONE_CHAR -// let sum = 0 -// for (const item of items) { -// sum += item.guessWidth() -// } -// sum += comma * (items.length - 1) -// sum += wrap -// return sum -// } -// } -// } diff --git a/src/js/brim/primitiveCell.ts b/src/js/brim/primitiveCell.ts deleted file mode 100644 index a846ad67d6..0000000000 --- a/src/js/brim/primitiveCell.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {ONE_CHAR, FIELD_PAD, PATH_PAD} from "app/viewer/measure" -import {zed} from "zealot" - -interface PrimitiveField { - name: string - data: zed.Primitive -} - -export function createPrimitiveCell({name, data}: PrimitiveField) { - const type = data.type - const value = data.value - - return { - name, - type, - value, - serialize() { - return {name, type, value} - }, - stringValue(): string { - if (value === null) return "null" - else if (Array.isArray(value)) return value.join(",") - else return value - }, - - toDate() { - return new Date(+this.value * 1000) - }, - compound() { - return false - }, - guessWidth() { - if (name === "_path") { - return this.display().length * ONE_CHAR + FIELD_PAD + PATH_PAD - } else { - return Math.ceil(this.display().length * ONE_CHAR + FIELD_PAD) - } - } - } -} diff --git a/src/js/components/LogCell/CompoundField.tsx b/src/js/components/LogCell/CompoundField.tsx deleted file mode 100644 index 83ce626ebb..0000000000 --- a/src/js/components/LogCell/CompoundField.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import React from "react" -import {zed} from "zealot" - -type Props = { - field: zed.Field - log: zed.Record - menuBuilder: Function -} - -export default function CompoundField(_: Props) { - return

Implement compound fields

- // // @ts-ignore - // const compound = createComplexCell(field) - // const render = [] - - // for (let i = 0; i < compound.length; ++i) { - // const item = new zed.Field({ - // name: field.name, - // // @ts-ignore - // data: field.data.items && field.data.items[i] - // }) - // if (item) { - // const menu = menuBuilder(item, log, true) - // render.push() - // } - // if (i !== compound.length - 1) { - // render.push() - // } - // } - - // return {render} -} - -// function Comma() { -// return -// } - -// function Extra({value, className}: {value: string | null; className?: string}) { -// return ( -//
{value}
-// ) -// } - -// type WrapperProps = { -// type: string | null -// children: any -// } - -// function Wrapper({type, children}: WrapperProps) { -// const [open, close] = getWrapper(type) -// return ( -// <> -// -// {children} -// -// -// ) -// } - -// function getWrapper(container) { -// switch (container) { -// case "set": -// return ["{", "}"] -// case "vector": -// // DELETE after vector-array is merged -// return ["[", "]"] -// case "array": -// return ["[", "]"] -// default: -// return [null, null] -// } -// } diff --git a/src/js/components/LogCell/SingleField.tsx b/src/js/components/LogCell/SingleField.tsx deleted file mode 100644 index b3331c3b15..0000000000 --- a/src/js/components/LogCell/SingleField.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import classNames from "classnames" -import React, {useEffect, useRef, useState} from "react" -import {zed} from "zealot" -import {$Menu} from "../../electron/menu" -import lib from "../../lib" -import {showContextMenu} from "../../lib/System" -import FieldCell from "../FieldCell" - -type Props = { - field: zed.Field - record: zed.Record - menu: $Menu -} - -const on = document.addEventListener -const off = document.removeEventListener - -export default function SingleField({field, menu, record}: Props) { - const [selected, setSelected] = useState(false) - const cell = useRef() - - function onClick(e) { - setSelected(true) - lib.win.selectText(e.currentTarget) - } - - function onOutsideClick(e: MouseEvent) { - if (cell.current && cell.current.contains(e.target)) return - setSelected(false) - off("click", onOutsideClick, false) - } - - useEffect(() => { - if (selected) { - on("click", onOutsideClick, false) - } - return () => { - off("click", onOutsideClick, false) - } - }, [selected]) - - return ( -
showContextMenu(menu)} - > - -
- ) -} diff --git a/src/js/components/LogCell/index.tsx b/src/js/components/LogCell/index.tsx deleted file mode 100644 index 892cd074a4..0000000000 --- a/src/js/components/LogCell/index.tsx +++ /dev/null @@ -1,67 +0,0 @@ -import {typeClassNames} from "app/core/utils/type-class-names" -import classNames from "classnames" -import React, {useState} from "react" -import {zed} from "zealot" -import {RightClickBuilder} from "../../types" -import Tooltip from "../Tooltip" -import CompoundField from "./CompoundField" -import SingleField from "./SingleField" - -type Props = { - field: zed.Field - log: zed.Record - style?: Object - rightClick: RightClickBuilder -} - -const getTooltipStyle = (el: Element) => { - if (!el) return {} - const {top, left} = el.getBoundingClientRect() - return {top: top - 21, left: left + 4} -} - -export default function LogCell({field, style, rightClick, log}: Props) { - const [hover, setHover] = useState(false) - const [tooltipStyle, setTooltipStyle] = useState({}) - const data = field.data - const name = field.name - - function handleMouseEnter(e) { - setHover(true) - setTooltipStyle(getTooltipStyle(e.currentTarget)) - } - - function handleMouseLeave() { - setHover(false) - } - return ( -
- - {hover && ( - - {name} - - )} -
- ) -} - -type FieldSwitchProps = { - field: zed.Field - log: zed.Record - menuBuilder: RightClickBuilder -} - -function FieldSwitch({field, log, menuBuilder}: FieldSwitchProps) { - if (field.data instanceof zed.Primitive) { - const menu = menuBuilder(field, log, false) - return - } else { - return - } -} diff --git a/src/js/components/LogRow.tsx b/src/js/components/LogRow.tsx index 0d43d490d9..cb60aa4a48 100644 --- a/src/js/components/LogRow.tsx +++ b/src/js/components/LogRow.tsx @@ -1,10 +1,11 @@ +import Cell from "app/viewer/cell" +import Value from "app/viewer/value" import classNames from "classnames" import isEqual from "lodash/isEqual" import React, {memo, MouseEvent} from "react" import {zed} from "zealot" import TableColumns from "../models/TableColumns" -import {RightClickBuilder, ViewerDimens} from "../types" -import LogCell from "./LogCell" +import {ViewerDimens} from "../types" import * as Styler from "./Viewer/Styler" type Props = { @@ -17,20 +18,10 @@ type Props = { columns: TableColumns onClick: (e: MouseEvent) => void onDoubleClick: (e: MouseEvent) => void - rightClick: RightClickBuilder } const LogRow = (props: Props) => { - const { - dimens, - highlight, - index, - log, - rightClick, - columns, - onClick, - onDoubleClick - } = props + const {dimens, highlight, index, log, columns, onClick, onDoubleClick} = props const renderCell = (column, colIndex) => { const width = dimens.rowWidth !== "auto" ? column.width || 300 : "auto" @@ -38,13 +29,15 @@ const LogRow = (props: Props) => { const key = `${index}-${colIndex}` if (field && field.data && !(field.data instanceof zed.Record)) { return ( - + + + ) } if (dimens.rowWidth !== "auto") { diff --git a/src/js/components/SearchResults/ResultsTable.tsx b/src/js/components/SearchResults/ResultsTable.tsx index 3f98a6a1d6..5b5756fcf5 100644 --- a/src/js/components/SearchResults/ResultsTable.tsx +++ b/src/js/components/SearchResults/ResultsTable.tsx @@ -1,6 +1,5 @@ import nextPageViewerSearch from "app/search/flows/next-page-viewer-search" import {isEmpty} from "lodash" -import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" import React, {useEffect} from "react" import {connect, useDispatch} from "react-redux" import {zed} from "zealot" @@ -101,10 +100,6 @@ export default function ResultsTable(props: Props) { dispatch(viewLogDetail(logs[index])) dispatch(openLogDetailsWindow()) }} - rightClick={searchFieldContextMenu( - props.program, - props.tableColumns.getColumns().map((c) => c.name) - )} /> ) } diff --git a/src/js/flows/rightclick/cellMenu.test.ts b/src/js/flows/rightclick/cellMenu.test.ts deleted file mode 100644 index e9f1b73fd3..0000000000 --- a/src/js/flows/rightclick/cellMenu.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import {MenuItemConstructorOptions} from "electron" -import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" -import {createRecord} from "test/factories/zed-factory" - -const conn = createRecord({ - _path: "conn", - id: {orig_h: "192.168.0.1"}, - ts: new Date(1234513 * 1000) -}) - -const dns = createRecord({query: "dns.query.yo"}) - -function menuText(menu: MenuItemConstructorOptions[]) { - return menu - .filter((item) => item.enabled) - .map((item) => item.label) - .join(", ") -} - -describe("Log Right Click", () => { - const program = "*" - const columnNames = conn.columns - - test("dns log", () => { - const log = dns - const field = log.getField("query") - const ctxMenu = searchFieldContextMenu(program, columnNames)( - field, - log, - false - ) - - expect(menuText(ctxMenu)).toMatch(/virustotal/i) - expect(menuText(ctxMenu)).toMatch(/count by/i) - }) - - test("time field for conn log", () => { - const log = conn - const field = log.getField("ts") - const ctxMenu = searchFieldContextMenu(program, columnNames)( - field, - log, - false - ) - - expect(menuText(ctxMenu)).toMatch(/"start" time/i) - expect(menuText(ctxMenu)).toMatch(/"end" time/i) - }) -}) - -describe("Analysis Right Click", () => { - const program = "* | count() by id.orig_h" - const columnNames = ["count", "id.orig_h"] - - test("nested field", () => { - const log = createRecord({ - count: 300, - id: {orig_h: "192.168.0.51"} - }) - const field = log.getField("id.orig_h") - const ctxMenu = searchFieldContextMenu(program, columnNames)( - field, - log, - false - ) - - expect(menuText(ctxMenu)).toMatch(/whois/i) - }) - - test("non-address field", () => { - const log = createRecord({count: 100, proto: "tcp"}) - const field = log.getField("proto") - const ctxMenu = searchFieldContextMenu("* | count() by proto", [ - "count", - "proto" - ])(field, log, false) - - expect(menuText(ctxMenu)).toMatch(/pivot/i) - }) -}) From 0642a3490998eaf6942fc50af59608d8d3ff5ffe Mon Sep 17 00:00:00 2001 From: James Kerr Date: Sun, 2 May 2021 17:02:42 -0700 Subject: [PATCH 20/41] Fixed ingest itest --- app/core/formatters/format-zed.ts | 8 ++ app/viewer/value.tsx | 7 +- itest/tests/__snapshots__/ingest.test.ts.snap | 120 +++++++++--------- src/js/zql/toZql.ts | 4 +- zealot/zed/utils.ts | 6 + zealot/zed/values/time.test.ts | 7 + zealot/zed/values/time.ts | 19 ++- 7 files changed, 105 insertions(+), 66 deletions(-) diff --git a/app/core/formatters/format-zed.ts b/app/core/formatters/format-zed.ts index e8464a9ee0..b444fb5f44 100644 --- a/app/core/formatters/format-zed.ts +++ b/app/core/formatters/format-zed.ts @@ -6,5 +6,13 @@ export function formatPrimitive(data: zed.Primitive) { if (data.isUnset()) return "⦻" if (zed.isInt(data)) return withCommas(data.toString()) if (zed.isTime(data)) return brim.time(data.toDate()).format() + if (zed.isStringy(data)) { + // only whitespace + if (data.toString().match(/^\s*$/)) { + return `"${data.toString()}"` + } else { + return data.toString() + } + } return data.toString() } diff --git a/app/viewer/value.tsx b/app/viewer/value.tsx index 540e6c7ca6..19955d40e8 100644 --- a/app/viewer/value.tsx +++ b/app/viewer/value.tsx @@ -13,6 +13,7 @@ const transHavelock = transparentize(0.75, havelock as string) const BG = styled.span` cursor: default; display: inline-block; + min-width: 7px; &:hover { background: ${transHavelock}; border-radius: 4px; @@ -30,7 +31,11 @@ type ValueProps = { padAfter?: boolean } -const pad = (bool) => (bool ? <> : null) +const Space = styled.span` + display: inline-block; + width: 7px; +` +const pad = (bool) => (bool ? : null) export default function Value(props: ValueProps) { if (props.value.isUnset()) { diff --git a/itest/tests/__snapshots__/ingest.test.ts.snap b/itest/tests/__snapshots__/ingest.test.ts.snap index 700fd4537f..6114810a68 100644 --- a/itest/tests/__snapshots__/ingest.test.ts.snap +++ b/itest/tests/__snapshots__/ingest.test.ts.snap @@ -47,91 +47,91 @@ Array [ "proto", "2020-02-25T16:03:05.983", "192.168.1.110", - "56625", + "56,625", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:06.154", "192.168.1.110", - "57591", + "57,591", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55635", + "55,635", "18.246.31.137", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55747", + "55,747", "13.52.5.22", "443", "tcp", "2020-02-25T16:03:08.210", "192.168.1.110", - "55344", + "55,344", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.334", "192.168.1.110", - "55346", + "55,346", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.449", "192.168.1.110", - "57326", + "57,326", "173.194.201.189", "443", "tcp", "2020-02-25T16:03:08.472", "192.168.1.110", - "57487", + "57,487", "192.30.253.125", "443", "tcp", "2020-02-25T16:03:08.807", "192.168.1.110", - "55354", + "55,354", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:10.264", "192.168.1.110", - "57332", + "57,332", "64.233.179.189", "443", "tcp", "2020-02-25T16:03:10.938", "192.168.1.110", - "55351", + "55,351", "18.205.93.211", "443", "tcp", "2020-02-25T16:03:12.860", "192.168.1.110", - "57635", + "57,635", "17.125.252.5", "443", "tcp", "2020-02-25T16:03:13.996", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "tcp", "2020-02-25T16:03:15.784", "192.168.1.110", - "57572", + "57,572", "172.217.6.138", "443", "tcp", "2020-02-25T16:03:15.986", "192.168.1.110", - "57540", + "57,540", "172.217.1.138", "443", "tcp", @@ -148,91 +148,91 @@ Array [ "proto", "2020-02-25T16:03:05.983", "192.168.1.110", - "56625", + "56,625", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:06.154", "192.168.1.110", - "57591", + "57,591", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55635", + "55,635", "18.246.31.137", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55747", + "55,747", "13.52.5.22", "443", "tcp", "2020-02-25T16:03:08.210", "192.168.1.110", - "55344", + "55,344", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.334", "192.168.1.110", - "55346", + "55,346", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.449", "192.168.1.110", - "57326", + "57,326", "173.194.201.189", "443", "tcp", "2020-02-25T16:03:08.472", "192.168.1.110", - "57487", + "57,487", "192.30.253.125", "443", "tcp", "2020-02-25T16:03:08.807", "192.168.1.110", - "55354", + "55,354", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:10.264", "192.168.1.110", - "57332", + "57,332", "64.233.179.189", "443", "tcp", "2020-02-25T16:03:10.938", "192.168.1.110", - "55351", + "55,351", "18.205.93.211", "443", "tcp", "2020-02-25T16:03:12.860", "192.168.1.110", - "57635", + "57,635", "17.125.252.5", "443", "tcp", "2020-02-25T16:03:13.996", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "tcp", "2020-02-25T16:03:15.784", "192.168.1.110", - "57572", + "57,572", "172.217.6.138", "443", "tcp", "2020-02-25T16:03:15.986", "192.168.1.110", - "57540", + "57,540", "172.217.1.138", "443", "tcp", @@ -249,91 +249,91 @@ Array [ "proto", "2020-02-25T16:03:05.983", "192.168.1.110", - "56625", + "56,625", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:06.154", "192.168.1.110", - "57591", + "57,591", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55635", + "55,635", "18.246.31.137", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55747", + "55,747", "13.52.5.22", "443", "tcp", "2020-02-25T16:03:08.210", "192.168.1.110", - "55344", + "55,344", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.334", "192.168.1.110", - "55346", + "55,346", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.449", "192.168.1.110", - "57326", + "57,326", "173.194.201.189", "443", "tcp", "2020-02-25T16:03:08.472", "192.168.1.110", - "57487", + "57,487", "192.30.253.125", "443", "tcp", "2020-02-25T16:03:08.807", "192.168.1.110", - "55354", + "55,354", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:10.264", "192.168.1.110", - "57332", + "57,332", "64.233.179.189", "443", "tcp", "2020-02-25T16:03:10.938", "192.168.1.110", - "55351", + "55,351", "18.205.93.211", "443", "tcp", "2020-02-25T16:03:12.860", "192.168.1.110", - "57635", + "57,635", "17.125.252.5", "443", "tcp", "2020-02-25T16:03:13.996", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "tcp", "2020-02-25T16:03:15.784", "192.168.1.110", - "57572", + "57,572", "172.217.6.138", "443", "tcp", "2020-02-25T16:03:15.986", "192.168.1.110", - "57540", + "57,540", "172.217.1.138", "443", "tcp", @@ -350,91 +350,91 @@ Array [ "proto", "2020-02-25T16:03:05.983", "192.168.1.110", - "56625", + "56,625", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:06.154", "192.168.1.110", - "57591", + "57,591", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55635", + "55,635", "18.246.31.137", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55747", + "55,747", "13.52.5.22", "443", "tcp", "2020-02-25T16:03:08.210", "192.168.1.110", - "55344", + "55,344", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.334", "192.168.1.110", - "55346", + "55,346", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.449", "192.168.1.110", - "57326", + "57,326", "173.194.201.189", "443", "tcp", "2020-02-25T16:03:08.472", "192.168.1.110", - "57487", + "57,487", "192.30.253.125", "443", "tcp", "2020-02-25T16:03:08.807", "192.168.1.110", - "55354", + "55,354", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:10.264", "192.168.1.110", - "57332", + "57,332", "64.233.179.189", "443", "tcp", "2020-02-25T16:03:10.938", "192.168.1.110", - "55351", + "55,351", "18.205.93.211", "443", "tcp", "2020-02-25T16:03:12.860", "192.168.1.110", - "57635", + "57,635", "17.125.252.5", "443", "tcp", "2020-02-25T16:03:13.996", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "tcp", "2020-02-25T16:03:15.784", "192.168.1.110", - "57572", + "57,572", "172.217.6.138", "443", "tcp", "2020-02-25T16:03:15.986", "192.168.1.110", - "57540", + "57,540", "172.217.1.138", "443", "tcp", diff --git a/src/js/zql/toZql.ts b/src/js/zql/toZql.ts index b7b530eccc..26c6f61a4e 100644 --- a/src/js/zql/toZql.ts +++ b/src/js/zql/toZql.ts @@ -30,7 +30,9 @@ function toZqlBool(bool: boolean) { } function toZqlZngPrimitive(data: zed.Primitive) { - if (data instanceof zed.Ip) { + if (data.isUnset()) { + return "null" + } else if (data instanceof zed.Ip) { return data.toString() } else { return toZqlString(data.toString()) diff --git a/zealot/zed/utils.ts b/zealot/zed/utils.ts index 33662b1458..2298c95839 100644 --- a/zealot/zed/utils.ts +++ b/zealot/zed/utils.ts @@ -1,9 +1,11 @@ import {TypeAlias, Uint16, Uint32, Uint64, Uint8} from "./index" import {ZedType} from "./types/types" +import {BString} from "./values/bstring" import {Int16} from "./values/int16" import {Int32} from "./values/int32" import {Int64} from "./values/int64" import {Int8} from "./values/int8" +import {String} from "./values/string" import {Time} from "./values/time" import {ZedInt} from "./values/types" @@ -57,3 +59,7 @@ export function trueType(start: ZedType): T { export function isNull(value): value is null { return value === null } + +export function isStringy(value): value is String | BString { + return value instanceof String || value instanceof BString +} diff --git a/zealot/zed/values/time.test.ts b/zealot/zed/values/time.test.ts index b044e7d23d..4dcb0a5f6b 100644 --- a/zealot/zed/values/time.test.ts +++ b/zealot/zed/values/time.test.ts @@ -15,7 +15,14 @@ test("toDate()", () => { new Time("2020-02-25T16:03:13Z").toDate() }) +test("parse epoch timestamp?", () => {}) + test("create record with time field", () => { const t = createData(new Date(0)) as zed.Time expect(t.toDate()).toEqual(new Date(0)) }) + +test("keeps the milliseconds", () => { + const date = new Time("2020-02-25T16:03:17.838527Z").toDate() + console.log(date) +}) diff --git a/zealot/zed/values/time.ts b/zealot/zed/values/time.ts index 580cbadbcc..535d94a9cf 100644 --- a/zealot/zed/values/time.ts +++ b/zealot/zed/values/time.ts @@ -1,8 +1,12 @@ import { + ChronoField, convert, DateTimeFormatter, + DateTimeFormatterBuilder, LocalDateTime, - nativeJs + nativeJs, + ZonedDateTime, + ZoneId } from "@js-joda/core" import {TypeTime} from "../types/type-time" import {isNull} from "../utils" @@ -40,9 +44,16 @@ export class Time extends Primitive { const parseEpochSec = (v) => { const d = new Date(+v * 1000) if (isNaN(d as any)) throw new Error("Not Epoch Seconds: " + v) - return LocalDateTime.from(nativeJs(d)) + return ZonedDateTime.from(nativeJs(d)) } -const NanoFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss[.n]'Z'") -const parseNano = (v) => LocalDateTime.parse(v, NanoFormat) + +const NanoFormat = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd'T'HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .appendLiteral("Z") + .toFormatter() + +const parseNano = (v) => + ZonedDateTime.of(LocalDateTime.parse(v, NanoFormat), ZoneId.of("UTC")) const PARSERS = [parseNano, parseEpochSec] From d4b768c14688247bd888e524a0390587bfe5ee09 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Sun, 2 May 2021 17:38:36 -0700 Subject: [PATCH 21/41] Fixed query test --- .../__snapshots__/contextMenu.test.ts.snap | 108 ++++++++++++++++++ itest/tests/__snapshots__/query.test.ts.snap | 60 +++++----- itest/tests/contextMenu.test.ts | 4 +- 3 files changed, 141 insertions(+), 31 deletions(-) diff --git a/itest/tests/__snapshots__/contextMenu.test.ts.snap b/itest/tests/__snapshots__/contextMenu.test.ts.snap index e0f01f1260..dfac867693 100644 --- a/itest/tests/__snapshots__/contextMenu.test.ts.snap +++ b/itest/tests/__snapshots__/contextMenu.test.ts.snap @@ -74,6 +74,15 @@ Array [ ] `; +exports[`type-wise Filter = value searches FilterEq0001: string scalar="-" 1`] = ` +Array [ + "id", + "scalar", + "16", + "-", +] +`; + exports[`type-wise Filter = value searches FilterEq0002: addr scalar="::" 1`] = ` Array [ "id", @@ -83,6 +92,51 @@ Array [ ] `; +exports[`type-wise Filter = value searches FilterEq0002: string scalar=" " 1`] = ` +Array [ + "id", + "scalar", + "14", + "mystr", + "15", + "(empty)", + "16", + "-", + "17", + "\\" \\"", + "18", + "\\"", + "19", + "'", + "20", + ",", + "21", + ",", + "22", + ";", + "23", + "∫£œßü™", + "24", + "\\"mystr\\"", + "25", + "'mystr'", + "26", + "*", + "27", + "\\"*\\"", + "28", + "1.1.1.1", + "54", + "\\" \\"", + "55", + "\\" \\"", + "56", + "⦻", + "57", + "null", +] +`; + exports[`type-wise Filter = value searches FilterEq0002: string scalar="\\x09" 1`] = ` Array [ "id", @@ -258,6 +312,60 @@ Array [ ] `; +exports[`type-wise Filter = value searches FilterEq0017: string record.scalar="-" 1`] = ` +Array [ + "id", + "record.scalar", + "31", + "-", +] +`; + +exports[`type-wise Filter = value searches FilterEq0018: string record.scalar=" " 1`] = ` +Array [ + "id", + "record.scalar", + "29", + "mystr", + "30", + "(empty)", + "31", + "-", + "32", + "\\" \\"", + "33", + "\\"", + "34", + "'", + "35", + ",", + "36", + ",", + "37", + ";", + "38", + "∫£œßü™", + "39", + "\\"mystr\\"", + "40", + "'mystr'", + "41", + "*", + "42", + "\\"*\\"", + "43", + "1.1.1.1", + "58", + "\\"\\"", + "59", + "\\" \\"", + "60", + "⦻", + "61", + "null", +] +`; + exports[`type-wise Filter = value searches FilterEq0018: string record.scalar="\\x09" 1`] = ` Array [ "id", diff --git a/itest/tests/__snapshots__/query.test.ts.snap b/itest/tests/__snapshots__/query.test.ts.snap index 264a98694a..b643e0ecca 100644 --- a/itest/tests/__snapshots__/query.test.ts.snap +++ b/itest/tests/__snapshots__/query.test.ts.snap @@ -47,115 +47,115 @@ Array [ "proto", "2020-02-25T16:03:05.983", "192.168.1.110", - "56625", + "56,625", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:06.154", "192.168.1.110", - "57591", + "57,591", "172.217.9.142", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55635", + "55,635", "18.246.31.137", "443", "tcp", "2020-02-25T16:03:07.715", "192.168.1.110", - "55747", + "55,747", "13.52.5.22", "443", "tcp", "2020-02-25T16:03:08.210", "192.168.1.110", - "55344", + "55,344", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.334", "192.168.1.110", - "55346", + "55,346", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:08.449", "192.168.1.110", - "57326", + "57,326", "173.194.201.189", "443", "tcp", "2020-02-25T16:03:08.472", "192.168.1.110", - "57487", + "57,487", "192.30.253.125", "443", "tcp", "2020-02-25T16:03:08.807", "192.168.1.110", - "55354", + "55,354", "52.37.243.173", "443", "tcp", "2020-02-25T16:03:09.440", "192.168.1.179", - "47783", + "47,783", "192.168.1.255", - "15600", + "15,600", "udp", "2020-02-25T16:03:10.264", "192.168.1.110", - "57332", + "57,332", "64.233.179.189", "443", "tcp", "2020-02-25T16:03:10.938", "192.168.1.110", - "55351", + "55,351", "18.205.93.211", "443", "tcp", "2020-02-25T16:03:11.275", "192.168.1.110", - "54375", + "54,375", "192.168.1.254", "53", "udp", "2020-02-25T16:03:12.860", "192.168.1.110", - "57635", + "57,635", "17.125.252.5", "443", "tcp", "2020-02-25T16:03:13.978", "192.168.1.110", - "51848", + "51,848", "192.168.1.254", "53", "udp", "2020-02-25T16:03:13.996", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "tcp", "2020-02-25T16:03:15.481", "192.168.1.179", - "51524", + "51,524", "192.168.1.255", - "15600", + "15,600", "udp", "2020-02-25T16:03:15.784", "192.168.1.110", - "57572", + "57,572", "172.217.6.138", "443", "tcp", "2020-02-25T16:03:15.986", "192.168.1.110", - "57540", + "57,540", "172.217.1.138", "443", "tcp", @@ -239,20 +239,19 @@ Array [ "ssl", "CeYm4A4XtVPVfF9wo6", "192.168.1.110", - "57640", + "57,640", "209.216.230.240", "443", "TLSv12", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "secp256r1", "news.ycombinator.com", - "F", + "false", "⦻", "http/1.1", - "T", - "FXkLzR25cwYgeI5yI7", - "Frbo5s1CRsV02JUoSe", - "F1VUER3CattO8PiHWc", + "true", + "[FXkLzR25cwYgeI5yI7,Frbo5s1CRsV02JUoSe,F1VUER3CattO8PiHWc]", + "[]", "CN=news.ycombinator.com,O=Y Combinator\\\\, Inc.,L=San Francisco,ST=California,C=US", "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US", "⦻", @@ -273,8 +272,11 @@ Array [ "2,048", "65537", "⦻", - "news.ycombinator.com", - "F", + "[news.ycombinator.com]", + "⦻", + "⦻", + "⦻", + "false", "⦻", ] `; diff --git a/itest/tests/contextMenu.test.ts b/itest/tests/contextMenu.test.ts index 38f6ce34cd..f38d1d48a0 100644 --- a/itest/tests/contextMenu.test.ts +++ b/itest/tests/contextMenu.test.ts @@ -16,8 +16,8 @@ const FIELDS = ["scalar", "record.scalar"] const UNSET = "⦻" const STRINGS = [ "mystr", - "\\x2d", - "\\x09", + "-", //"\\x2d", + "\t", // "\\x09", '"', "'", ",", From 2170253386ce12b737137d02a6df1baa6f29ec8c Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 13:03:46 -0700 Subject: [PATCH 22/41] Refactor context menu integration tests --- app/core/formatters/format-zed.ts | 4 +- app/viewer/value.tsx | 3 +- itest/lib/createTestBrim.ts | 21 +- .../__snapshots__/contextMenu.test.ts.snap | 432 ++++-------------- itest/tests/contextMenu.test.ts | 163 +++---- package.json | 3 +- src/js/test/createLocator.ts | 4 +- 7 files changed, 172 insertions(+), 458 deletions(-) diff --git a/app/core/formatters/format-zed.ts b/app/core/formatters/format-zed.ts index b444fb5f44..df11ca3eb2 100644 --- a/app/core/formatters/format-zed.ts +++ b/app/core/formatters/format-zed.ts @@ -9,9 +9,9 @@ export function formatPrimitive(data: zed.Primitive) { if (zed.isStringy(data)) { // only whitespace if (data.toString().match(/^\s*$/)) { - return `"${data.toString()}"` + // return `"${data.toString()}"` } else { - return data.toString() + // return data.toString() } } return data.toString() diff --git a/app/viewer/value.tsx b/app/viewer/value.tsx index 19955d40e8..32f5129149 100644 --- a/app/viewer/value.tsx +++ b/app/viewer/value.tsx @@ -1,9 +1,10 @@ import {formatPrimitive} from "app/core/formatters/format-zed" import {typeClassNames} from "app/core/utils/type-class-names" -import {cssVar, transparentize} from "polished" +import {transparentize} from "polished" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" import React from "react" import {useDispatch} from "react-redux" +import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" import {zed} from "zealot" diff --git a/itest/lib/createTestBrim.ts b/itest/lib/createTestBrim.ts index e57f0aec76..62c897f6d3 100644 --- a/itest/lib/createTestBrim.ts +++ b/itest/lib/createTestBrim.ts @@ -1,7 +1,9 @@ +import {isString} from "lodash" import {htmlContextMenu} from "src/js/test/locators" import lib from "../../src/js/lib" import {Locator} from "../../src/js/test/createLocator" import appStep from "./appStep/api" +import {getResults} from "./appStep/api/search" import takeScreenshot from "./appStep/api/takeScreenshot" import waitForHook from "./appStep/api/waitForHook" // TODO in a future PR: remove direct logStep uses here. @@ -23,8 +25,9 @@ export default (name: string) => { }) return { - $(locator: Locator) { - return app.client.$(locator.css) + $(locator: Locator | string): WebdriverIO.Element { + if (isString(locator)) return app.client.$(locator) + else return app.client.$(locator.css) }, hook(name, opts?) { @@ -93,8 +96,14 @@ export default (name: string) => { return appStep.click(app, locator.css) }, - rightClick(locator: Locator) { - return appStep.rightClick(app, locator.css) + rightClick(locator: Locator | string | WebdriverIO.Element) { + if (isString(locator)) { + return appStep.rightClick(app, locator) + } else if ("css" in locator) { + return appStep.rightClick(app, locator.css) + } else { + return locator.click({button: "right"}) + } }, hasText(input: string | RegExp, locator: Locator | string = "body") { @@ -125,6 +134,10 @@ export default (name: string) => { wait(ms: number) { return lib.sleep(ms) + }, + + viewerResults(opts: {headers: boolean} = {headers: true}) { + return getResults(app, opts.headers) } } } diff --git a/itest/tests/__snapshots__/contextMenu.test.ts.snap b/itest/tests/__snapshots__/contextMenu.test.ts.snap index dfac867693..1bf988d23f 100644 --- a/itest/tests/__snapshots__/contextMenu.test.ts.snap +++ b/itest/tests/__snapshots__/contextMenu.test.ts.snap @@ -1,44 +1,15 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`type-wise Filter = value searches FilterEq ensure ingest 1`] = ` -Array [ - "_path", - "count", - "addr", - "11", - "string", - "48", -] -`; - -exports[`type-wise Filter = value searches FilterEq unset/⦻ string 1`] = ` +exports[`context menu tests rightclick scalar addrs :: 1`] = ` Array [ "id", "scalar", - "61", - "⦻", - "60", - "⦻", - "59", - "⦻", - "58", - "⦻", - "53", - "⦻", - "52", - "⦻", - "51", - "⦻", - "50", - "⦻", - "49", - "⦻", - "48", - "⦻", + "2", + "::", ] `; -exports[`type-wise Filter = value searches FilterEq0000: addr scalar="1.1.1.1" 1`] = ` +exports[`context menu tests rightclick scalar addrs 1.1.1.1 1`] = ` Array [ "id", "scalar", @@ -47,16 +18,7 @@ Array [ ] `; -exports[`type-wise Filter = value searches FilterEq0000: string scalar="mystr" 1`] = ` -Array [ - "id", - "scalar", - "14", - "mystr", -] -`; - -exports[`type-wise Filter = value searches FilterEq0001: addr scalar="fe80::58d2:2d09:e8cb:a8ad" 1`] = ` +exports[`context menu tests rightclick scalar addrs fe80::58d2:2d09:e8cb:a8ad 1`] = ` Array [ "id", "scalar", @@ -65,133 +27,72 @@ Array [ ] `; -exports[`type-wise Filter = value searches FilterEq0001: string scalar="\\x2d" 1`] = ` +exports[`context menu tests rightclick scalar strings " 1`] = ` Array [ "id", "scalar", - "16", - "\\\\x2d", + "18", + "\\"", ] `; -exports[`type-wise Filter = value searches FilterEq0001: string scalar="-" 1`] = ` +exports[`context menu tests rightclick scalar strings "*" 1`] = ` Array [ "id", "scalar", - "16", - "-", + "27", + "\\"*\\"", ] `; -exports[`type-wise Filter = value searches FilterEq0002: addr scalar="::" 1`] = ` +exports[`context menu tests rightclick scalar strings "mystr" 1`] = ` Array [ "id", "scalar", - "2", - "::", + "24", + "\\"mystr\\"", ] `; -exports[`type-wise Filter = value searches FilterEq0002: string scalar=" " 1`] = ` +exports[`context menu tests rightclick scalar strings ' ' (space) 1`] = ` Array [ "id", "scalar", - "14", - "mystr", - "15", - "(empty)", - "16", - "-", - "17", - "\\" \\"", - "18", - "\\"", - "19", - "'", - "20", - ",", - "21", - ",", - "22", - ";", - "23", - "∫£œßü™", - "24", - "\\"mystr\\"", - "25", - "'mystr'", - "26", - "*", - "27", - "\\"*\\"", - "28", - "1.1.1.1", "54", - "\\" \\"", + "", "55", - "\\" \\"", - "56", - "⦻", - "57", - "null", + "", ] `; -exports[`type-wise Filter = value searches FilterEq0002: string scalar="\\x09" 1`] = ` +exports[`context menu tests rightclick scalar strings ' 1`] = ` Array [ "id", "scalar", - "17", - "\\\\x09", -] -`; - -exports[`type-wise Filter = value searches FilterEq0003: addr record.scalar="1.1.1.1" 1`] = ` -Array [ - "id", - "record.scalar", - "5", - "1.1.1.1", + "19", + "'", ] `; -exports[`type-wise Filter = value searches FilterEq0003: string scalar=""" 1`] = ` +exports[`context menu tests rightclick scalar strings 'mystr' 1`] = ` Array [ "id", "scalar", - "18", - "\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0004: addr record.scalar="fe80::58d2:2d09:e8cb:a8ad" 1`] = ` -Array [ - "id", - "record.scalar", - "7", - "fe80::58d2:2d09:e8cb:a8ad", + "25", + "'mystr'", ] `; -exports[`type-wise Filter = value searches FilterEq0004: string scalar="'" 1`] = ` +exports[`context menu tests rightclick scalar strings * 1`] = ` Array [ "id", "scalar", - "19", - "'", -] -`; - -exports[`type-wise Filter = value searches FilterEq0005: addr record.scalar="::" 1`] = ` -Array [ - "id", - "record.scalar", - "6", - "::", + "26", + "*", ] `; -exports[`type-wise Filter = value searches FilterEq0005: string scalar="," 1`] = ` +exports[`context menu tests rightclick scalar strings , 1`] = ` Array [ "id", "scalar", @@ -202,7 +103,7 @@ Array [ ] `; -exports[`type-wise Filter = value searches FilterEq0006: string scalar=";" 1`] = ` +exports[`context menu tests rightclick scalar strings ; 1`] = ` Array [ "id", "scalar", @@ -211,285 +112,128 @@ Array [ ] `; -exports[`type-wise Filter = value searches FilterEq0007: string scalar="∫£œßü™" 1`] = ` +exports[`context menu tests rightclick scalar strings \\t (tab) 1`] = ` Array [ "id", "scalar", + "14", + "mystr", + "15", + "(empty)", + "16", + "-", + "17", + "", + "18", + "\\"", + "19", + "'", + "20", + ",", + "21", + ",", + "22", + ";", "23", "∫£œßü™", -] -`; - -exports[`type-wise Filter = value searches FilterEq0008: string scalar=""mystr"" 1`] = ` -Array [ - "id", - "scalar", "24", "\\"mystr\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0009: string scalar="'mystr'" 1`] = ` -Array [ - "id", - "scalar", "25", "'mystr'", -] -`; - -exports[`type-wise Filter = value searches FilterEq0010: string scalar="*" 1`] = ` -Array [ - "id", - "scalar", "26", "*", -] -`; - -exports[`type-wise Filter = value searches FilterEq0011: string scalar=""*"" 1`] = ` -Array [ - "id", - "scalar", "27", "\\"*\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0012: string scalar="1.1.1.1" 1`] = ` -Array [ - "id", - "scalar", "28", "1.1.1.1", -] -`; - -exports[`type-wise Filter = value searches FilterEq0013: string scalar=" " 1`] = ` -Array [ - "id", - "scalar", "54", "", "55", "", -] -`; - -exports[`type-wise Filter = value searches FilterEq0014: string scalar="null" 1`] = ` -Array [ - "id", - "scalar", + "56", + "⦻", "57", "null", ] `; -exports[`type-wise Filter = value searches FilterEq0015: string scalar="⦻" 1`] = ` +exports[`context menu tests rightclick scalar strings - 1`] = ` Array [ "id", "scalar", - "56", - "⦻", -] -`; - -exports[`type-wise Filter = value searches FilterEq0016: string record.scalar="mystr" 1`] = ` -Array [ - "id", - "record.scalar", - "29", - "mystr", -] -`; - -exports[`type-wise Filter = value searches FilterEq0017: string record.scalar="\\x2d" 1`] = ` -Array [ - "id", - "record.scalar", - "31", - "\\\\x2d", -] -`; - -exports[`type-wise Filter = value searches FilterEq0017: string record.scalar="-" 1`] = ` -Array [ - "id", - "record.scalar", - "31", + "16", "-", ] `; -exports[`type-wise Filter = value searches FilterEq0018: string record.scalar=" " 1`] = ` +exports[`context menu tests rightclick scalar strings 1.1.1.1 1`] = ` Array [ "id", - "record.scalar", - "29", - "mystr", - "30", - "(empty)", - "31", - "-", - "32", - "\\" \\"", - "33", - "\\"", - "34", - "'", - "35", - ",", - "36", - ",", - "37", - ";", - "38", - "∫£œßü™", - "39", - "\\"mystr\\"", - "40", - "'mystr'", - "41", - "*", - "42", - "\\"*\\"", - "43", + "scalar", + "28", "1.1.1.1", - "58", - "\\"\\"", - "59", - "\\" \\"", - "60", - "⦻", - "61", - "null", -] -`; - -exports[`type-wise Filter = value searches FilterEq0018: string record.scalar="\\x09" 1`] = ` -Array [ - "id", - "record.scalar", - "32", - "\\\\x09", ] `; -exports[`type-wise Filter = value searches FilterEq0019: string record.scalar=""" 1`] = ` +exports[`context menu tests rightclick scalar strings mystr 1`] = ` Array [ "id", - "record.scalar", - "33", - "\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0020: string record.scalar="'" 1`] = ` -Array [ - "id", - "record.scalar", - "34", - "'", -] -`; - -exports[`type-wise Filter = value searches FilterEq0021: string record.scalar="," 1`] = ` -Array [ - "id", - "record.scalar", - "35", - ",", - "36", - ",", + "scalar", + "14", + "mystr", ] `; -exports[`type-wise Filter = value searches FilterEq0022: string record.scalar=";" 1`] = ` +exports[`context menu tests rightclick scalar strings null 1`] = ` Array [ "id", - "record.scalar", - "37", - ";", + "scalar", + "57", + "null", ] `; -exports[`type-wise Filter = value searches FilterEq0023: string record.scalar="∫£œßü™" 1`] = ` +exports[`context menu tests rightclick scalar strings ∫£œßü™ 1`] = ` Array [ "id", - "record.scalar", - "38", + "scalar", + "23", "∫£œßü™", ] `; -exports[`type-wise Filter = value searches FilterEq0024: string record.scalar=""mystr"" 1`] = ` -Array [ - "id", - "record.scalar", - "39", - "\\"mystr\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0025: string record.scalar="'mystr'" 1`] = ` -Array [ - "id", - "record.scalar", - "40", - "'mystr'", -] -`; - -exports[`type-wise Filter = value searches FilterEq0026: string record.scalar="*" 1`] = ` -Array [ - "id", - "record.scalar", - "41", - "*", -] -`; - -exports[`type-wise Filter = value searches FilterEq0027: string record.scalar=""*"" 1`] = ` -Array [ - "id", - "record.scalar", - "42", - "\\"*\\"", -] -`; - -exports[`type-wise Filter = value searches FilterEq0028: string record.scalar="1.1.1.1" 1`] = ` -Array [ - "id", - "record.scalar", - "43", - "1.1.1.1", -] -`; - -exports[`type-wise Filter = value searches FilterEq0029: string record.scalar=" " 1`] = ` +exports[`context menu tests rightclick scalar strings ⦻ 1`] = ` Array [ "id", - "record.scalar", - "59", - "", + "scalar", + "56", + "⦻", ] `; -exports[`type-wise Filter = value searches FilterEq0030: string record.scalar="null" 1`] = ` +exports[`context menu tests rightclick scalar unset unset 1`] = ` Array [ "id", - "record.scalar", + "scalar", "61", - "null", -] -`; - -exports[`type-wise Filter = value searches FilterEq0031: string record.scalar="⦻" 1`] = ` -Array [ - "id", - "record.scalar", + "⦻", "60", "⦻", + "59", + "⦻", + "58", + "⦻", + "53", + "⦻", + "52", + "⦻", + "51", + "⦻", + "50", + "⦻", + "49", + "⦻", + "48", + "⦻", ] `; diff --git a/itest/tests/contextMenu.test.ts b/itest/tests/contextMenu.test.ts index f38d1d48a0..0067b13980 100644 --- a/itest/tests/contextMenu.test.ts +++ b/itest/tests/contextMenu.test.ts @@ -1,117 +1,70 @@ -import {basename} from "path" -import {sprintf} from "sprintf-js" +import createTestBrim from "itest/lib/createTestBrim" +import {viewerResults} from "src/js/test/locators" -import {getResults, runSearch} from "../lib/appStep/api/search" -import appStep from "../lib/appStep/api" -import newAppInstance from "../lib/newAppInstance" -import {handleError} from "../lib/jest" -import {selectors} from "../../src/js/test/integration" +describe("context menu tests", () => { + let brim = createTestBrim("context-menu-test") -// Fields we can test in types.tsv -const FIELDS = ["scalar", "record.scalar"] -// Brim character that represents an unset zng value. It must be -// handled specially as it can be a string value or the unset -// representation. That representation is only used to find a cell, or -// to exclude the actual character from search results. -const UNSET = "⦻" -const STRINGS = [ - "mystr", - "-", //"\\x2d", - "\t", // "\\x09", - '"', - "'", - ",", - ";", - "∫£œßü™", - '"mystr"', - "'mystr'", - "*", - '"*"', - "1.1.1.1", - " ", - "null", - "⦻" // The actual character, not unset. -] -const ADDRS = ["1.1.1.1", "fe80::58d2:2d09:e8cb:a8ad", "::"] + beforeAll(async () => { + await brim.ingest("types.tsv") + }) -describe("type-wise Filter = value searches", () => { - let app + const quote = (v) => (v.includes('"') ? `'${v}'` : `"${v}"`) + const cellContaining = (text) => + viewerResults.xpath + `//span[contains(text(), ${quote(text)})]` - beforeAll(async (done) => { - app = newAppInstance(basename(__filename) + "-types", 0) - await appStep.startApp(app) - await appStep.ingestFile(app, "types.tsv") - await appStep.setSpan(app, "Whole Space") - done() - }) + async function runTest(query, cell, rightClick) { + await brim.search(query) + await brim.rightClick(cell) + await brim.clickContextMenuItem(rightClick) + expect(await brim.viewerResults()).toMatchSnapshot() + } - afterAll(() => { - if (app && app.isRunning()) { - return app.stop() + describe("rightclick scalar strings", () => { + function scalarString(value: string): () => Promise { + const path = "string" + const fieldName = "scalar" + const query = `_path=${path} ${fieldName}!=null | cut id, ${fieldName} | sort id` + const cell = cellContaining(value) + return () => runTest(query, cell, "Filter = value") } + test("mystr", scalarString("mystr")) + test("-", scalarString("-")) + test('"', scalarString('"')) + test("'", scalarString("'")) + test(",", scalarString(",")) + test(";", scalarString(";")) + test("∫£œßü™", scalarString("∫£œßü™")) + test('"mystr"', scalarString('"mystr"')) + test("'mystr'", scalarString("'mystr'")) + test("*", scalarString("*")) + test('"*"', scalarString('"*"')) + test("1.1.1.1", scalarString("1.1.1.1")) + test("null", scalarString("null")) + test("⦻", scalarString("⦻")) + test("\\t (tab)", scalarString("\t")) + test("' ' (space)", scalarString(" ")) }) - test("FilterEq ensure ingest", (done) => { - appStep - .search(app, `* | count() by _path | sort _path`) - .then((results) => { - expect(results).toMatchSnapshot() - done() - }) - .catch((err) => { - handleError(app, err, done) - }) + describe("rightclick scalar addrs", () => { + function scalarAddr(value) { + const path = "addr" + const fieldName = "scalar" + const query = `_path=${path} ${fieldName}!=null | cut id, ${fieldName} | sort id` + const cell = cellContaining(value) + return () => runTest(query, cell, "Filter = value") + } + test("1.1.1.1", scalarAddr("1.1.1.1")) + test("fe80::58d2:2d09:e8cb:a8ad", scalarAddr("fe80::58d2:2d09:e8cb:a8ad")) + test("::", scalarAddr("::")) }) - const run = (path: string, values: string[]) => { - let testIdx = 0 - FIELDS.forEach((fieldName) => { - values.forEach((s) => { - const testId = sprintf("%04d", testIdx++) - test(`FilterEq${testId}: ${path} ${fieldName}="${s}"`, (done) => { - runSearch( - app, - `_path=${path} ${fieldName}!=null | cut id, ${fieldName} | sort id` - ) - .then(async () => { - await appStep.rightClick( - app, - selectors.viewer.resultCellContaining(s) - ) - await appStep.click( - app, - selectors.viewer.contextMenuItem("Filter = value") - ) - expect(await getResults(app)).toMatchSnapshot() - done() - }) - .catch((err) => { - handleError(app, err, done) - }) - }) - }) - }) - } - run("string", STRINGS) - - test(`FilterEq unset/${UNSET} string`, (done) => { - runSearch(app, `_path=string | cut id, scalar | sort -r id | head 10`) - .then(async () => { - await appStep.rightClick( - app, - selectors.viewer.resultCellContaining(UNSET) - ) - await appStep.click( - app, - selectors.viewer.contextMenuItem("Filter = value") - ) - expect(await getResults(app)).toMatchSnapshot() - done() - }) - .catch((err) => { - handleError(app, err, done) - }) + describe("rightclick scalar unset", () => { + const UNSET = "⦻" + function scalarUnset(value) { + const query = `_path=string | cut id, scalar | sort -r id | head 10` + const cell = cellContaining(value) + return () => runTest(query, cell, "Filter = value") + } + test("unset", scalarUnset(UNSET)) }) - - run("addr", ADDRS) }) diff --git a/package.json b/package.json index d04e45420f..92b3a674d5 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "format-check": "npm run format && git diff --exit-code -- src itest", "release": "npx electron-builder --mac", "prerelease": "npm run build", - "postinstall": "node scripts/post-install" + "postinstall": "node scripts/post-install", + "watch": "node scripts/watch" }, "devDependencies": { "@babel/cli": "^7.10.5", diff --git a/src/js/test/createLocator.ts b/src/js/test/createLocator.ts index 58aee2b086..2145c1e210 100644 --- a/src/js/test/createLocator.ts +++ b/src/js/test/createLocator.ts @@ -1,11 +1,13 @@ export type Locator = { props: {"data-test-locator": string} css: string + xpath: string } export default function createLocator(name: string): Locator { return { props: {"data-test-locator": name}, - css: `[data-test-locator="${name}"]` + css: `[data-test-locator="${name}"]`, + xpath: `//*[@data-test-locator="${name}"]` } } From 59ce8ffad6f238eaa4a4cec33ef9f59b500a9333 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 13:23:22 -0700 Subject: [PATCH 23/41] Fix pcap integration tests --- itest/tests/pcaps.test.ts | 4 ++-- plugins/brimcap/brimcap-plugin.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/itest/tests/pcaps.test.ts b/itest/tests/pcaps.test.ts index 215ff9ba3b..35e5e25cc2 100644 --- a/itest/tests/pcaps.test.ts +++ b/itest/tests/pcaps.test.ts @@ -47,7 +47,7 @@ describe("Test PCAPs", () => { .then(async () => { await appStep.click(app, selectors.viewer.resultCellContaining("ssl")) await appStep.savePcap(app) - const fileBasename = "packets-1582646593.996366.pcap" + const fileBasename = "packets-2020-02-25T16:03:13.996366Z.pcap" const pcapAbspath = path.join(await pcapsDir(app), fileBasename) expect(md5(readFileSync(pcapAbspath))).toBe( "888453c81738fd8ade4c7f9888d86f86" @@ -64,7 +64,7 @@ describe("Test PCAPs", () => { .then(async () => { await appStep.click(app, selectors.viewer.resultCellContaining("conn")) await appStep.savePcap(app) - const fileBasename = "packets-1582646589.440467.pcap" + const fileBasename = "packets-2020-02-25T16:03:09.440467Z.pcap" const pcapAbspath = path.join(await pcapsDir(app), fileBasename) expect(md5(readFileSync(pcapAbspath))).toBe( "678442857027fdc5ad1e3418614dcdb8" diff --git a/plugins/brimcap/brimcap-plugin.ts b/plugins/brimcap/brimcap-plugin.ts index 9571625986..9c9437a682 100644 --- a/plugins/brimcap/brimcap-plugin.ts +++ b/plugins/brimcap/brimcap-plugin.ts @@ -183,7 +183,7 @@ export default class BrimcapPlugin { return { dstIp: log.get("id.resp_h").toString(), dstPort: log.get("id.resp_p").toString(), - duration: dur.toString(), + duration: dur.isSet() ? dur.toString() : "0s", proto: log.get("proto").toString(), root: this.brimcapDataRoot, srcIp: log.get("id.orig_h").toString(), From 3234691ca6033859efd5e4cc6a72f5ceaac8bab8 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 13:28:33 -0700 Subject: [PATCH 24/41] Fixed unit tests --- .../histogram-search.test.ts.snap | 24 +++++++++---------- ppl/detail/flows/fetch.test.ts | 2 +- src/css/_log-viewer.scss | 5 ++-- src/css/shared/_type-colors.scss | 2 +- zealot/zed/values/time.test.ts | 4 +--- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/app/search/flows/__snapshots__/histogram-search.test.ts.snap b/app/search/flows/__snapshots__/histogram-search.test.ts.snap index 706b0407cf..d986d20ab6 100644 --- a/app/search/flows/__snapshots__/histogram-search.test.ts.snap +++ b/app/search/flows/__snapshots__/histogram-search.test.ts.snap @@ -13,46 +13,46 @@ Object { "stats", ], "table": Object { - "1582675385000": Object { + "1582646585000": Object { "conn": 1, "stats": 1, }, - "1582675386000": Object { + "1582646586000": Object { "conn": 1, "weird": 1, }, - "1582675387000": Object { + "1582646587000": Object { "conn": 2, }, - "1582675388000": Object { + "1582646588000": Object { "conn": 5, }, - "1582675389000": Object { + "1582646589000": Object { "conn": 1, }, - "1582675390000": Object { + "1582646590000": Object { "conn": 2, }, - "1582675391000": Object { + "1582646591000": Object { "conn": 1, "dns": 1, }, - "1582675392000": Object { + "1582646592000": Object { "conn": 1, }, - "1582675393000": Object { + "1582646593000": Object { "conn": 2, "dns": 1, }, - "1582675394000": Object { + "1582646594000": Object { "files": 4, "ssl": 1, "x509": 1, }, - "1582675395000": Object { + "1582646595000": Object { "conn": 3, }, - "1582675397000": Object { + "1582646597000": Object { "capture_loss": 1, }, }, diff --git a/ppl/detail/flows/fetch.test.ts b/ppl/detail/flows/fetch.test.ts index 13013f61cb..d4f9758a2b 100644 --- a/ppl/detail/flows/fetch.test.ts +++ b/ppl/detail/flows/fetch.test.ts @@ -17,7 +17,7 @@ const suricata = createRecord({ }) const uidOrCommunityIdZql = - 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" and ts >= 1582675393 and ts < 1582675483.016) | head 100' + 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" or (community_id = "1:N7YGmWjwTmMKNhsZHBR618n3ReA=" and ts >= 1582646593.978 and ts < 1582646683.994) | head 100' const uidZql = 'uid="CbOjYpkXn9LfqV51c" or "CbOjYpkXn9LfqV51c" in conn_uids or "CbOjYpkXn9LfqV51c" in uids or referenced_file.uid="CbOjYpkXn9LfqV51c" | head 100' diff --git a/src/css/_log-viewer.scss b/src/css/_log-viewer.scss index ce79846053..3a318a5a1c 100644 --- a/src/css/_log-viewer.scss +++ b/src/css/_log-viewer.scss @@ -11,7 +11,7 @@ * { @include type-colors; - } + } } .viewer .view { @@ -72,8 +72,7 @@ .int8, .int16, .int32, - .int64 - { + .int64 { text-align: right; } } diff --git a/src/css/shared/_type-colors.scss b/src/css/shared/_type-colors.scss index 2a8c37bbea..7ab3b62d29 100644 --- a/src/css/shared/_type-colors.scss +++ b/src/css/shared/_type-colors.scss @@ -39,6 +39,6 @@ } &.time { - color: var(--interval) + color: var(--interval); } } diff --git a/zealot/zed/values/time.test.ts b/zealot/zed/values/time.test.ts index 4dcb0a5f6b..98a94203a8 100644 --- a/zealot/zed/values/time.test.ts +++ b/zealot/zed/values/time.test.ts @@ -15,8 +15,6 @@ test("toDate()", () => { new Time("2020-02-25T16:03:13Z").toDate() }) -test("parse epoch timestamp?", () => {}) - test("create record with time field", () => { const t = createData(new Date(0)) as zed.Time expect(t.toDate()).toEqual(new Date(0)) @@ -24,5 +22,5 @@ test("create record with time field", () => { test("keeps the milliseconds", () => { const date = new Time("2020-02-25T16:03:17.838527Z").toDate() - console.log(date) + expect(date?.toISOString()).toEqual("2020-02-25T16:03:17.838Z") }) From 58b7248ed12458f84fb16f938f6ec758ea9ef8dd Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 13:29:08 -0700 Subject: [PATCH 25/41] Fix lint --- zealot/zed/values/time.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/zealot/zed/values/time.ts b/zealot/zed/values/time.ts index 535d94a9cf..f784b1943b 100644 --- a/zealot/zed/values/time.ts +++ b/zealot/zed/values/time.ts @@ -1,7 +1,6 @@ import { ChronoField, convert, - DateTimeFormatter, DateTimeFormatterBuilder, LocalDateTime, nativeJs, From 172a69106fa27b42075490829c8948b90cac911d Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 13:42:25 -0700 Subject: [PATCH 26/41] Added array renderer --- app/viewer/value.tsx | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/app/viewer/value.tsx b/app/viewer/value.tsx index 32f5129149..780d67c315 100644 --- a/app/viewer/value.tsx +++ b/app/viewer/value.tsx @@ -2,7 +2,7 @@ import {formatPrimitive} from "app/core/formatters/format-zed" import {typeClassNames} from "app/core/utils/type-class-names" import {transparentize} from "polished" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" -import React from "react" +import React, {Fragment} from "react" import {useDispatch} from "react-redux" import {cssVar} from "src/js/lib/cssVar" import styled from "styled-components" @@ -43,6 +43,8 @@ export default function Value(props: ValueProps) { return } else if (props.value instanceof zed.Set) { return + } else if (props.value instanceof zed.Array) { + return } else { return } @@ -78,22 +80,44 @@ export function SetValue(props: ValueProps) { const firstItem = (i) => i === 0 return ( <> - {pad(props.padBefore)} |[ {set.items.map((v, i) => ( - <> + - {lastItem(i) ? null : ,} - + {lastItem(i) ? null : ,} + ))} ]| {pad(props.padBefore)} ) } + +export function ArrayValue(props: ValueProps) { + const array = props.value as zed.Array + const lastItem = (i) => i === array.items.length - 1 + const firstItem = (i) => i === 0 + return ( + <> + [ + {array.items.map((v, i) => ( + + + {lastItem(i) ? null : ,} + + ))} + ] + {pad(props.padBefore)} + + ) +} From 23e3c62a4930ff27106ae96a2c70d82c960d92fe Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 16:48:05 -0700 Subject: [PATCH 27/41] Update zed branch --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 349c3f6977..3abbef3f50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25429,8 +25429,8 @@ } }, "zed": { - "version": "git+https://github.com/brimdata/zed.git#3c17abe55921b3b63023f3cfd361899793b1cf85", - "from": "git+https://github.com/brimdata/zed.git#3c17abe55921b3b63023f3cfd361899793b1cf85" + "version": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc", + "from": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc" }, "zip-stream": { "version": "4.0.2", diff --git a/package.json b/package.json index e0d82f234f..311bc3b939 100644 --- a/package.json +++ b/package.json @@ -155,7 +155,7 @@ "styled-components": "^5.1.1", "tree-model": "^1.0.7", "valid-url": "^1.0.9", - "zed": "git+https://github.com/brimdata/zed.git#3c17abe55921b3b63023f3cfd361899793b1cf85" + "zed": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc" }, "optionalDependencies": { "electron-installer-debian": "^3.0.0", From f9c49fb049d038df785d9b5a47dd53d9baabbbae Mon Sep 17 00:00:00 2001 From: James Kerr Date: Mon, 3 May 2021 17:13:35 -0700 Subject: [PATCH 28/41] Clean up straggler files --- errors.txt | 20 --------- map.zson | 1 - port-array.zson | 3 -- src/js/flows/magic.ts | 41 ------------------- .../-from 1583768524.415634000 -to 158377486 | 11 ----- 5 files changed, 76 deletions(-) delete mode 100644 errors.txt delete mode 100644 map.zson delete mode 100644 port-array.zson delete mode 100644 src/js/flows/magic.ts delete mode 100644 src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 diff --git a/errors.txt b/errors.txt deleted file mode 100644 index aab8ee749e..0000000000 --- a/errors.txt +++ /dev/null @@ -1,20 +0,0 @@ -app/search/flows/next-page-viewer-search.test.ts -ppl/detail/flows/fetch.test.ts -ppl/detail/flows/get-correlation-query.test.ts -src/js/brim/program.test.ts -src/js/components/RightPane.test.tsx -src/js/electron/menu/actions/detailActions.ts -src/js/electron/menu/actions/searchActions.ts -src/js/flows/rightclick/cellMenu.test.ts -src/js/models/TableColumns.ts -src/js/searches/programs.test.ts -src/js/state/Chart/test.ts -src/js/state/Columns/models/columnSet.ts -src/js/state/Columns/selectors.ts -src/js/state/Columns/touch.test.ts -src/js/state/Columns/touch.ts -src/js/state/LogDetails/test.ts -src/js/state/SearchBar/test.ts -src/js/state/Viewer/reducer.ts -src/js/state/Viewer/test.ts -zealot/fetcher/records_callback.ts diff --git a/map.zson b/map.zson deleted file mode 100644 index 931756f0d6..0000000000 --- a/map.zson +++ /dev/null @@ -1 +0,0 @@ -{ my_map: |{ {"name", "james"}, {"age", "thirty"} }| } diff --git a/port-array.zson b/port-array.zson deleted file mode 100644 index ca6481b1e2..0000000000 --- a/port-array.zson +++ /dev/null @@ -1,3 +0,0 @@ -{ ports: [80, 445] ([port=(uint16)])} -{ portset: |[100]| (|[port]|) } - diff --git a/src/js/flows/magic.ts b/src/js/flows/magic.ts deleted file mode 100644 index 4c4a7b3331..0000000000 --- a/src/js/flows/magic.ts +++ /dev/null @@ -1,41 +0,0 @@ -// import brimcap from "src/pkg/brimcap" -// import {file} from "tmp" -// import brim from "../brim" -// import space from "../brim/space" -// import transaction from "../lib/transaction" - -// function loadFile(file, space) { -// return brimcap // promise -// .load({ -// space, -// file, -// progress: trackProgress -// }) -// } - -// function trackProgress() { -// // do someting here -// } - -// function isPcap() {} - -// // pcap format -// function activate() { -// brim.importers.add({ -// name: "PCAP Loader", -// match: isPcap, -// import: loadFile -// }) -// } - -// // default loading -// brim.importers.add({ -// name: "Default Loader", -// match: () => true, -// import: ({files, space, progress}) => { -// await brim.backend.post(file, space) -// progress(0.5) -// } -// }) - -// const importer = brim.importers.matching(file) diff --git a/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 b/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 deleted file mode 100644 index 876a280360..0000000000 --- a/src/pkg/brimcap/-from 1583768524.415634000 -to 158377486 +++ /dev/null @@ -1,11 +0,0 @@ --from 1583768524.415634000 -to 1583774869.009422 -p tcp 192.168.10.120:62458 34.232.129.83:443 - -# Successful brimcap launch -build/dist/brimcap launch -root ~/work/pcaps/root -ts "2020-03-09T15:42:04.415Z" -duration "6344.593788000s" -proto=tcp -src.ip 192.168.10.120 -src.port 62458 -dst.ip 34.232.129.83 -dst.port 443 - - -ts_sec=1583768524 -ts_ns=415634000 -duration_sec=6344 -duration_ns=593788000 -proto=tcp&src_host=192.168.10.120&src_port=62458&dst_host=34.232.129.83&dst_port=443 \ No newline at end of file From 2fc56dbcd1484b54f938fbe77d572d98767047b8 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 09:57:46 -0700 Subject: [PATCH 29/41] No colons in pcap names --- itest/tests/pcaps.test.ts | 4 ++-- plugins/brimcap/brimcap-plugin.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/itest/tests/pcaps.test.ts b/itest/tests/pcaps.test.ts index 35e5e25cc2..777b37af95 100644 --- a/itest/tests/pcaps.test.ts +++ b/itest/tests/pcaps.test.ts @@ -47,7 +47,7 @@ describe("Test PCAPs", () => { .then(async () => { await appStep.click(app, selectors.viewer.resultCellContaining("ssl")) await appStep.savePcap(app) - const fileBasename = "packets-2020-02-25T16:03:13.996366Z.pcap" + const fileBasename = "packets-2020-02-25T16_03_13.996366Z.pcap" const pcapAbspath = path.join(await pcapsDir(app), fileBasename) expect(md5(readFileSync(pcapAbspath))).toBe( "888453c81738fd8ade4c7f9888d86f86" @@ -64,7 +64,7 @@ describe("Test PCAPs", () => { .then(async () => { await appStep.click(app, selectors.viewer.resultCellContaining("conn")) await appStep.savePcap(app) - const fileBasename = "packets-2020-02-25T16:03:09.440467Z.pcap" + const fileBasename = "packets-2020-02-25T16_03_09.440467Z.pcap" const pcapAbspath = path.join(await pcapsDir(app), fileBasename) expect(md5(readFileSync(pcapAbspath))).toBe( "678442857027fdc5ad1e3418614dcdb8" diff --git a/plugins/brimcap/brimcap-plugin.ts b/plugins/brimcap/brimcap-plugin.ts index 106f833b22..9f38da77ee 100644 --- a/plugins/brimcap/brimcap-plugin.ts +++ b/plugins/brimcap/brimcap-plugin.ts @@ -178,7 +178,10 @@ export default class BrimcapPlugin { const tsString = ts.toString() const dur = log.get("duration") as zed.Duration - const dest = join(this.api.getTempDir(), `packets-${ts.toString()}.pcap`) + const dest = join( + this.api.getTempDir(), + `packets-${ts.toString()}.pcap`.replaceAll(":", "_") + ) return { dstIp: log.get("id.resp_h").toString(), From 19545f979369406984f7279e4c40ac040221654b Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 09:59:07 -0700 Subject: [PATCH 30/41] Update zed to master --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3abbef3f50..b3c962589e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25429,8 +25429,8 @@ } }, "zed": { - "version": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc", - "from": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc" + "version": "git+https://github.com/brimdata/zed.git#1683914e5ace6cc5621a949dccc5cf12a8818153", + "from": "git+https://github.com/brimdata/zed.git#1683914e5ace6cc5621a949dccc5cf12a8818153" }, "zip-stream": { "version": "4.0.2", diff --git a/package.json b/package.json index 311bc3b939..d32df48f80 100644 --- a/package.json +++ b/package.json @@ -155,7 +155,7 @@ "styled-components": "^5.1.1", "tree-model": "^1.0.7", "valid-url": "^1.0.9", - "zed": "git+https://github.com/brimdata/zed.git#e28c017a25624ec2754482efa6faf7f4b1d7f2cc" + "zed": "git+https://github.com/brimdata/zed.git#1683914e5ace6cc5621a949dccc5cf12a8818153" }, "optionalDependencies": { "electron-installer-debian": "^3.0.0", From c370cba5148f5275215eba5d983ffb158936d66d Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:13:48 -0700 Subject: [PATCH 31/41] Cleanup commented out code --- app/core/formatters/format-zed.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/core/formatters/format-zed.ts b/app/core/formatters/format-zed.ts index df11ca3eb2..e8464a9ee0 100644 --- a/app/core/formatters/format-zed.ts +++ b/app/core/formatters/format-zed.ts @@ -6,13 +6,5 @@ export function formatPrimitive(data: zed.Primitive) { if (data.isUnset()) return "⦻" if (zed.isInt(data)) return withCommas(data.toString()) if (zed.isTime(data)) return brim.time(data.toDate()).format() - if (zed.isStringy(data)) { - // only whitespace - if (data.toString().match(/^\s*$/)) { - // return `"${data.toString()}"` - } else { - // return data.toString() - } - } return data.toString() } From b5f816655449a8efc4aa7c69d858631bbba9c895 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:17:57 -0700 Subject: [PATCH 32/41] Remove unused test file --- test/fixtures/zjson-types.ts | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 test/fixtures/zjson-types.ts diff --git a/test/fixtures/zjson-types.ts b/test/fixtures/zjson-types.ts deleted file mode 100644 index f3638a5eca..0000000000 --- a/test/fixtures/zjson-types.ts +++ /dev/null @@ -1,7 +0,0 @@ -import {PrimitiveType} from "zealot/zjson" - -export const STRING = {kind: "primitive", name: "string"} as PrimitiveType -export const TIME = {kind: "primitive", name: "time"} as PrimitiveType -export const COUNT = {kind: "primitive", name: "count"} as PrimitiveType -export const INTERVAL = {kind: "primitive", name: "interval"} as PrimitiveType -export const IP = {kind: "primitive", name: "ip"} as PrimitiveType From 49f45cc38f8930f8fbce605dbe84db0f24aca454 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:23:55 -0700 Subject: [PATCH 33/41] Remove old reference files --- zealot/zed/primitive.ts | 54 --------------------- zealot/zed/record.ts | 103 ---------------------------------------- zealot/zed/type-def.ts | 41 ---------------- 3 files changed, 198 deletions(-) delete mode 100644 zealot/zed/primitive.ts delete mode 100644 zealot/zed/record.ts delete mode 100644 zealot/zed/type-def.ts diff --git a/zealot/zed/primitive.ts b/zealot/zed/primitive.ts deleted file mode 100644 index 864ec87a95..0000000000 --- a/zealot/zed/primitive.ts +++ /dev/null @@ -1,54 +0,0 @@ -// export class zed.Primitive { -// public typeName?: string // the logical type name -// public type: string // the actual type -// public value: string | null - -// constructor(args: {type: string; value: string | null; typeName?: string}) { -// this.type = args.type -// this.value = args.value -// this.typeName = args.typeName -// } - -// isUnset() { -// return this.value === null -// } - -// toDate() { -// if (this.type !== "time") { -// throw new Error(`Cannot make type: ${this.type} into a Date`) -// } -// return new Date(+this.value * 1000) -// } - -// toFloat() { -// if (this.isUnset()) return 0 - -// try { -// return parseFloat(this.value) -// } catch { -// throw new Error(`Cannot make type: ${this.type} into a Float`) -// } -// } - -// toInt() { -// if (this.isUnset()) throw new Error("value is unset") -// const int = parseInt(this.value) -// if (isNaN(int)) { -// throw new Error(`Cannot make type: ${this.type} into an Integer`) -// } -// return int -// } - -// toString(): string { -// return this.value || "" -// } - -// serialize() { -// return { -// kind: "primitive", -// type: this.type, -// value: this.value, -// typeName: this.typeName -// } -// } -// } diff --git a/zealot/zed/record.ts b/zealot/zed/record.ts deleted file mode 100644 index 0be1b1be23..0000000000 --- a/zealot/zed/record.ts +++ /dev/null @@ -1,103 +0,0 @@ -// import {zed.Field} from "./field" -// import {deserialize} from "./json" - -// export class zed.Record { -// fields: zed.Field[] | null -// typeName?: string - -// constructor(args: {fields: zed.Field[]; typeName?: string}) { -// this.fields = args.fields -// this.typeName = args.typeName -// } - -// static deserialize(data): zed.Record { -// return deserialize(data) -// } - -// [Symbol.iterator]() { -// let index = 0 -// const next = () => -// index < this.fields.length -// ? {done: false, value: this.at(index++)} -// : {done: true, value: undefined} - -// return {next} -// } - -// get columns() { -// return this.fields.map((f) => f.name) -// } - -// at(index: number) { -// return this.fields[index].data -// } - -// get(name: string) { -// return this.getField(name).data -// } - -// try(name: string) { -// try { -// return this.get(name) -// } catch { -// return null -// } -// } - -// isUnset() { -// return this.fields === null -// } - -// private _getField(name: string) { -// if (this.isUnset()) throw new Error("Record is unset") -// const field = this.fields.find((f) => f.name == name) -// if (!field) throw new UnknownColumnError(name, this.columns) -// return field -// } - -// getField(name: string) { -// return name -// .split(".") -// .reduce((field: zed.Field, namePart: string) => { -// if (field.data instanceof zed.Record) { -// return new zed.Field({name, data: field.data._getField(namePart).data}) -// } else { -// throw new Error("Dot syntax is only for nested records") -// } -// }, new zed.Field({name: "", data: this})) -// } - -// tryField(name: string) { -// try { -// return this.getField(name) -// } catch { -// return null -// } -// } - -// has(name: string) { -// return this.columns.includes(name) -// } - -// flatten(prefix = ""): zed.Record { -// let fields = [] - -// this.fields.forEach((field) => { -// if (field.data instanceof zed.Record) { -// const nested = field.data.flatten(field.name + ".") -// fields = fields.concat(nested.fields) -// } else { -// fields.push({data: field.data, name: prefix + field.name}) -// } -// }) -// return new zed.Record({fields}) -// } - -// serialize() { -// return { -// kind: "record", -// typeName: this.typeName, -// fields: this.fields.map((f) => f.serialize()) -// } -// } -// } diff --git a/zealot/zed/type-def.ts b/zealot/zed/type-def.ts deleted file mode 100644 index ff98f89e5d..0000000000 --- a/zealot/zed/type-def.ts +++ /dev/null @@ -1,41 +0,0 @@ -// import {RecordFieldType, TypeDefType} from "./zjson" - -// export default class ZedTypeDef { -// _type: TypeDefType - -// constructor({type}: {type: TypeDefType}) { -// this._type = type -// } - -// get name() { -// return this._type.name -// } - -// get innerType() { -// return this._type.type -// } - -// flatten(): ZedTypeDef { -// const inner = this.innerType -// if (inner.kind !== "record") return this - -// const flat = ( -// fields: RecordFieldType[], -// prefix = "" -// ): RecordFieldType[] => { -// return fields.flatMap((field) => { -// const name = prefix + field.name -// if (field.type.kind == "record") { -// return flat(field.type.fields, name + ".") -// } else { -// return {...field, name} -// } -// }) -// } -// const fields = flat(inner.fields) - -// return new ZedTypeDef({ -// type: {name: this.name, kind: "typedef", type: {kind: "record", fields}} -// }) -// } -// } From 23274d269febd666d6ffdab5183f14d90d09aa3f Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:28:59 -0700 Subject: [PATCH 34/41] Remove unused file --- zealot/zed/index.ts | 1 - zealot/zed/types/type-def.ts | 12 ------------ 2 files changed, 13 deletions(-) delete mode 100644 zealot/zed/types/type-def.ts diff --git a/zealot/zed/index.ts b/zealot/zed/index.ts index bf3565d1df..f4304674cf 100644 --- a/zealot/zed/index.ts +++ b/zealot/zed/index.ts @@ -4,7 +4,6 @@ export {TypeArray} from "./types/type-array" export {TypeBool} from "./types/type-bool" export {TypeBString} from "./types/type-bstring" export {TypeBytes} from "./types/type-bytes" -export {TypeDef} from "./types/type-def" export {TypeDuration} from "./types/type-duration" export {TypeError} from "./types/type-error" export {TypeFloat64} from "./types/type-float64" diff --git a/zealot/zed/types/type-def.ts b/zealot/zed/types/type-def.ts deleted file mode 100644 index eb7bb10f2a..0000000000 --- a/zealot/zed/types/type-def.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {ZedType} from "./types" - -export class TypeDef { - kind = "typedef" - - constructor(public name: string, public type: ZedType) {} - - create(value) { - console.log(value) - throw new Error("Im here!") - } -} From 1df49717bd2837b57ac9835bc75f073ee905721c Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:29:12 -0700 Subject: [PATCH 35/41] Handle 64 bit integers --- zealot/zed/values/int64.ts | 2 +- zealot/zed/values/uint64.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zealot/zed/values/int64.ts b/zealot/zed/values/int64.ts index 537648d9c1..c8bfbd50aa 100644 --- a/zealot/zed/values/int64.ts +++ b/zealot/zed/values/int64.ts @@ -7,6 +7,6 @@ export class Int64 extends Primitive { toInt() { if (isNull(this.value)) return null - return parseInt(this.value) + return BigInt(this.value) } } diff --git a/zealot/zed/values/uint64.ts b/zealot/zed/values/uint64.ts index 45b9ad68c4..fb1e4b8ba1 100644 --- a/zealot/zed/values/uint64.ts +++ b/zealot/zed/values/uint64.ts @@ -7,6 +7,6 @@ export class Uint64 extends Primitive { toInt() { if (isNull(this.value)) return null - return parseInt(this.value) + return BigInt(this.value) } } From 788b9919155c12a07980470e1e2786b0509b2b3e Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:32:48 -0700 Subject: [PATCH 36/41] Delete old script --- scripts/rename.js | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 scripts/rename.js diff --git a/scripts/rename.js b/scripts/rename.js deleted file mode 100644 index f5f96063c1..0000000000 --- a/scripts/rename.js +++ /dev/null @@ -1,22 +0,0 @@ -const {Project} = require("ts-morph") -const _ = require("lodash") - -const project = new Project({ - tsConfigFilePath: "./tsconfig.json" -}) - -project.getSourceFiles().forEach((srcFile) => { - // const oldName = srcFile.getBaseName() - // const name = srcFile.getBaseNameWithoutExtension() - // const newName = kebabCase(name) + srcFile.getExtension() - // const newPath = srcFile.getFilePath().replace(oldName, newName) - // console.log("--> " + newPath) - // srcFile.moveImmediatelySync(newPath) - const oldName = srcFile.getBaseName() - - if (oldName.endsWith("-test.ts")) { - const newName = oldName.replace("-test.ts", ".test.ts") - srcFile.moveImmediatelySync(newName) - console.log("==>" + newName) - } -}) From ed672420be1f3a3af03bfdaa3f463757b4009dd2 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:33:23 -0700 Subject: [PATCH 37/41] Remove old scaffold file --- src/pkg/brimcap/index.ts | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/pkg/brimcap/index.ts diff --git a/src/pkg/brimcap/index.ts b/src/pkg/brimcap/index.ts deleted file mode 100644 index c8223c6aaa..0000000000 --- a/src/pkg/brimcap/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -function load() {} - -export default { - load -} From 5845d88192e973ad5443b1509f20cbd1c43a59f7 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:34:18 -0700 Subject: [PATCH 38/41] Remove extra null check --- app/search/flows/next-page-viewer-search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/search/flows/next-page-viewer-search.ts b/app/search/flows/next-page-viewer-search.ts index 271b983261..89e8407a0e 100644 --- a/app/search/flows/next-page-viewer-search.ts +++ b/app/search/flows/next-page-viewer-search.ts @@ -40,7 +40,7 @@ function nextPageArgs( const index = indexOfLastChange(logs, (log) => log.try("ts")?.toString()) if (index >= 0) { const ts = logs[index].try("ts") - if (ts && ts instanceof zed.Time) { + if (ts instanceof zed.Time) { const prevTs = ts.toDate() nextSpan[1] = brim .time(prevTs) From badbe8b656c3fdd6cd1ee5376fd0fccb0e510097 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 13:43:09 -0700 Subject: [PATCH 39/41] Revert "Handle 64 bit integers" This reverts commit 1df49717bd2837b57ac9835bc75f073ee905721c. --- zealot/zed/values/int64.ts | 2 +- zealot/zed/values/uint64.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zealot/zed/values/int64.ts b/zealot/zed/values/int64.ts index c8bfbd50aa..537648d9c1 100644 --- a/zealot/zed/values/int64.ts +++ b/zealot/zed/values/int64.ts @@ -7,6 +7,6 @@ export class Int64 extends Primitive { toInt() { if (isNull(this.value)) return null - return BigInt(this.value) + return parseInt(this.value) } } diff --git a/zealot/zed/values/uint64.ts b/zealot/zed/values/uint64.ts index fb1e4b8ba1..45b9ad68c4 100644 --- a/zealot/zed/values/uint64.ts +++ b/zealot/zed/values/uint64.ts @@ -7,6 +7,6 @@ export class Uint64 extends Primitive { toInt() { if (isNull(this.value)) return null - return BigInt(this.value) + return parseInt(this.value) } } From dc21f591dccf6abf167a364c7631a57efe2d7e66 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 14:17:28 -0700 Subject: [PATCH 40/41] Removed unused file --- src/pkg/brimcap/load.test.ts | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/pkg/brimcap/load.test.ts diff --git a/src/pkg/brimcap/load.test.ts b/src/pkg/brimcap/load.test.ts deleted file mode 100644 index 82f2dcec55..0000000000 --- a/src/pkg/brimcap/load.test.ts +++ /dev/null @@ -1,5 +0,0 @@ -import brimcap from "./" - -test("load function", async () => { - await brimcap.load() -}) From 914e47c46c0c29ff992c2daef2f37f875b202fb1 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Tue, 4 May 2021 14:20:46 -0700 Subject: [PATCH 41/41] Put the zeek and suricata colors back in --- app/viewer/measure.ts | 15 ++++++++++++++- app/viewer/value.tsx | 14 +++++++++++++- ppl/suricata/suricata-plugin.tsx | 26 ++++++++++++++++++++++++++ ppl/zeek/zeek-plugin.tsx | 24 ++++++++++++++++++++++++ src/css/_log-viewer.scss | 3 +++ src/js/models/TableColumns.ts | 2 +- 6 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 ppl/suricata/suricata-plugin.tsx create mode 100644 ppl/zeek/zeek-plugin.tsx diff --git a/app/viewer/measure.ts b/app/viewer/measure.ts index 7be50e8fe2..fa9999d86e 100644 --- a/app/viewer/measure.ts +++ b/app/viewer/measure.ts @@ -1,4 +1,6 @@ import {formatPrimitive} from "app/core/formatters/format-zed" +import {isEventType} from "ppl/suricata/suricata-plugin" +import {isPath} from "ppl/zeek/zeek-plugin" import {zed} from "zealot" const ONE_CHAR = 7.39 @@ -14,12 +16,23 @@ export function estimateHeaderWidth(name: string) { return Math.min(MAX_WIDTH, width) } -export function estimateCellWidth(value: zed.AnyValue) { +export function estimateCellWidth(value: zed.AnyValue, name: string) { let width = MIN_WIDTH if (value instanceof zed.Primitive) { width = Math.ceil(formatPrimitive(value).length * ONE_CHAR + CELL_PAD) } else { width = Math.ceil(value.toString().length * ONE_CHAR + CELL_PAD) } + + // Move to plugin + if (isPath(new zed.Field(name, value))) { + width += 12 // padding + } + + // Move to plugin + if (isEventType(new zed.Field(name, value))) { + width += 12 // padding + } + return Math.min(MAX_WIDTH, width) } diff --git a/app/viewer/value.tsx b/app/viewer/value.tsx index 780d67c315..cbc5de478b 100644 --- a/app/viewer/value.tsx +++ b/app/viewer/value.tsx @@ -2,6 +2,8 @@ import {formatPrimitive} from "app/core/formatters/format-zed" import {typeClassNames} from "app/core/utils/type-class-names" import {transparentize} from "polished" import searchFieldContextMenu from "ppl/menus/searchFieldContextMenu" +import {isEventType, SuricataEventType} from "ppl/suricata/suricata-plugin" +import {isPath, ZeekPath} from "ppl/zeek/zeek-plugin" import React, {Fragment} from "react" import {useDispatch} from "react-redux" import {cssVar} from "src/js/lib/cssVar" @@ -68,12 +70,22 @@ export function PrimitiveValue(props: ValueProps) { } > {pad(props.padBefore)} - {formatPrimitive(props.value as zed.Primitive)} + {renderValue(props)} {pad(props.padAfter)} ) } +function renderValue(props) { + if (isPath(props.field)) { + return + } else if (isEventType(props.field)) { + return + } else { + return formatPrimitive(props.value as zed.Primitive) + } +} + export function SetValue(props: ValueProps) { const set = props.value as zed.Set const lastItem = (i) => i === set.items.length - 1 diff --git a/ppl/suricata/suricata-plugin.tsx b/ppl/suricata/suricata-plugin.tsx new file mode 100644 index 0000000000..a392da5843 --- /dev/null +++ b/ppl/suricata/suricata-plugin.tsx @@ -0,0 +1,26 @@ +/** + * Move this code in to a suricata plugin + */ + +import React from "react" +import {zed} from "zealot" +import {isStringy} from "zealot/zed" + +export function isEventType(field) { + return field.name === "event_type" && isStringy(field.value) +} + +export default function eventTypeClassNames(record: zed.Record) { + const severity = record.try("alert.severity") + if (severity instanceof zed.Primitive && severity.isSet()) { + return `path-tag alert-${severity.toString()}-bg-color` + } else return "" +} + +export function SuricataEventType(props) { + return ( + + {props.field.value.toString()} + + ) +} diff --git a/ppl/zeek/zeek-plugin.tsx b/ppl/zeek/zeek-plugin.tsx new file mode 100644 index 0000000000..f491211ab5 --- /dev/null +++ b/ppl/zeek/zeek-plugin.tsx @@ -0,0 +1,24 @@ +/** + * Move this code in to a zeek plugin + */ + +import React from "react" +import {zed} from "zealot" +import {isStringy} from "zealot/zed" + +export function isPath(field) { + return field.name === "_path" && isStringy(field.value) +} + +export default function pathClassNames(field: zed.Field) { + const path = field.value.toString() + return `path-tag ${path}-bg-color` +} + +export function ZeekPath(props) { + return ( + + {props.field.value.toString()} + + ) +} diff --git a/src/css/_log-viewer.scss b/src/css/_log-viewer.scss index 3a318a5a1c..14d618d244 100644 --- a/src/css/_log-viewer.scss +++ b/src/css/_log-viewer.scss @@ -75,4 +75,7 @@ .int64 { text-align: right; } + .path-tag { + @include path-tag; + } } diff --git a/src/js/models/TableColumns.ts b/src/js/models/TableColumns.ts index f3a4c21018..843f056f67 100644 --- a/src/js/models/TableColumns.ts +++ b/src/js/models/TableColumns.ts @@ -38,7 +38,7 @@ export default class TableColumns { records.forEach((r) => { const data = r.try(col.name) if (!data) return - const width = estimateCellWidth(data) + const width = estimateCellWidth(data, col.name) if (width > max) max = width }) col.width = max
{column.name}