From 8ad4abffdaadcf9cd8a3da0ffac8e012f4736672 Mon Sep 17 00:00:00 2001 From: hazedav Date: Wed, 9 Jun 2021 19:35:53 -0500 Subject: [PATCH] feat(cli): LQLv2 support (#441) ALLY-504 --- api/lql.go | 47 ++++++++++++----- api/lql_test.go | 75 +++++++++++++++------------ api/lql_update.go | 41 --------------- api/lql_update_test.go | 20 +++---- cli/cmd/lql.go | 24 ++++++--- cli/cmd/lql_create.go | 8 +-- cli/cmd/lql_update.go | 4 +- integration/lql_test.go | 8 +-- integration/test_resources/lql/my.lql | 6 --- 9 files changed, 113 insertions(+), 120 deletions(-) delete mode 100644 api/lql_update.go delete mode 100644 integration/test_resources/lql/my.lql diff --git a/api/lql.go b/api/lql.go index 178dc5d95..026f52ca4 100644 --- a/api/lql.go +++ b/api/lql.go @@ -22,8 +22,8 @@ import ( "encoding/json" "fmt" "net/url" - "regexp" "strconv" + "strings" "time" "github.com/pkg/errors" @@ -32,15 +32,14 @@ import ( ) const ( - reLQL string = `(?ms)^(\w+)\([^)]+\)\s*{` LQLQueryTranslateError string = "unable to translate query blob" ) type LQLQuery struct { - ID string `json:"LQL_ID,omitempty"` - StartTimeRange string `json:"START_TIME_RANGE,omitempty"` - EndTimeRange string `json:"END_TIME_RANGE,omitempty"` - QueryText string `json:"QUERY_TEXT"` + ID string `json:"lql_id,omitempty"` + StartTimeRange string `json:"start_time_range,omitempty"` + EndTimeRange string `json:"end_time_range,omitempty"` + QueryText string `json:"query_text"` // QueryBlob is a special string that supports type conversion // back and forth from LQL to JSON QueryBlob string `json:"-"` @@ -84,13 +83,12 @@ func (q *LQLQuery) Translate() error { } func (q *LQLQuery) TranslateQuery() error { - // empty + // if query text is already populated if q.QueryText != "" { return nil } - // json + // valid json var t LQLQuery - if err := json.Unmarshal([]byte(q.QueryBlob), &t); err == nil { if q.StartTimeRange == "" { q.StartTimeRange = t.StartTimeRange @@ -101,11 +99,23 @@ func (q *LQLQuery) TranslateQuery() error { q.QueryText = t.QueryText return err } - // lql - if matched, _ := regexp.MatchString(reLQL, q.QueryBlob); matched { + // invalid json + qblob := strings.ToLower(q.QueryBlob) + if strings.Contains(qblob, "start_time_range") || + strings.Contains(qblob, "end_time_range") || + strings.Contains(qblob, "lql_id") || + strings.Contains(qblob, "query_text") { + + return errors.New(LQLQueryTranslateError) + } + // valid lql text + if strings.Contains(q.QueryBlob, "{") && + strings.Contains(q.QueryBlob, "}") { q.QueryText = q.QueryBlob + return nil } + // invalid lql text return errors.New(LQLQueryTranslateError) } @@ -175,7 +185,7 @@ type LQLService struct { } func (svc *LQLService) CreateQuery(query string) ( - response LQLQueryResponse, + response LQLQuery, err error, ) { lqlQuery := LQLQuery{QueryBlob: query} @@ -187,6 +197,19 @@ func (svc *LQLService) CreateQuery(query string) ( return } +func (svc *LQLService) UpdateQuery(query string) ( + response LQLQuery, + err error, +) { + lqlQuery := LQLQuery{QueryBlob: query} + if err = lqlQuery.Validate(true); err != nil { + return + } + + err = svc.client.RequestEncoderDecoder("PATCH", apiLQL, lqlQuery, &response) + return +} + func (svc *LQLService) GetQueries() (LQLQueryResponse, error) { return svc.GetQueryByID("") } diff --git a/api/lql_test.go b/api/lql_test.go index 9baa1d547..baa96c503 100644 --- a/api/lql_test.go +++ b/api/lql_test.go @@ -32,14 +32,12 @@ import ( ) var ( - lqlQueryID = "my_lql" - lqlQueryStr = "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" - lqlCreateData = `[ - { - "lql_id": "my_lql", - "query_text": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" - } -]` + lqlQueryID = "my_lql" + lqlQueryStr = "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" + lqlCreateResponse = `{ + "lql_id": "my_lql", + "query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" +}` lqlRunData = `[ { "INSERT_ID": "35308423" @@ -285,24 +283,35 @@ var lqlQueryTypeTests = []LQLQueryTest{ QueryBlob: `this is junk`, }, }, + LQLQueryTest{ + Name: "partial-blob", + Input: &api.LQLQuery{ + QueryBlob: `{`, + }, + Return: api.LQLQueryTranslateError, + Expected: &api.LQLQuery{ + QueryText: ``, + QueryBlob: `{`, + }, + }, LQLQueryTest{ Name: "json-blob", Input: &api.LQLQuery{ QueryBlob: `{ -"START_TIME_RANGE": "678910", -"END_TIME_RANGE": "111213141516", -"QUERY_TEXT": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"start_time_range": "678910", +"end_time_range": "111213141516", +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, Return: nil, Expected: &api.LQLQuery{ StartTimeRange: "1970-01-01T00:11:18Z", EndTimeRange: "1973-07-11T04:32:21Z", - QueryText: "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }", + QueryText: "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }", QueryBlob: `{ -"START_TIME_RANGE": "678910", -"END_TIME_RANGE": "111213141516", -"QUERY_TEXT": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"start_time_range": "678910", +"end_time_range": "111213141516", +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, }, @@ -312,30 +321,30 @@ var lqlQueryTypeTests = []LQLQueryTest{ QueryBlob: `{ "start_time_range": "678910", "end_time_range": "111213141516", -"query_text": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, Return: nil, Expected: &api.LQLQuery{ StartTimeRange: "1970-01-01T00:11:18Z", EndTimeRange: "1973-07-11T04:32:21Z", - QueryText: "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }", + QueryText: "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }", QueryBlob: `{ "start_time_range": "678910", "end_time_range": "111213141516", -"query_text": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, }, LQLQueryTest{ Name: "lql-blob", Input: &api.LQLQuery{ - QueryBlob: "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }", + QueryBlob: "--a comment\nmy_lql { source { CloudTrailRawEvents } return { INSERT_ID } }", }, Return: nil, Expected: &api.LQLQuery{ - QueryText: "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }", - QueryBlob: "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }", + QueryText: "--a comment\nmy_lql { source { CloudTrailRawEvents } return { INSERT_ID } }", + QueryBlob: "--a comment\nmy_lql { source { CloudTrailRawEvents } return { INSERT_ID } }", }, }, LQLQueryTest{ @@ -345,9 +354,9 @@ var lqlQueryTypeTests = []LQLQueryTest{ EndTimeRange: "1", QueryText: "should not overwrite", QueryBlob: `{ -"START_TIME_RANGE": "678910", -"END_TIME_RANGE": "111213141516", -"QUERY_TEXT": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"start_time_range": "678910", +"end_time_range": "111213141516", +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, Return: nil, @@ -356,9 +365,9 @@ var lqlQueryTypeTests = []LQLQueryTest{ EndTimeRange: "1970-01-01T00:00:00Z", QueryText: "should not overwrite", QueryBlob: `{ -"START_TIME_RANGE": "678910", -"END_TIME_RANGE": "111213141516", -"QUERY_TEXT": "my_lql(CloudTrailRawEvents e) { SELECT INSERT_ID LIMIT 10 }" +"start_time_range": "678910", +"end_time_range": "111213141516", +"query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID } }" }`, }, }, @@ -435,13 +444,11 @@ func TestLQLCreateBadInput(t *testing.T) { } func TestLQLCreateOK(t *testing.T) { - mockResponse := mockLQLDataResponse(lqlCreateData) - fakeServer := lacework.MockServer() fakeServer.MockAPI( "external/lql", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, mockResponse) + fmt.Fprint(w, lqlCreateResponse) }, ) defer fakeServer.Close() @@ -452,10 +459,10 @@ func TestLQLCreateOK(t *testing.T) { ) assert.Nil(t, err) - createExpected := api.LQLQueryResponse{} - _ = json.Unmarshal([]byte(mockResponse), &createExpected) + createExpected := api.LQLQuery{} + _ = json.Unmarshal([]byte(lqlCreateResponse), &createExpected) - var createActual api.LQLQueryResponse + var createActual api.LQLQuery createActual, err = c.LQL.CreateQuery(lqlQueryStr) assert.Nil(t, err) @@ -510,7 +517,7 @@ func TestLQLGetQueriesMethod(t *testing.T) { } func TestLQLGetQueryByIDOK(t *testing.T) { - mockResponse := mockLQLDataResponse(lqlCreateData) + mockResponse := mockLQLDataResponse("[" + lqlCreateResponse + "]") fakeServer := lacework.MockServer() fakeServer.MockAPI( diff --git a/api/lql_update.go b/api/lql_update.go deleted file mode 100644 index 846652632..000000000 --- a/api/lql_update.go +++ /dev/null @@ -1,41 +0,0 @@ -// -// Author:: Salim Afiune Maya () -// Copyright:: Copyright 2020, Lacework Inc. -// License:: Apache License, Version 2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package api - -type LQLUpdateResponse struct { - Ok bool `json:"ok"` - Message LQLUpdateMessage `json:"message"` -} - -type LQLUpdateMessage struct { - ID string `json:"lqlUpdated"` -} - -func (svc *LQLService) UpdateQuery(query string) ( - response LQLUpdateResponse, - err error, -) { - lqlQuery := LQLQuery{QueryBlob: query} - if err = lqlQuery.Validate(true); err != nil { - return - } - - err = svc.client.RequestEncoderDecoder("PATCH", apiLQL, lqlQuery, &response) - return -} diff --git a/api/lql_update_test.go b/api/lql_update_test.go index 687974a71..2a258424d 100644 --- a/api/lql_update_test.go +++ b/api/lql_update_test.go @@ -29,6 +29,13 @@ import ( "github.com/stretchr/testify/assert" ) +var ( + lqlUpdateResponse = `{ + "lql_id": "my_lql", + "query_text": "my_lql { source { CloudTrailRawEvents } return { INSERT_ID, INSERT_TIME } }" + }` +) + func TestLQLUpdateMethod(t *testing.T) { fakeServer := lacework.MockServer() fakeServer.MockAPI( @@ -71,16 +78,11 @@ func TestLQLUpdateBadInput(t *testing.T) { } func TestLQLUpdateOK(t *testing.T) { - mockResponse := mockLQLMessageResponse( - fmt.Sprintf(`"lqlUpdated": "%s"`, lqlQueryID), - "true", - ) - fakeServer := lacework.MockServer() fakeServer.MockAPI( "external/lql", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, mockResponse) + fmt.Fprint(w, lqlUpdateResponse) }, ) defer fakeServer.Close() @@ -91,10 +93,10 @@ func TestLQLUpdateOK(t *testing.T) { ) assert.Nil(t, err) - updateExpected := api.LQLUpdateResponse{} - _ = json.Unmarshal([]byte(mockResponse), &updateExpected) + updateExpected := api.LQLQuery{} + _ = json.Unmarshal([]byte(lqlUpdateResponse), &updateExpected) - var updateActual api.LQLUpdateResponse + var updateActual api.LQLQuery updateActual, err = c.LQL.UpdateQuery(lqlQueryStr) assert.Nil(t, err) diff --git a/cli/cmd/lql.go b/cli/cmd/lql.go index 684c2734c..468fa2776 100644 --- a/cli/cmd/lql.go +++ b/cli/cmd/lql.go @@ -80,7 +80,7 @@ Start and End times are required to run a query: 2. Start and End times must be specified in one of the following ways: A. As StartTimeRange and EndTimeRange in the ParamInfo block within the LQL query - B. As START_TIME_RANGE and END_TIME_RANGE if specifying JSON + B. As start_time_range and end_time_range if specifying JSON C. As --start and --end CLI flags 3. Start and End time precedence: @@ -246,15 +246,19 @@ func queryErrorCrumbs(query string, err error) error { return err } // smells like json - query = strings.TrimLeft(query, " ") - if strings.HasPrefix(query, "{") || strings.HasPrefix(query, "[") { + query = strings.ToLower(query) + if strings.Contains(query, "start_time_range") || + strings.Contains(query, "end_time_range") || + strings.Contains(query, "lql_id") || + strings.Contains(query, "query_text") { + return errors.New(`invalid query It looks like you attempted to submit an LQL query in JSON format. Please validate that the JSON is formatted properly and adheres to the following schema: { - "QUERY_TEXT": "MyLQL(CloudTrailRawEvents e) { SELECT INSERT_ID }" + "query_text": "MyLQL { source { CloudTrailRawEvents } filter { EVENT_SOURCE = 's3.amazonaws.com' } return { INSERT_ID } }" }`) } // smells like plain text @@ -263,8 +267,16 @@ Please validate that the JSON is formatted properly and adheres to the following It looks like you attempted to submit an LQL query in plain text format. Please validate that the text adheres to the following schema: -MyLQL(CloudTrailRawEvents e) { - SELECT INSERT_ID +MyLQL { + source { + CloudTrailRawEvents + } + filter { + EVENT_SOURCE = 's3.amazonaws.com' + } + return { + INSERT_ID + } } `) } diff --git a/cli/cmd/lql_create.go b/cli/cmd/lql_create.go index fd33d27a7..860a33daf 100644 --- a/cli/cmd/lql_create.go +++ b/cli/cmd/lql_create.go @@ -57,12 +57,8 @@ func createQuery(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "unable to create LQL query") } if cli.JSONOutput() { - return cli.OutputJSON(create.Data) + return cli.OutputJSON(create) } - queryID := "unknown" - if len(create.Data) > 0 { - queryID = create.Data[0].ID - } - cli.OutputHuman(fmt.Sprintf("LQL query (%s) created successfully.\n", queryID)) + cli.OutputHuman(fmt.Sprintf("LQL query (%s) created successfully.\n", create.ID)) return nil } diff --git a/cli/cmd/lql_update.go b/cli/cmd/lql_update.go index 81550782c..cce553fe9 100644 --- a/cli/cmd/lql_update.go +++ b/cli/cmd/lql_update.go @@ -59,9 +59,9 @@ func updateQuery(cmd *cobra.Command, args []string) error { return errors.Wrap(err, lqlUpdateUnableMsg) } if cli.JSONOutput() { - return cli.OutputJSON(update.Message) + return cli.OutputJSON(update) } cli.OutputHuman( - fmt.Sprintf("LQL query (%s) updated successfully.\n", update.Message.ID)) + fmt.Sprintf("LQL query (%s) updated successfully.\n", update.ID)) return nil } diff --git a/integration/lql_test.go b/integration/lql_test.go index ba50064ab..a24202335 100644 --- a/integration/lql_test.go +++ b/integration/lql_test.go @@ -29,9 +29,9 @@ import ( const ( lqlQueryID string = "MyLQL" - lqlQueryText string = "MyLQL(CloudTrailRawEvents e) {SELECT INSERT_ID LIMIT 1}" - lqlQueryUpdate string = "MyLQL(CloudTrailRawEvents e) {SELECT INSERT_ID, INSERT_TIME LIMIT 1}" - lqlQueryURL string = "https://raw.githubusercontent.com/lacework/go-sdk/main/integration/test_resources/lql/my.lql" + lqlQueryText string = "MyLQL { source { CloudTrailRawEvents } return { INSERT_ID } }" + lqlQueryUpdate string = "MyLQL { source { CloudTrailRawEvents } return { INSERT_ID, INSERT_TIME } }" + lqlQueryURL string = "https://raw.githubusercontent.com/lacework/go-sdk/main/integration/test_resources/lql/MyLQL.lql" ) var ( @@ -156,7 +156,7 @@ func TestQueryRunFileJSONCrumb(t *testing.T) { // run _, stderr, exitcode := LaceworkCLIWithTOMLConfig( "query", "run", "-f", file.Name(), "--start", lqlQueryStart, "--end", lqlQueryEnd) - assert.Contains(t, stderr.String(), "LQL query in JSON format") + assert.Contains(t, stderr.String(), "LQL query in plain text format") assert.Equal(t, 1, exitcode, "EXITCODE is not the expected one") } diff --git a/integration/test_resources/lql/my.lql b/integration/test_resources/lql/my.lql deleted file mode 100644 index e1dc3163d..000000000 --- a/integration/test_resources/lql/my.lql +++ /dev/null @@ -1,6 +0,0 @@ ---- This query produces a web (github) hosted object for "lacework query" integration testing ---- Specifically: TestQueryRunURL -MyLQL(CloudTrailRawEvents e) { - SELECT INSERT_ID - LIMIT 1 -}