From 9cddb1145ab495eb989ea23ecb8a63b41bb02cb2 Mon Sep 17 00:00:00 2001 From: Rawlin Peters Date: Thu, 2 Sep 2021 15:36:57 -0600 Subject: [PATCH] Add support for cdn and maxRevalDurationDays qparams to GET /jobs (#6154) * Add support for cdnId and maxRevalDurationDays qparams to GET /jobs For the `GET /jobs` API, add support for the following query parameters: - `cdn` to return jobs for delivery services in the given CDN with this name - `maxRevalDurationDays` to return jobs with a start_time that is greater than now() - $maxRevalDurationDays (a parameter in the GLOBAL profile which controls how long any created invalidation job can last) Related: #5674 * Use cdn (name) query param instead of cdnId * Use startTime in the docs instead of start_time, log test error --- CHANGELOG.md | 4 ++ docs/source/api/v2/jobs.rst | 38 +++++++------ docs/source/api/v3/jobs.rst | 38 +++++++------ docs/source/api/v4/jobs.rst | 38 +++++++------ ...3121285700_add_job_start_time_idx.down.sql | 18 ++++++ ...083121285700_add_job_start_time_idx.up.sql | 18 ++++++ traffic_ops/testing/api/v4/jobs_test.go | 56 +++++++++++++++++++ traffic_ops/testing/api/v4/tc-fixtures.json | 24 ++++++++ .../invalidationjobs/invalidationjobs.go | 22 +++++++- 9 files changed, 203 insertions(+), 53 deletions(-) create mode 100644 traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.down.sql create mode 100644 traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.up.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ab0c8e9c0..53321c566a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## unreleased (but not 6.0) +### Added +- [#5674](https://github.com/apache/trafficcontrol/issues/5674) Added new query parameters `cdn` and `maxRevalDurationDays` to the `GET /api/x/jobs` Traffic Ops API to filter by CDN name and within the start_time window defined by the `maxRevalDurationDays` GLOBAL profile parameter, respectively. + ## unreleased ### Added - [#4982](https://github.com/apache/trafficcontrol/issues/4982) Added the ability to support queueing updates by server type and profile diff --git a/docs/source/api/v2/jobs.rst b/docs/source/api/v2/jobs.rst index c64d277d9d..58bcea5a4d 100644 --- a/docs/source/api/v2/jobs.rst +++ b/docs/source/api/v2/jobs.rst @@ -31,23 +31,27 @@ Request Structure ----------------- .. table:: Request Query Parameters - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | Name | Required | Description | - +=================+==========+======================================================================================================================+ - | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | createdBy | no | Return only invalidation jobs that were created by the user with this username | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | id | no | Return only the single invalidation job identified by this integral, unique identifer | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Name | Required | Description | + +======================+==========+==================================================================================================================================================================+ + | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | cdn | no | Return only invalidation jobs for delivery services with this CDN name | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | createdBy | no | Return only invalidation jobs that were created by the user with this username | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | id | no | Return only the single invalidation job identified by this integral, unique identifer | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | maxRevalDurationDays | no | Return only invalidation jobs with a startTime that is within the window defined by the ``maxRevalDurationDays`` :term:`Parameter` in :ref:`the-global-profile` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. code-block:: http diff --git a/docs/source/api/v3/jobs.rst b/docs/source/api/v3/jobs.rst index 5e0f85b7c0..f5e7b02deb 100644 --- a/docs/source/api/v3/jobs.rst +++ b/docs/source/api/v3/jobs.rst @@ -31,23 +31,27 @@ Request Structure ----------------- .. table:: Request Query Parameters - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | Name | Required | Description | - +=================+==========+======================================================================================================================+ - | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | createdBy | no | Return only invalidation jobs that were created by the user with this username | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | id | no | Return only the single invalidation job identified by this integral, unique identifer | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Name | Required | Description | + +======================+==========+==================================================================================================================================================================+ + | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | cdn | no | Return only invalidation jobs for delivery services with this CDN name | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | createdBy | no | Return only invalidation jobs that were created by the user with this username | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | id | no | Return only the single invalidation job identified by this integral, unique identifer | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | maxRevalDurationDays | no | Return only invalidation jobs with a startTime that is within the window defined by the ``maxRevalDurationDays`` :term:`Parameter` in :ref:`the-global-profile` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. code-block:: http diff --git a/docs/source/api/v4/jobs.rst b/docs/source/api/v4/jobs.rst index 357ff3acfd..26954317a4 100644 --- a/docs/source/api/v4/jobs.rst +++ b/docs/source/api/v4/jobs.rst @@ -31,23 +31,27 @@ Request Structure ----------------- .. table:: Request Query Parameters - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | Name | Required | Description | - +=================+==========+======================================================================================================================+ - | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | createdBy | no | Return only invalidation jobs that were created by the user with this username | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | id | no | Return only the single invalidation job identified by this integral, unique identifer | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ - | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | - +-----------------+----------+----------------------------------------------------------------------------------------------------------------------+ + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Name | Required | Description | + +======================+==========+==================================================================================================================================================================+ + | assetUrl | no | Return only invalidation jobs that operate on URLs by matching this regular expression | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | cdn | no | Return only invalidation jobs for delivery services with this CDN name | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | createdBy | no | Return only invalidation jobs that were created by the user with this username | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | deliveryService | no | Return only invalidation jobs that operate on the :term:`Delivery Service` with this :ref:`ds-xmlid` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | dsId | no | Return only invalidation jobs pending on the :term:`Delivery Service` identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | id | no | Return only the single invalidation job identified by this integral, unique identifer | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | keyword | no | Return only invalidation jobs that have this "keyword" - only "PURGE" should exist | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | maxRevalDurationDays | no | Return only invalidation jobs with a startTime that is within the window defined by the ``maxRevalDurationDays`` :term:`Parameter` in :ref:`the-global-profile` | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | userId | no | Return only invalidation jobs created by the user identified by this integral, unique identifier | + +----------------------+----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. code-block:: http diff --git a/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.down.sql b/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.down.sql new file mode 100644 index 0000000000..bef55a7ef1 --- /dev/null +++ b/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.down.sql @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +DROP INDEX job_start_time_idx; diff --git a/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.up.sql b/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.up.sql new file mode 100644 index 0000000000..1d6cb5364b --- /dev/null +++ b/traffic_ops/app/db/migrations/2021083121285700_add_job_start_time_idx.up.sql @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +CREATE INDEX job_start_time_idx ON job (start_time DESC NULLS LAST); diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 84c1a9fe99..45dd0d3764 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -482,6 +482,62 @@ func GetTestJobsByValidData(t *testing.T) { } else { t.Error("No user response available for get user by name") } + + // get maxRevalDurationDays parameter value + opts = client.NewRequestOptions() + opts.QueryParameters.Set("name", "maxRevalDurationDays") + params, _, err := TOSession.GetParameters(opts) + if err != nil { + t.Fatalf("unexpected error getting maxRevalDurationDays parameter: %v", err) + } + if len(params.Response) != 1 { + t.Fatalf("expected 1 maxRevalDurationDays parameter, got %d", len(params.Response)) + } + maxRevalDurationDays, err := strconv.Atoi(params.Response[0].Value) + if err != nil { + t.Fatalf("unexpected error converting maxRevalDurationDays value to int: %v", err) + } + + // get jobs created within maxRevalDurationDays window + opts = client.NewRequestOptions() + opts.QueryParameters.Set("maxRevalDurationDays", "") + maxRevalJobs, _, err := TOSession.GetInvalidationJobs(opts) + if err != nil { + t.Errorf("unexpected error getting jobs by maxRevalDurationDays: %v", err) + } else if len(maxRevalJobs.Response) < 1 { + t.Errorf("GET /jobs by maxRevalDurationDays - expected at least 1 job") + } + for _, j := range maxRevalJobs.Response { + if time.Since((*j.StartTime).Time) > time.Duration(maxRevalDurationDays)*24*time.Hour { + t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, *j.DeliveryService, *j.AssetURL, *j.StartTime) + } + } + + // create DS xml_id -> cdn_id lookup map + dses, _, err := TOSession.GetDeliveryServices(client.NewRequestOptions()) + if err != nil { + t.Fatalf("unexpectd error getting delivery services: %v", err) + } + dsToCDN := make(map[string]string, len(dses.Response)) + for _, ds := range dses.Response { + dsToCDN[*ds.XMLID] = *ds.CDNName + } + + cdn := "cdn2" + // get jobs by CDN ID + opts = client.NewRequestOptions() + opts.QueryParameters.Set("cdn", cdn) + cdnJobs, _, err := TOSession.GetInvalidationJobs(opts) + if err != nil { + t.Errorf("unexpected error getting jobs by cdn: %v", err) + } else if len(cdnJobs.Response) < 1 { + t.Errorf("GET /jobs by cdn - expected at least 1 job") + } + for _, j := range cdnJobs.Response { + if dsToCDN[*j.DeliveryService] != cdn { + t.Errorf("GET /jobs by cdn returned job that does not belong to CDN %s: {%s, %s, %v}", cdn, *j.DeliveryService, *j.AssetURL, *j.StartTime) + } + } } func GetTestJobsByInvalidData(t *testing.T) { diff --git a/traffic_ops/testing/api/v4/tc-fixtures.json b/traffic_ops/testing/api/v4/tc-fixtures.json index 11df51da34..c3b4cead4c 100644 --- a/traffic_ops/testing/api/v4/tc-fixtures.json +++ b/traffic_ops/testing/api/v4/tc-fixtures.json @@ -5476,6 +5476,30 @@ "regex": "\\/some-path?.+\\.jpg", "startTime": "2100-06-19T13:57:51-06:00", "ttl": 2.1 + }, + { + "deliveryService": "ds1", + "regex": "/oldest", + "startTime": 1000, + "ttl": 2160 + }, + { + "deliveryService": "ds1", + "regex": "/older", + "startTime": 1001, + "ttl": 2160 + }, + { + "deliveryService": "ds1", + "regex": "/old", + "startTime": 1002, + "ttl": 2160 + }, + { + "deliveryService": "ds-forked-topology", + "regex": "/this-ds-is-in-cdn2", + "startTime": 4117118271000, + "ttl": 2160 } ], "statsSummaries": [ diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index c7f6d7871b..6124863387 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -228,10 +228,28 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err if err != nil { return nil, nil, fmt.Errorf("getting accessible tenants for user - %v", err), http.StatusInternalServerError, nil } + cdn := "" + if cdnName, ok := job.APIInfo().Params["cdn"]; ok { + queryValues["cdn"] = cdnName + cdn = ` AND ds.cdn_id = (SELECT id FROM cdn WHERE name = :cdn) ` + } + maxDays := "" + if _, ok := job.APIInfo().Params["maxRevalDurationDays"]; ok { + // jobs started within the last $maxRevalDurationDays days (defaulting to 90 days if the parameter doesn't exist) + maxDays = ` AND job.start_time >= NOW() - CAST( + (SELECT COALESCE( + (SELECT value + FROM parameter + WHERE name = 'maxRevalDurationDays' + AND config_file = 'regex_revalidate.config' + LIMIT 1), + '90')) + || ' days' AS INTERVAL) ` + } if len(where) > 0 { - where += " AND ds.tenant_id = ANY(:tenants) " + where += " AND ds.tenant_id = ANY(:tenants) " + maxDays + cdn } else { - where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + maxDays + cdn } queryValues["tenants"] = pq.Array(accessibleTenants)