From d4b2b7d1c75c1744efc3657488e29933d6bcf180 Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Wed, 21 Mar 2018 16:20:38 -0400 Subject: [PATCH 1/8] Implements https://github.com/apache/incubator-trafficcontrol/issues/1907 Contains the following changes: Traffic Router - Backup edge configuration parsing and decision making Traffic Ops - Mojolicious Perl CRUD APIs Address comments from Eric and Jeremy Addressing Dylan's comment on avoiding nested routes Addessing Dylan's comment on POSt Request / Response . API endpoints changes based on Jeremey's suggestion Resolved conflict in traffic_ops.rst Addressing Jeremy's comment on GET endpoint's return statements. Changes: 1. POST - Add new fallbacks. 2. PUT - Update existing fallbacks. Renamed 20180321000000_cache_group_fallback.sql -> 20180521000000_cache_group_fallback.sql --- CHANGELOG.md | 1 + docs/source/admin/traffic_router.rst | 6 +- .../v12/cachegroup_fallbacks.rst | 288 ++++++++++++++++++ .../20180521000000_cache_group_fallback.sql | 38 +++ traffic_ops/app/lib/API/Cachegroup.pm | 11 + traffic_ops/app/lib/API/CachegroupFallback.pm | 278 +++++++++++++++++ .../app/lib/Schema/Result/Cachegroup.pm | 38 +++ .../lib/Schema/Result/CachegroupFallback.pm | 138 +++++++++ traffic_ops/app/lib/TrafficOpsRoutes.pm | 7 + traffic_ops/app/lib/UI/Topology.pm | 9 + .../DeepCoverageZoneController.java | 2 +- .../core/cache/CacheLocation.java | 40 +++ .../core/config/ConfigHandler.java | 16 +- .../core/router/StatTracker.java | 15 +- .../core/router/TrafficRouter.java | 59 +++- .../core/loc/CoverageZoneTest.java | 2 +- 16 files changed, 928 insertions(+), 20 deletions(-) create mode 100644 docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst create mode 100644 traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql create mode 100644 traffic_ops/app/lib/API/CachegroupFallback.pm create mode 100644 traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm diff --git a/CHANGELOG.md b/CHANGELOG.md index 09998ca623..f976a40d0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - /api/1.3/types `(GET,POST,PUT,DELETE)` - Fair Queuing Pacing: Using the FQ Pacing Rate parameter in Delivery Services allows operators to limit the rate of individual sessions to the edge cache. This feature requires a Trafficserver RPM containing the fq_pacing experimental plugin AND setting 'fq' as the default Linux qdisc in sysctl. - Traffic Ops rpm changed to remove world-read permission from configuration files. +- Backup Edge Cache group: If the matched group in the CZF is not available, this list of backup edge cache group configured via Traffic Ops API can be used as backup. In the event of all backup edge cache groups not available, GEO location can be optionally used as further backup. APIs detailed here [here](http://traffic-control-cdn.readthedocs.io/en/latest/development/traffic_ops_api/v12/cachegroup_fallbacks.html) ### Changed - Reformatted this CHANGELOG file to the keep-a-changelog format diff --git a/docs/source/admin/traffic_router.rst b/docs/source/admin/traffic_router.rst index 963fe1516f..b8ed3ba470 100644 --- a/docs/source/admin/traffic_router.rst +++ b/docs/source/admin/traffic_router.rst @@ -200,7 +200,7 @@ Fields Always Present +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ |rloc |GeoLocation of result |Latitude and Longitude in Decimal Degrees | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -|rdtl |Result Details Associated with unusual conditions |One of DS_NOT_FOUND, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY | +|rdtl |Result Details Associated with unusual conditions |One of DS_NOT_FOUND, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CZ_BACKUP_CG | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ |rerr |Message about internal Traffic Router Error |String | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ @@ -266,8 +266,8 @@ Fields Always Present +---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ | "-" |The request was not redirected. This is usually a result of a DNS request to the Traffic Router or an explicit denial for that request. | +---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ - - +|DS_CZ_BACKUP_CG |Traffic Router found a backup cache via fallback (cr-config's edgeLocation) / coordinates (CZF) configuration | ++---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ --------------- diff --git a/docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst b/docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst new file mode 100644 index 0000000000..cabc4500b8 --- /dev/null +++ b/docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst @@ -0,0 +1,288 @@ +.. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. +.. + +.. _to-api-v12-cachegroupfallbacks: + +Cache Group Fallback +==================== + +.. _to-api-v12-cachegroupfallbacks-route: + +/api/1.2/cachegroup_fallbacks +++++++++++++++++++++++++++++++ + +**GET /api/1.2/cachegroup_fallbacks?cacheGroupId={id}** +**GET /api/1.2/cachegroup_fallbacks?fallbackId={id}** +**GET /api/1.2/cachegroup_fallbacks?cacheGroupId={id}&fallbackId={id}** + + Retrieve fallback related configurations for a cache group. + + Authentication Required: Yes + + Role(s) Required: None + + **Request Query Parameters** + + Query parameter is mandatory. Either one of the parameters must be used. Both can also be used simultaneously. + + +-----------------+---------------------------------------------------------------------------+ + | Name | Description | + +=================+===========================================================================+ + | cacheGroupId | The id of the cache group whose backup configurations has to be retrieved | + +-----------------+---------------------------------------------------------------------------+ + | fallbackId | The id of the fallback cache group associated with a cache group | + +-----------------+---------------------------------------------------------------------------+ + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Response Example** :: + + { + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":2, + "fallbackOrder":10, + "fallbackName":"GROUP2" + } + ] + } + +| + +**POST /api/1.2/cachegroup_fallbacks** + + Creates fallback configuration for the cache group. New fallbacks can be added only via POST. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Parameters** + The request parameters should be in array format. + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | Fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Request Example** :: + + [ + { + "cacheGroupId": 1, + "fallbackId": 3, + "fallbackOrder": 10 + } + ] + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``alerts`` | array | A collection of alert messages. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>text`` | string | Alert message. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + + **Response Example** :: + + { + "alerts": [ + { + "level":"success", + "text":"Backup configuration CREATE for cache group 1 successful." + } + ], + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":3, + "fallbackName":"GROUP2", + "fallbackorder":10, + } + ] + } + +| + +**PUT /api/1.2/cachegroup_fallbacks** + + Updates an existing fallback configuration for the cache group. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Parameters** + The request parameters should be in array format. + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | Fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Request Example** :: + + [ + { + "cacheGroupId": 1, + "fallbackId": 3, + "fallbackOrder": 10 + } + ] + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``alerts`` | array | A collection of alert messages. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>text`` | string | Alert message. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + + **Response Example** :: + + { + "alerts": [ + { + "level":"success", + "text":"Backup configuration UPDATE for cache group 1 successful." + } + ], + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":3, + "fallbackName":"GROUP2", + "fallbackorder":10, + } + ] + } + +| + +**DELETE /api/1.2/cachegroup_fallbacks?cacheGroupId={id}** +**DELETE /api/1.2/cachegroup_fallbacks?fallbackId={id}** +**DELETE /api/1.2/cachegroup_fallbacks?fallbackId={id}&cacheGroupId={id}** + + Delete fallback list assigned to the cache group. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Query Parameters** + + Query parameter is mandatory. Either one of the parameters must be used. Both can also be used simultaneously. + + +-----------------+----------+--------------------------------------------------------------------------------------+ + | Name | Required | Description | + +=================+==========+======================================================================================+ + | cacheGroupId | Yes | The id of the cache group whose backup configurations has to be deleted | + +-----------------+----------+--------------------------------------------------------------------------------------+ + | fallbackId | Yes | The id of the fallback cachegroup which has to be deleted from the list of fallbacks | + +-----------------+----------+--------------------------------------------------------------------------------------+ + + **Response Properties** + + +-------------+--------+----------------------------------+ + | Parameter | Type | Description | + +=============+========+==================================+ + | ``alerts`` | array | A collection of alert messages. | + +-------------+--------+----------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-------------+--------+----------------------------------+ + | ``>text`` | string | Alert message. | + +-------------+--------+----------------------------------+ + + **Response Example** :: + + { + "alerts": [ + { + "level": "success", + "text": "Backup configuration DELETED" + } + ], + } + +| + diff --git a/traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql b/traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql new file mode 100644 index 0000000000..923895714e --- /dev/null +++ b/traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql @@ -0,0 +1,38 @@ +/* + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied + + +-- cachegroup_fallbacks +CREATE TABLE cachegroup_fallbacks ( + primary_cg bigint, + backup_cg bigint CHECK (primary_cg != backup_cg), + set_order bigint NOT NULL, + CONSTRAINT fk_primary_cg FOREIGN KEY (primary_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, + CONSTRAINT fk_backup_cg FOREIGN KEY (backup_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, + UNIQUE (primary_cg, backup_cg), + UNIQUE (primary_cg, set_order) +); + +ALTER TABLE cachegroup ADD COLUMN fallback_to_closest BOOLEAN DEFAULT TRUE; + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back + +ALTER TABLE cachegroup DROP COLUMN fallback_to_closest; + +DROP TABLE cachegroup_fallbacks; diff --git a/traffic_ops/app/lib/API/Cachegroup.pm b/traffic_ops/app/lib/API/Cachegroup.pm index 8d0f4f03cf..db430b2f80 100644 --- a/traffic_ops/app/lib/API/Cachegroup.pm +++ b/traffic_ops/app/lib/API/Cachegroup.pm @@ -57,6 +57,7 @@ sub index { "lastUpdated" => $row->last_updated, "parentCachegroupId" => $row->parent_cachegroup_id, "parentCachegroupName" => ( defined $row->parent_cachegroup_id ) ? $idnames{ $row->parent_cachegroup_id } : undef, + "fallbackToClosest" => \$row->fallback_to_closest, "secondaryParentCachegroupId" => $row->secondary_parent_cachegroup_id, "secondaryParentCachegroupName" => ( defined $row->secondary_parent_cachegroup_id ) ? $idnames{ $row->secondary_parent_cachegroup_id } @@ -110,6 +111,7 @@ sub show { "lastUpdated" => $row->last_updated, "parentCachegroupId" => $row->parent_cachegroup_id, "parentCachegroupName" => ( defined $row->parent_cachegroup_id ) ? $idnames{ $row->parent_cachegroup_id } : undef, + "fallbackToClosest" => \$row->fallback_to_closest, "secondaryParentCachegroupId" => $row->secondary_parent_cachegroup_id, "secondaryParentCachegroupName" => ( defined $row->secondary_parent_cachegroup_id ) ? $idnames{ $row->secondary_parent_cachegroup_id } @@ -158,12 +160,18 @@ sub update { } } + my $fallback_to_closest = $params->{fallbackToClosest}; + if ( !defined ($fallback_to_closest) ) { + $fallback_to_closest = $cachegroup->fallback_to_closest; + } + my $values = { name => $params->{name}, short_name => $params->{shortName}, latitude => $params->{latitude}, longitude => $params->{longitude}, parent_cachegroup_id => $params->{parentCachegroupId}, + fallback_to_closest => $fallback_to_closest, secondary_parent_cachegroup_id => $params->{secondaryParentCachegroupId}, type => $params->{typeId} }; @@ -189,6 +197,7 @@ sub update { ( defined $rs->parent_cachegroup_id ) ? $idnames{ $rs->parent_cachegroup_id } : undef; + $response->{fallbackToClosest} = $rs->fallback_to_closest; $response->{secondaryParentCachegroupId} = $rs->secondary_parent_cachegroup_id; $response->{secondaryParentCachegroupName} = ( defined $rs->secondary_parent_cachegroup_id ) @@ -239,6 +248,7 @@ sub create { latitude => $params->{latitude}, longitude => $params->{longitude}, parent_cachegroup_id => $params->{parentCachegroupId}, + fallback_to_closest => exists ($params->{fallbackToClosest}) ? $params->{fallbackToClosest} : 1,# defaults to true secondary_parent_cachegroup_id => $params->{secondaryParentCachegroupId}, type => $params->{typeId} }; @@ -265,6 +275,7 @@ sub create { ( defined $rs->parent_cachegroup_id ) ? $idnames{ $rs->parent_cachegroup_id } : undef; + $response->{fallbackToClosest} = $rs->fallback_to_closest; $response->{secondaryParentCachegroupId} = $rs->secondary_parent_cachegroup_id; $response->{secondaryParentCachegroupName} = ( defined $rs->secondary_parent_cachegroup_id ) diff --git a/traffic_ops/app/lib/API/CachegroupFallback.pm b/traffic_ops/app/lib/API/CachegroupFallback.pm new file mode 100644 index 0000000000..e0a831fdbc --- /dev/null +++ b/traffic_ops/app/lib/API/CachegroupFallback.pm @@ -0,0 +1,278 @@ +package API::CachegroupFallback; +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# a note about locations and cachegroups. This used to be "Location", before we had physical locations in 12M. Very confusing. +# What used to be called a location is now called a "cache group" and location is now a physical address, not a group of caches working together. +# + +# JvD Note: you always want to put Utils as the first use. Sh*t don't work if it's after the Mojo lines. +use UI::Utils; +use Mojo::Base 'Mojolicious::Controller'; +use Data::Dumper; +use JSON; +use MojoPlugins::Response; +use Validate::Tiny ':all'; + +sub delete { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $fallback_id = $self->param('fallbackId'); + my $params = $self->req->json; + my $rs_backups = undef; + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + if ( defined ($cache_id) && defined($fallback_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id , backup_cg => $fallback_id} ); + } elsif (defined ($cache_id)) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id} ); + } elsif (defined ($fallback_id)) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { backup_cg => $fallback_id} ); + } + + if ( ($rs_backups->count > 0) ) { + my $del_records = $rs_backups->delete(); + if ($del_records) { + &log( $self, "Backup configuration DELETED", "APICHANGE"); + return $self->success_message("Backup configuration DELETED"); + } else { + return $self->alert( "Backup configuration DELETED." ); + } + } else { + &log( $self, "No backup Cachegroups found"); + return $self->not_found(); + } +} + +sub show { + my $self = shift; + my $cache_id = $self->param("cacheGroupId"); + my $fallback_id = $self->param("fallbackId"); + my $id = $cache_id ? $cache_id : $fallback_id; + + #only integers + if ( $id !~ /^\d+?$/ ) { + &log( $self, "No such Cachegroup id $id"); + return $self->success([]); + } + + my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $id } )->single(); + if ( !defined($cachegroup) ) { + &log( $self, "No such Cachegroup $id"); + return $self->success([]); + } + + if ( ($cachegroup->type->name ne "EDGE_LOC") ) { + &log( $self, "cachegroup should be type EDGE_LOC."); + return $self->success([]); + } + + my $rs_backups = undef; + + if ( defined ($cache_id) && defined ($fallback_id)) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id, backup_cg => $fallback_id}, {order_by => 'set_order'}); + } elsif ( defined ($cache_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + } elsif ( defined ($fallback_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ backup_cg => $fallback_id}, {order_by => 'set_order'}); + } + + if ( defined ($rs_backups) && ($rs_backups->count > 0) ) { + my $response; + my $backup_cnt = 0; + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $row->primary_cg->id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + return $self->success( $response ); + } else { + &log( $self, "No backup Cachegroups"); + return $self->success([]); + } +} + +sub create { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $params = $self->req->json; + + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; + } + + if ( !defined($params) ) { + return $self->alert("parameters must be in JSON format, please check!"); + } + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + #only integers + if ( $cache_id !~ /^\d+?$/ ) { + &log( $self, "No such Cachegroup id $cache_id"); + return $self->not_found(); + } + + my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); + if ( !defined($cachegroup) ) { + return $self->not_found(); + } + + if ( ($cachegroup->type->name ne "EDGE_LOC") ) { + return $self->alert("cachegroup should be type EDGE_LOC."); + } + + foreach my $config (@{ $params }) { + my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); + if ( !defined($rs_backup) ) { + &log( $self, "ERROR Backup config: No such Cache Group $config->{fallbackId}"); + next; + } + + if ( ($rs_backup->type->name ne "EDGE_LOC") ) { + &log( $self, "ERROR Backup config: $config->{name} is not EDGE_LOC"); + next; + } + + my $existing_row = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id, backup_cg => $config->{fallbackId} } ); + if ( defined ($existing_row->next) ) { + next;#Skip existing rows + } + + my $values = { + primary_cg => $cache_id , + backup_cg => $config->{fallbackId}, + set_order => $config->{fallbackOrder} + }; + + my $rs_data = $self->db->resultset('CachegroupFallback')->create($values)->insert(); + if ( !defined($rs_data)) { + &log( $self, "Database operation for backup configuration for cache group $cache_id failed."); + } + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + my $response; + my $backup_cnt = 0; + if ( ($rs_backups->count > 0) ) { + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $cache_id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + &log( $self, "Backup configuration UPDATED for $cache_id", "APICHANGE"); + return $self->success( $response, "Backup configuration CREATE for cache group $cache_id successful." ); + } else { + return $self->alert("Backup configuration CREATE for cache group $cache_id Failed." ); + } +} + + +sub update { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $params = $self->req->json; + + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; + } + + if ( !defined($params) ) { + return $self->alert("parameters must be in JSON format, please check!"); + } + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + #only integers + if ( $cache_id !~ /^\d+?$/ ) { + &log( $self, "No such Cachegroup id $cache_id"); + return $self->not_found(); + } + + my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); + if ( !defined($cachegroup) ) { + return $self->not_found(); + } + + if ( ($cachegroup->type->name ne "EDGE_LOC") ) { + return $self->alert("cachegroup should be type EDGE_LOC."); + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id } ); + if ( !defined ($rs_backups->next) ) { + return $self->alert( "Backup list not configured for $cache_id, create and update" ); + } + + foreach my $config (@{ $params }) { + my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); + if ( !defined($rs_backup) ) { + &log( $self, "ERROR Backup config: No such Cache Group $config->{fallbackId}"); + next; + } + + if ( ($rs_backup->type->name ne "EDGE_LOC") ) { + &log( $self, "ERROR Backup config: $config->{name} is not EDGE_LOC"); + next; + } + + my $values = { + primary_cg => $cache_id , + backup_cg => $config->{fallbackId}, + set_order => $config->{fallbackOrder} + }; + + my $existing_row = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id, backup_cg => $config->{fallbackId} } ); + #New row creation disabled for PUT.Only existing rows can be updated + if ( defined ($existing_row->next) ) { + $existing_row->update($values); + } + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + my $response; + my $backup_cnt = 0; + if ( ($rs_backups->count > 0) ) { + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $cache_id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + &log( $self, "Backup configuration UPDATED for $cache_id", "APICHANGE"); + return $self->success( $response, "Backup configuration UPDATE for cache group $cache_id successful." ); + } else { + return $self->alert("Backup configuration UPDATE for cache group $cache_id Failed." ); + } +} + +1; diff --git a/traffic_ops/app/lib/Schema/Result/Cachegroup.pm b/traffic_ops/app/lib/Schema/Result/Cachegroup.pm index fea22b3ee2..a93c039906 100644 --- a/traffic_ops/app/lib/Schema/Result/Cachegroup.pm +++ b/traffic_ops/app/lib/Schema/Result/Cachegroup.pm @@ -75,6 +75,12 @@ __PACKAGE__->table("cachegroup"); is_nullable: 1 original: {default_value => \"now()"} +=head2 fallback_to_closest + + data_type: 'boolean' + default_value: true + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -106,6 +112,8 @@ __PACKAGE__->add_columns( is_nullable => 1, original => { default_value => \"now()" }, }, + "fallback_to_closest", + { data_type => "boolean", default_value => \"true", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -177,6 +185,36 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 cachegroup_fallbacks_backup_cgs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "cachegroup_fallbacks_backup_cgs", + "Schema::Result::CachegroupFallback", + { "foreign.backup_cg" => "self.id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 cachegroup_fallbacks_primary_cgs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "cachegroup_fallbacks_primary_cgs", + "Schema::Result::CachegroupFallback", + { "foreign.primary_cg" => "self.id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + =head2 cachegroup_parameters Type: has_many diff --git a/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm b/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm new file mode 100644 index 0000000000..58046b3a88 --- /dev/null +++ b/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm @@ -0,0 +1,138 @@ +use utf8; +package Schema::Result::CachegroupFallback; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Schema::Result::CachegroupFallback + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("cachegroup_fallbacks"); + +=head1 ACCESSORS + +=head2 primary_cg + + data_type: 'bigint' + is_foreign_key: 1 + is_nullable: 1 + +=head2 backup_cg + + data_type: 'bigint' + is_foreign_key: 1 + is_nullable: 1 + +=head2 set_order + + data_type: 'bigint' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "primary_cg", + { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 }, + "backup_cg", + { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 }, + "set_order", + { data_type => "bigint", is_nullable => 0 }, +); + +=head1 UNIQUE CONSTRAINTS + +=head2 C + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->add_unique_constraint( + "cachegroup_fallbacks_primary_cg_backup_cg_key", + ["primary_cg", "backup_cg"], +); + +=head2 C + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->add_unique_constraint( + "cachegroup_fallbacks_primary_cg_set_order_key", + ["primary_cg", "set_order"], +); + +=head1 RELATIONS + +=head2 backup_cg + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "backup_cg", + "Schema::Result::Cachegroup", + { id => "backup_cg" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + +=head2 primary_cg + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "primary_cg", + "Schema::Result::Cachegroup", + { id => "primary_cg" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07048 @ 2018-03-20 04:15:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9bJ/JA5FNpy0LYu1KRdQqA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/traffic_ops/app/lib/TrafficOpsRoutes.pm b/traffic_ops/app/lib/TrafficOpsRoutes.pm index 5c944b957f..a371442c1d 100644 --- a/traffic_ops/app/lib/TrafficOpsRoutes.pm +++ b/traffic_ops/app/lib/TrafficOpsRoutes.pm @@ -419,6 +419,13 @@ sub api_routes { $r->put("/api/$version/cachegroups/:id" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 )->to( 'Cachegroup#update', namespace => $namespace ); $r->delete("/api/$version/cachegroups/:id" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 )->to( 'Cachegroup#delete', namespace => $namespace ); + + # -- CACHEGROUP-Fallbacks: CRUD + $r->get("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#show', namespace => $namespace ); + $r->post("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#create', namespace => $namespace ); + $r->put("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#update', namespace => $namespace ); + $r->delete("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#delete', namespace => $namespace ); + # -- CACHEGROUPS: ASSIGN DELIVERYSERVICES $r->post("/api/$version/cachegroups/:id/deliveryservices" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 ) ->to( 'DeliveryServiceServer#assign_ds_to_cachegroup', namespace => $namespace ); diff --git a/traffic_ops/app/lib/UI/Topology.pm b/traffic_ops/app/lib/UI/Topology.pm index 2107baacdb..c87f8f0cd4 100644 --- a/traffic_ops/app/lib/UI/Topology.pm +++ b/traffic_ops/app/lib/UI/Topology.pm @@ -250,6 +250,15 @@ sub gen_crconfig_json { if ( $row->type->name =~ m/^EDGE/ ) { $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'latitude'} = $row->cachegroup->latitude + 0; $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'longitude'} = $row->cachegroup->longitude + 0; + $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'backupLocations'}->{'fallbackToClosest'} = $row->cachegroup->fallback_to_closest ? "true" : "false"; + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $row->cachegroup->id}, {order_by => 'set_order'}); + my $backup_cnt = 0; + + while ( my $backup_row = $rs_backups->next ) { + $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'backupLocations'}->{'list'}[$backup_cnt] = $backup_row->backup_cg->name; + $backup_cnt++; + } } if ( !exists $cache_tracker{ $row->id } ) { diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java index 6503bcb9c8..56d5ab06cc 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java @@ -35,7 +35,7 @@ public class DeepCoverageZoneController { public @ResponseBody ResponseEntity getCacheLocationForIp(@RequestParam(name = "ip") final String ip, @RequestParam(name = "deliveryServiceId") final String deliveryServiceId) { - final CacheLocation cacheLocation = trafficRouterManager.getTrafficRouter().getCoverageZoneCacheLocation(ip, deliveryServiceId, true); + final CacheLocation cacheLocation = trafficRouterManager.getTrafficRouter().getCoverageZoneCacheLocation(ip, deliveryServiceId, true, null); if (cacheLocation == null) { return ResponseEntity.status(HttpStatus.NOT_FOUND).body(null); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java index 67159c1162..9c2b93d8f5 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java @@ -38,6 +38,8 @@ public class CacheLocation { private final Geolocation geolocation; private final Map caches; + private List backupCacheGroups = null; + private boolean useClosestGeoOnBackupFailure = true; /** * Creates a CacheLocation with the specified ID at the specified location. @@ -48,8 +50,28 @@ public class CacheLocation { * the coordinates of this location */ public CacheLocation(final String id, final Geolocation geolocation) { + this(id, geolocation, null, true); + } + + /** + * Creates a CacheLocation with the specified ID at the specified location. + * + * @param id + * the id of the location + * @param geolocation + * the coordinates of this location + * + * @param backupCacheGroups + * the backup cache groups for this id + * + * @param useClosestGeoOnBackupFailure + * the backup fallback setting for this id + */ + public CacheLocation(final String id, final Geolocation geolocation, final List backupCacheGroups, final boolean useClosestGeoOnBackupFailure) { this.id = id; this.geolocation = geolocation; + this.backupCacheGroups = backupCacheGroups; + this.useClosestGeoOnBackupFailure = useClosestGeoOnBackupFailure; caches = new HashMap(); } @@ -128,6 +150,24 @@ public Geolocation getGeolocation() { return geolocation; } + /** + * Gets backupCacheGroups. + * + * @return the backupCacheGroups + */ + public List getBackupCacheGroups() { + return backupCacheGroups; + } + + /** + * Tests useClosestGeoOnBackupFailure. + * + * @return useClosestGeoOnBackupFailure + */ + public boolean isUseClosestGeoLoc() { + return useClosestGeoOnBackupFailure; + } + /** * Gets id. * diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 2c33409e1a..6ab8fa1fdd 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -672,8 +672,22 @@ private void parseLocationConfig(final JsonNode locationsJo, final CacheRegister while (locIter.hasNext()) { final String loc = locIter.next(); final JsonNode jo = JsonUtils.getJsonNode(locationsJo, loc); + List backupCacheGroups = null; + boolean useClosestOnBackupFailure = true; + + if (jo != null && jo.has("backupLocations")) { + final JsonNode backupConfigJson = JsonUtils.getJsonNode(jo, "backupLocations"); + backupCacheGroups = new ArrayList<>(); + if (backupConfigJson.has("list")) { + for (final JsonNode cacheGroup : JsonUtils.getJsonNode(backupConfigJson, "list")) { + backupCacheGroups.add(cacheGroup.asText()); + } + useClosestOnBackupFailure = JsonUtils.optBoolean(backupConfigJson, "fallbackToClosest", false); + } + + } try { - locations.add(new CacheLocation(loc, new Geolocation(JsonUtils.getDouble(jo, "latitude"), JsonUtils.getDouble(jo, "longitude")))); + locations.add(new CacheLocation(loc, new Geolocation(JsonUtils.getDouble(jo, "latitude"), JsonUtils.getDouble(jo, "longitude")), backupCacheGroups, useClosestOnBackupFailure)); } catch (JsonUtilsException e) { LOGGER.warn(e,e); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java index ccc675cb97..b4cb3f7130 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java @@ -116,7 +116,7 @@ public static enum ResultType { public enum ResultDetails { NO_DETAILS, DS_NOT_FOUND, DS_TLS_MISMATCH, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CLIENT_GEO_UNSUPPORTED, GEO_NO_CACHE_FOUND, - REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE + REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE, DS_CZ_BACKUP_CG } long time; @@ -130,6 +130,11 @@ public enum ResultDetails { boolean isClientGeolocationQueried; RegionalGeoResult regionalGeoResult; + boolean fromBackupCzGroup; + // in memory switch to track if need to continue geo based + // defaulting to true, changes the false by router at runtime when primary cache group is configured using fallbackToClosedGeoLoc + // to false and backup group list is configured and failing + boolean continueGeo = true; public Track() { start(); @@ -185,6 +190,14 @@ public RegionalGeoResult getRegionalGeoResult() { return regionalGeoResult; } + public void setFromBackupCzGroup(final boolean fromBackupCzGroup) { + this.fromBackupCzGroup = fromBackupCzGroup; + } + + public boolean isFromBackupCzGroup() { + return fromBackupCzGroup; + } + public final void start() { time = System.currentTimeMillis(); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java index 78ec478a85..caaf4e8cb0 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java @@ -274,8 +274,9 @@ public List getCachesByGeo(final DeliveryService ds, final Geolocation cl protected List selectCaches(final HTTPRequest request, final DeliveryService ds, final Track track) throws GeolocationException { CacheLocation cacheLocation; ResultType result = ResultType.CZ; + final boolean useDeep = (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS); - if (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS) { + if (useDeep) { // Deep caching is enabled. See if there are deep caches available cacheLocation = getDeepCoverageZoneCacheLocation(request.getClientIP(), ds); if (cacheLocation != null && cacheLocation.getCaches().size() != 0) { @@ -287,7 +288,7 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ } } else { // Deep caching not enabled for this Delivery Service; use the regular CZ - cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, useDeep, track); } Listcaches = selectCachesByCZ(ds, cacheLocation, track, result); @@ -304,7 +305,8 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ track.setResult(ResultType.MISS); track.setResultDetails(ResultDetails.DS_CZ_ONLY); } - } else { + } else if (track.continueGeo) { + // continue Geo can be disabled when backup group is used -- ended up an empty cache list if reach here caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); } @@ -312,7 +314,6 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ } public List selectCachesByGeo(final String clientIp, final DeliveryService deliveryService, final CacheLocation cacheLocation, final Track track) throws GeolocationException { - Geolocation clientLocation = null; try { @@ -374,7 +375,7 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws return result; } - final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, false, track); List caches = selectCachesByCZ(ds, cacheLocation, track); if (caches != null) { @@ -403,7 +404,9 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws LOGGER.error("Bad client address: '" + request.getClientIP() + "'"); } - caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + if (track.continueGeo) { + caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + } if (caches != null) { track.setResult(ResultType.GEO); @@ -495,6 +498,9 @@ private List selectCachesByCZ(final DeliveryService ds, final CacheLocati if (caches != null && track != null) { track.setResult(result); + if (track.isFromBackupCzGroup()) { + track.setResultDetails(ResultDetails.DS_CZ_BACKUP_CG); + } track.setResultLocation(cacheLocation.getGeolocation()); } @@ -676,11 +682,11 @@ protected NetworkNode getNetworkNode(final String ip) { } public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId) { - return getCoverageZoneCacheLocation(ip, deliveryServiceId, false); // default is not deep + return getCoverageZoneCacheLocation(ip, deliveryServiceId, false, null); // default is not deep } @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"}) - public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep) { + public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep, final Track track) { final NetworkNode networkNode = useDeep ? getDeepNetworkNode(ip) : getNetworkNode(ip); if (networkNode == null) { @@ -716,17 +722,44 @@ public CacheLocation getCoverageZoneCacheLocation(final String ip, final String return cacheLocation; } + if (cacheLocation != null && cacheLocation.getBackupCacheGroups() != null) { + for (final String cacheGroup : cacheLocation.getBackupCacheGroups()) { + final CacheLocation bkCacheLocation = getCacheRegister().getCacheLocationById(cacheGroup); + if (bkCacheLocation != null && !getSupportingCaches(bkCacheLocation.getCaches(), deliveryService).isEmpty()) { + LOGGER.debug("Got backup CZ cache group " + bkCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + } + return bkCacheLocation; + } + } + // track.continueGeo + // will become to false only when backups are configured and (primary group's) fallbackToClosedGeo is configured (non-empty list) to false + // False signals subsequent cacheSelection routine to stop geo based selection. + if (!cacheLocation.isUseClosestGeoLoc()) { + track.continueGeo = false; + return null; + } + } + // We had a hit in the CZF but the name does not match a known cache location. // Check whether the CZF entry has a geolocation and use it if so. - return getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + final CacheLocation closestCacheLocation = getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + if (closestCacheLocation != null) { + LOGGER.debug("Got closest CZ cache group " + closestCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + } + } + return closestCacheLocation; } public CacheLocation getDeepCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { - return getCoverageZoneCacheLocation(ip, deliveryService, true); + return getCoverageZoneCacheLocation(ip, deliveryService, true, null); } - protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep) { - return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep); + protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep, final Track track) { + return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep, track); } protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { @@ -744,7 +777,7 @@ public Cache consistentHashForCoverageZone(final String ip, final String deliver return null; } - final CacheLocation coverageZoneCacheLocation = getCoverageZoneCacheLocation(ip, deliveryService, useDeep); + final CacheLocation coverageZoneCacheLocation = getCoverageZoneCacheLocation(ip, deliveryService, useDeep, null); final List caches = selectCachesByCZ(deliveryService, coverageZoneCacheLocation); if (caches == null || caches.isEmpty()) { diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java index 0387c89c29..f340b2efb7 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java @@ -88,7 +88,7 @@ public void before() throws Exception { trafficRouter = PowerMockito.mock(TrafficRouter.class); Whitebox.setInternalState(trafficRouter, "cacheRegister", cacheRegister); when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1")).thenCallRealMethod(); - when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false)).thenCallRealMethod(); + when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false, null)).thenCallRealMethod(); when(trafficRouter.getCacheRegister()).thenReturn(cacheRegister); when(trafficRouter.orderCacheLocations(cacheGroups,testLocation)).thenCallRealMethod(); when(trafficRouter.getSupportingCaches(anyListOf(Cache.class), eq(deliveryService))).thenCallRealMethod(); From 14aa10bf3d97b239c16a3385c13732ffa11f234b Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Thu, 24 May 2018 09:57:08 -0400 Subject: [PATCH 2/8] Moved cachegroup_fallbacks.rst from development/traffic_ops_api/v12/cachegroup_fallbacks.rst to docs/source/api/v12 --- .../traffic_ops_api => api}/v12/cachegroup_fallbacks.rst | 0 docs/source/api/v12/index.rst | 1 + 2 files changed, 1 insertion(+) rename docs/source/{development/traffic_ops_api => api}/v12/cachegroup_fallbacks.rst (100%) diff --git a/docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst b/docs/source/api/v12/cachegroup_fallbacks.rst similarity index 100% rename from docs/source/development/traffic_ops_api/v12/cachegroup_fallbacks.rst rename to docs/source/api/v12/cachegroup_fallbacks.rst diff --git a/docs/source/api/v12/index.rst b/docs/source/api/v12/index.rst index 15e839c882..b60943d975 100644 --- a/docs/source/api/v12/index.rst +++ b/docs/source/api/v12/index.rst @@ -26,6 +26,7 @@ Traffic Ops API V1.2 cache cachegroup cachegroup_parameter + cachegroup_fallbacks cache_stats capability cdn From efed1cfeea3b854df914991e3de6cf64eb649b4c Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Fri, 25 May 2018 02:36:55 -0400 Subject: [PATCH 3/8] CrConfig's backupLocation Json node generation logic in GoLang --- lib/go-tc/cachegroupfallback.go | 55 +++++++++++++++++++ lib/go-tc/crconfig.go | 13 ++++- .../crconfig/edgelocations.go | 30 ++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 lib/go-tc/cachegroupfallback.go diff --git a/lib/go-tc/cachegroupfallback.go b/lib/go-tc/cachegroupfallback.go new file mode 100644 index 0000000000..7e1f95b88d --- /dev/null +++ b/lib/go-tc/cachegroupfallback.go @@ -0,0 +1,55 @@ +package tc + + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// A List of CACHEGROUPFALLBACKs Response +// swagger:response CACHEGROUPFALLBACKsResponse +// in: body +type CACHEGROUPFALLBACKsResponse struct { + // in: body + Response []CACHEGROUPFALLBACK `json:"response"` +} + +// A Single CACHEGROUPFALLBACK Response for Update and Create to depict what changed +// swagger:response CACHEGROUPFALLBACKResponse +// in: body +type CACHEGROUPFALLBACKResponse struct { + // in: body + Response CACHEGROUPFALLBACK `json:"response"` +} + +// CACHEGROUPFALLBACK ... +type CACHEGROUPFALLBACK struct { + + PrimaryCgId int `json:"primaryId" db:"primary_cg"` + BackupCgId int `json:"backupId" db:"backup_cg"` + SetOrder int `json:"setOrder" db:"set_order"` + +} + +// CACHEGROUPFALLBACKNullable ... +type CACHEGROUPFALLBACKNullable struct { + + PrimaryCgId *int `json:"primaryId" db:"primary_cg"` + BackupCgId *int `json:"backupId" db:"backup_cg"` + SetOrder *int `json:"setOrder" db:"set_order"` +} + diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index 66832714d2..18133bf0c4 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -151,9 +151,18 @@ type CRConfigDispersion struct { Shuffled bool `json:"shuffled,string"` } + +type CRConfigBackupLocations struct { + FallbackToClosest bool `json:"fallbackToClosest,string"` + List []string `json:"list,omitempty"` + +} + type CRConfigLatitudeLongitude struct { - Lat float64 `json:"latitude"` - Lon float64 `json:"longitude"` + Lat float64 `json:"latitude"` + Lon float64 `json:"longitude"` + BackupLocations CRConfigBackupLocations `json:"backupLocations,omitempty"` + } type CRConfigLatitudeLongitudeShort struct { diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index 2d43fd6547..1d523567c3 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -57,6 +57,36 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') if ttype == RouterTypeName { routerLocs[cachegroup] = latlon } else { + primaryCacheId := "" + if err := db.QueryRow(`select id from cachegroup where name = $1`, cachegroup).Scan(&primaryCacheId); err != nil { + return nil, nil, errors.New("Failed while retrieving from cachegroup: " + err.Error()) + } + + dbRows, err := db.Query(`select backup_cg from cachegroup_fallbacks where primary_cg = $1 order by set_order`, primaryCacheId) + + if err != nil { + return nil, nil, errors.New("Error retrieving from cachegroup_fallbacks: " + err.Error()) + } + defer dbRows.Close() + + index := 0 + for dbRows.Next() { + backup_id := "" + backup_name := "" + if err := dbRows.Scan(&backup_id); err != nil { + return nil, nil, errors.New("Error while scanning from cachegroup_fallbacks: " + err.Error()) + } + if err := db.QueryRow(`select name from cachegroup where id = $1`, backup_id).Scan(&backup_name); err != nil { + return nil, nil, errors.New("Error scanning cachegroup: " + err.Error()) + } else { + latlon.BackupLocations.List = append(latlon.BackupLocations.List, backup_name) + index++ + } + } + + if err := dbRows.Err(); err != nil { + return nil, nil, errors.New("Error iterating cachegroup_fallbacks rows: " + err.Error()) + } edgeLocs[cachegroup] = latlon } } From 7a902f00d4c92a4aecd097f3ca62ab66cb9a1dbd Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Fri, 25 May 2018 07:29:11 -0400 Subject: [PATCH 4/8] Addressing Rawlin's comment on; 1. Updating the document for cachegroup APIs 2. Validating fallback inputs --- docs/source/api/v12/cachegroup.rst | 35 ++++++-- traffic_ops/app/lib/API/CachegroupFallback.pm | 83 ++++++++++--------- 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/docs/source/api/v12/cachegroup.rst b/docs/source/api/v12/cachegroup.rst index 1831f49602..2c6d8f9832 100644 --- a/docs/source/api/v12/cachegroup.rst +++ b/docs/source/api/v12/cachegroup.rst @@ -23,7 +23,7 @@ Cache Group /api/1.2/cachegroups ++++++++++++++++++++ -**GET /api/1.1/cachegroups** +**GET /api/1.2/cachegroups** Authentication Required: Yes @@ -66,6 +66,8 @@ Cache Group +-----------------------------------+--------+--------------------------------------------------------------------------+ | ``typeName`` | string | The name of the type of Cache Group entry | +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/ failure of configured fallbacks | + +-----------------------------------+--------+--------------------------------------------------------------------------+ **Response Example** :: @@ -83,7 +85,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":true }, { "id": "22", @@ -97,7 +100,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":false } ], } @@ -168,6 +172,8 @@ Cache Group +-----------------------------------+--------+--------------------------------------------------------------------------+ | ``typeName`` | string | The name of the type of Cache Group entry | +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/ failure of configured fallbacks | + +-----------------------------------+--------+--------------------------------------------------------------------------+ **Response Example** :: @@ -185,7 +191,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":true } ], } @@ -448,6 +455,8 @@ Cache Group +---------------------------------+----------+-------------------------------------------------------------------+ | ``typeId`` | yes | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +---------------------------------+----------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | no | Behaviour on configured fallbacks failure, true / false | + +---------------------------------+----------+-------------------------------------------------------------------+ **Request Example** :: @@ -457,7 +466,8 @@ Cache Group "latitude": 12, "longitude": 45, "parentCachegroup": "cache_group_mid", - "typeId": 6 + "typeId": 6, + "fallbackToClosest":true } **Response Properties** @@ -485,6 +495,8 @@ Cache Group +------------------------------------+--------+-------------------------------------------------------------------+ | ``typeName`` | string | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +------------------------------------+--------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/failure of configured fallbacks | + +------------------------------------+--------+-------------------------------------------------------------------+ | ``lastUpdated`` | string | The Time / Date this entry was last updated | +------------------------------------+--------+-------------------------------------------------------------------+ | ``alerts`` | array | A collection of alert messages. | @@ -514,7 +526,8 @@ Cache Group 'typeName' : 'EDGE_LOC', 'id' : '104', 'parentCachegroupId' : '103', - 'secondaryParentCachegroupId' : null + 'secondaryParentCachegroupId' : null, + 'fallbackToClosest':true } } @@ -555,6 +568,8 @@ Cache Group +---------------------------------+----------+-------------------------------------------------------------------+ | ``typeName`` | yes | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +---------------------------------+----------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | no | Behaviour on configured fallbacks failure, true / false | + +---------------------------------+----------+-------------------------------------------------------------------+ **Request Example** :: @@ -564,7 +579,8 @@ Cache Group "latitude": 12, "longitude": 45, "parentCachegroup": "cache_group_mid", - "typeName": "EDGE_LOC" + "typeName": "EDGE_LOC", + "fallbackToClosest":true } **Response Properties** @@ -592,6 +608,8 @@ Cache Group +------------------------------------+--------+-------------------------------------------------------------------+ | ``typeName`` | string | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +------------------------------------+--------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/failure of configured fallbacks | + +------------------------------------+--------+-------------------------------------------------------------------+ | ``lastUpdated`` | string | The Time / Date this entry was last updated | +------------------------------------+--------+-------------------------------------------------------------------+ | ``alerts`` | array | A collection of alert messages. | @@ -621,7 +639,8 @@ Cache Group 'typeName' : 'EDGE_LOC', 'id' : '104', 'parentCachegroupId' : '103', - 'secondaryParentCachegroupId' : null + 'secondaryParentCachegroupId' : null, + 'fallbackToClosest':true } } diff --git a/traffic_ops/app/lib/API/CachegroupFallback.pm b/traffic_ops/app/lib/API/CachegroupFallback.pm index e0a831fdbc..0dfde46755 100644 --- a/traffic_ops/app/lib/API/CachegroupFallback.pm +++ b/traffic_ops/app/lib/API/CachegroupFallback.pm @@ -116,32 +116,23 @@ sub create { my $cache_id = $self->param('cacheGroupId'); my $params = $self->req->json; - if ( !defined($cache_id)) { - my @param_array = @{$params}; - $cache_id = $param_array[0]{cacheGroupId}; + if ( !&is_oper($self) ) { + return $self->forbidden(); } if ( !defined($params) ) { return $self->alert("parameters must be in JSON format, please check!"); } - if ( !&is_oper($self) ) { - return $self->forbidden(); - } - - #only integers - if ( $cache_id !~ /^\d+?$/ ) { - &log( $self, "No such Cachegroup id $cache_id"); - return $self->not_found(); + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; } - my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); - if ( !defined($cachegroup) ) { - return $self->not_found(); - } + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback($params, $cache_id); - if ( ($cachegroup->type->name ne "EDGE_LOC") ) { - return $self->alert("cachegroup should be type EDGE_LOC."); + if ( !$is_valid ) { + return $self->alert($result); } foreach my $config (@{ $params }) { @@ -198,32 +189,17 @@ sub update { my $cache_id = $self->param('cacheGroupId'); my $params = $self->req->json; - if ( !defined($cache_id)) { - my @param_array = @{$params}; - $cache_id = $param_array[0]{cacheGroupId}; - } - - if ( !defined($params) ) { - return $self->alert("parameters must be in JSON format, please check!"); - } - if ( !&is_oper($self) ) { return $self->forbidden(); } - #only integers - if ( $cache_id !~ /^\d+?$/ ) { - &log( $self, "No such Cachegroup id $cache_id"); - return $self->not_found(); - } - - my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); - if ( !defined($cachegroup) ) { - return $self->not_found(); + if ( !defined($params) ) { + return $self->alert("parameters must be in JSON format, please check!"); } - if ( ($cachegroup->type->name ne "EDGE_LOC") ) { - return $self->alert("cachegroup should be type EDGE_LOC."); + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; } my $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id } ); @@ -231,6 +207,12 @@ sub update { return $self->alert( "Backup list not configured for $cache_id, create and update" ); } + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback($params, $cache_id); + + if ( !$is_valid ) { + return $self->alert($result); + } + foreach my $config (@{ $params }) { my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); if ( !defined($rs_backup) ) { @@ -275,4 +257,31 @@ sub update { } } +sub is_valid_cachegroup_fallback { + my $self = shift; + my $params = shift; + my $cache_id = shift; + + if ( $cache_id !~ /^\d+?$/ ) { + return ( 0, "Invalid cachegroup id" ); + } + + my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); + if ( !defined($cachegroup) ) { + return ( 0, "Invalid cachegroup id, should be an integer" ); + } + + if ( ($cachegroup->type->name ne "EDGE_LOC") ) { + return ( 0, "cachegroup is not of type EDGE_LOC" ); + } + + foreach my $config (@{ $params }) { + if ( $config->{fallbackId} !~ /^\d+?$/ ) { + return ( 0, "Invalid cachegroup specified as fallback, should be an integer" ); + } + } + + return ( 1, "success" ); +} + 1; From 85a10676b80bb82863d598dab95d55c02c685d70 Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Mon, 28 May 2018 08:22:40 -0400 Subject: [PATCH 5/8] Addressing Rawlin's comments: 1. Avoiding multiple DB querries using Join 2. &log to error logs --- lib/go-tc/cachegroupfallback.go | 24 ++++++------ ...> 20180528000000_cache_group_fallback.sql} | 4 +- traffic_ops/app/lib/API/CachegroupFallback.pm | 37 +++++++------------ .../crconfig/edgelocations.go | 28 ++++++-------- 4 files changed, 39 insertions(+), 54 deletions(-) rename traffic_ops/app/db/migrations/{20180521000000_cache_group_fallback.sql => 20180528000000_cache_group_fallback.sql} (93%) diff --git a/lib/go-tc/cachegroupfallback.go b/lib/go-tc/cachegroupfallback.go index 7e1f95b88d..e7ffc182c3 100644 --- a/lib/go-tc/cachegroupfallback.go +++ b/lib/go-tc/cachegroupfallback.go @@ -20,24 +20,24 @@ package tc * under the License. */ -// A List of CACHEGROUPFALLBACKs Response -// swagger:response CACHEGROUPFALLBACKsResponse +// A List of cachegroupFallbacks Response +// swagger:response cachegroupFallbacksResponse // in: body -type CACHEGROUPFALLBACKsResponse struct { +type cachegroupFallbacksResponse struct { // in: body - Response []CACHEGROUPFALLBACK `json:"response"` + Response []cachegroupFallback `json:"response"` } -// A Single CACHEGROUPFALLBACK Response for Update and Create to depict what changed -// swagger:response CACHEGROUPFALLBACKResponse +// A Single cachegroupFallback Response for Update and Create to depict what changed +// swagger:response cachegroupFallbackResponse // in: body -type CACHEGROUPFALLBACKResponse struct { +type cachegroupFallbackResponse struct { // in: body - Response CACHEGROUPFALLBACK `json:"response"` + Response cachegroupFallback `json:"response"` } -// CACHEGROUPFALLBACK ... -type CACHEGROUPFALLBACK struct { +// cachegroupFallback ... +type cachegroupFallback struct { PrimaryCgId int `json:"primaryId" db:"primary_cg"` BackupCgId int `json:"backupId" db:"backup_cg"` @@ -45,8 +45,8 @@ type CACHEGROUPFALLBACK struct { } -// CACHEGROUPFALLBACKNullable ... -type CACHEGROUPFALLBACKNullable struct { +// cachegroupFallbackNullable ... +type cachegroupFallbackNullable struct { PrimaryCgId *int `json:"primaryId" db:"primary_cg"` BackupCgId *int `json:"backupId" db:"backup_cg"` diff --git a/traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql b/traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql similarity index 93% rename from traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql rename to traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql index 923895714e..dbce434efd 100644 --- a/traffic_ops/app/db/migrations/20180521000000_cache_group_fallback.sql +++ b/traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql @@ -19,8 +19,8 @@ -- cachegroup_fallbacks CREATE TABLE cachegroup_fallbacks ( - primary_cg bigint, - backup_cg bigint CHECK (primary_cg != backup_cg), + primary_cg bigint NOT NULL, + backup_cg bigint CHECK (primary_cg != backup_cg) NOT NULL, set_order bigint NOT NULL, CONSTRAINT fk_primary_cg FOREIGN KEY (primary_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, CONSTRAINT fk_backup_cg FOREIGN KEY (backup_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, diff --git a/traffic_ops/app/lib/API/CachegroupFallback.pm b/traffic_ops/app/lib/API/CachegroupFallback.pm index 0dfde46755..888fd0eb96 100644 --- a/traffic_ops/app/lib/API/CachegroupFallback.pm +++ b/traffic_ops/app/lib/API/CachegroupFallback.pm @@ -52,10 +52,10 @@ sub delete { &log( $self, "Backup configuration DELETED", "APICHANGE"); return $self->success_message("Backup configuration DELETED"); } else { - return $self->alert( "Backup configuration DELETED." ); + return $self->alert( "Backup configuration DELETE Failed!." ); } } else { - &log( $self, "No backup Cachegroups found"); + $self->app->log->error("No backup Cachegroups found"); return $self->not_found(); } } @@ -66,21 +66,10 @@ sub show { my $fallback_id = $self->param("fallbackId"); my $id = $cache_id ? $cache_id : $fallback_id; - #only integers - if ( $id !~ /^\d+?$/ ) { - &log( $self, "No such Cachegroup id $id"); - return $self->success([]); - } + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback(undef, $cache_id); - my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $id } )->single(); - if ( !defined($cachegroup) ) { - &log( $self, "No such Cachegroup $id"); - return $self->success([]); - } - - if ( ($cachegroup->type->name ne "EDGE_LOC") ) { - &log( $self, "cachegroup should be type EDGE_LOC."); - return $self->success([]); + if ( !$is_valid ) { + return $self->alert($result); } my $rs_backups = undef; @@ -106,7 +95,7 @@ sub show { } return $self->success( $response ); } else { - &log( $self, "No backup Cachegroups"); + $self->app->log->error("No backup Cachegroups"); return $self->success([]); } } @@ -138,12 +127,12 @@ sub create { foreach my $config (@{ $params }) { my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); if ( !defined($rs_backup) ) { - &log( $self, "ERROR Backup config: No such Cache Group $config->{fallbackId}"); + $self->app->log->error("ERROR Backup config: No such Cache Group $config->{fallbackId}"); next; } if ( ($rs_backup->type->name ne "EDGE_LOC") ) { - &log( $self, "ERROR Backup config: $config->{name} is not EDGE_LOC"); + $self->app->log->error("ERROR Backup config: $config->{name} is not EDGE_LOC"); next; } @@ -160,7 +149,7 @@ sub create { my $rs_data = $self->db->resultset('CachegroupFallback')->create($values)->insert(); if ( !defined($rs_data)) { - &log( $self, "Database operation for backup configuration for cache group $cache_id failed."); + $self->app->log->error("Database operation for backup configuration for cache group $cache_id failed."); } } @@ -216,12 +205,12 @@ sub update { foreach my $config (@{ $params }) { my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); if ( !defined($rs_backup) ) { - &log( $self, "ERROR Backup config: No such Cache Group $config->{fallbackId}"); + $self->app->log->error("ERROR Backup config: No such Cache Group $config->{fallbackId}"); next; } if ( ($rs_backup->type->name ne "EDGE_LOC") ) { - &log( $self, "ERROR Backup config: $config->{name} is not EDGE_LOC"); + $self->app->log->error("ERROR Backup config: $config->{name} is not EDGE_LOC"); next; } @@ -263,12 +252,12 @@ sub is_valid_cachegroup_fallback { my $cache_id = shift; if ( $cache_id !~ /^\d+?$/ ) { - return ( 0, "Invalid cachegroup id" ); + return ( 0, "Invalid cachegroup id, should be an integer" ); } my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); if ( !defined($cachegroup) ) { - return ( 0, "Invalid cachegroup id, should be an integer" ); + return ( 0, "Invalid cachegroup id" ); } if ( ($cachegroup->type->name ne "EDGE_LOC") ) { diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index 1d523567c3..b981aa5712 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -32,7 +32,7 @@ func makeLocations(cdn string, db *sql.DB) (map[string]tc.CRConfigLatitudeLongit // TODO test whether it's faster to do a single query, joining lat/lon into servers q := ` -select cg.name, t.name as type, cg.latitude, cg.longitude from cachegroup as cg +select cg.name, cg.id, t.name as type, cg.latitude, cg.longitude from cachegroup as cg inner join server as s on s.cachegroup = cg.id inner join type as t on t.id = s.type inner join status as st ON st.id = s.status @@ -49,20 +49,20 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') for rows.Next() { cachegroup := "" + primaryCacheID := "" ttype := "" latlon := tc.CRConfigLatitudeLongitude{} - if err := rows.Scan(&cachegroup, &ttype, &latlon.Lat, &latlon.Lon); err != nil { + if err := rows.Scan(&cachegroup, &primaryCacheID, &ttype, &latlon.Lat, &latlon.Lon); err != nil { return nil, nil, errors.New("Error scanning cachegroup: " + err.Error()) } if ttype == RouterTypeName { routerLocs[cachegroup] = latlon } else { - primaryCacheId := "" - if err := db.QueryRow(`select id from cachegroup where name = $1`, cachegroup).Scan(&primaryCacheId); err != nil { - return nil, nil, errors.New("Failed while retrieving from cachegroup: " + err.Error()) - } - - dbRows, err := db.Query(`select backup_cg from cachegroup_fallbacks where primary_cg = $1 order by set_order`, primaryCacheId) + q := `select cachegroup.name from cachegroup_fallbacks +join cachegroup on cachegroup_fallbacks.backup_cg = cachegroup.id +and cachegroup_fallbacks.primary_cg = $1 +` + dbRows, err := db.Query(q, primaryCacheID) if err != nil { return nil, nil, errors.New("Error retrieving from cachegroup_fallbacks: " + err.Error()) @@ -71,20 +71,16 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') index := 0 for dbRows.Next() { - backup_id := "" - backup_name := "" - if err := dbRows.Scan(&backup_id); err != nil { + backupName := "" + if err := dbRows.Scan(&backupName); err != nil { return nil, nil, errors.New("Error while scanning from cachegroup_fallbacks: " + err.Error()) - } - if err := db.QueryRow(`select name from cachegroup where id = $1`, backup_id).Scan(&backup_name); err != nil { - return nil, nil, errors.New("Error scanning cachegroup: " + err.Error()) } else { - latlon.BackupLocations.List = append(latlon.BackupLocations.List, backup_name) + latlon.BackupLocations.List = append(latlon.BackupLocations.List, backupName) index++ } } - if err := dbRows.Err(); err != nil { + if err := dbRows.Err(); err != nil { return nil, nil, errors.New("Error iterating cachegroup_fallbacks rows: " + err.Error()) } edgeLocs[cachegroup] = latlon From 5f6524498b740ddba011155355a6b1fb5b0fdb69 Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Tue, 29 May 2018 13:24:57 -0400 Subject: [PATCH 6/8] Rawlin's comments 1. Adding cache group id for audit logs for DELETE. 2. GoLang naming conventions --- lib/go-tc/cachegroupfallback.go | 24 +++++++++---------- traffic_ops/app/lib/API/CachegroupFallback.pm | 12 +++++++--- .../crconfig/edgelocations.go | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/lib/go-tc/cachegroupfallback.go b/lib/go-tc/cachegroupfallback.go index e7ffc182c3..ecd1d8c3f4 100644 --- a/lib/go-tc/cachegroupfallback.go +++ b/lib/go-tc/cachegroupfallback.go @@ -20,24 +20,24 @@ package tc * under the License. */ -// A List of cachegroupFallbacks Response -// swagger:response cachegroupFallbacksResponse +// A List of CacheGroupFallbacks Response +// swagger:response CacheGroupFallbacksResponse // in: body -type cachegroupFallbacksResponse struct { +type CacheGroupFallbacksResponse struct { // in: body - Response []cachegroupFallback `json:"response"` + Response []CacheGroupFallback `json:"response"` } -// A Single cachegroupFallback Response for Update and Create to depict what changed -// swagger:response cachegroupFallbackResponse +// A Single CacheGroupFallback Response for Update and Create to depict what changed +// swagger:response CacheGroupFallbackResponse // in: body -type cachegroupFallbackResponse struct { +type CacheGroupFallbackResponse struct { // in: body - Response cachegroupFallback `json:"response"` + Response CacheGroupFallback `json:"response"` } -// cachegroupFallback ... -type cachegroupFallback struct { +// CacheGroupFallback ... +type CacheGroupFallback struct { PrimaryCgId int `json:"primaryId" db:"primary_cg"` BackupCgId int `json:"backupId" db:"backup_cg"` @@ -45,8 +45,8 @@ type cachegroupFallback struct { } -// cachegroupFallbackNullable ... -type cachegroupFallbackNullable struct { +// CacheGroupFallbackNullable ... +type CacheGroupFallbackNullable struct { PrimaryCgId *int `json:"primaryId" db:"primary_cg"` BackupCgId *int `json:"backupId" db:"backup_cg"` diff --git a/traffic_ops/app/lib/API/CachegroupFallback.pm b/traffic_ops/app/lib/API/CachegroupFallback.pm index 888fd0eb96..a9114dac57 100644 --- a/traffic_ops/app/lib/API/CachegroupFallback.pm +++ b/traffic_ops/app/lib/API/CachegroupFallback.pm @@ -33,6 +33,7 @@ sub delete { my $fallback_id = $self->param('fallbackId'); my $params = $self->req->json; my $rs_backups = undef; + my $result = ""; if ( !&is_oper($self) ) { return $self->forbidden(); @@ -40,22 +41,27 @@ sub delete { if ( defined ($cache_id) && defined($fallback_id) ) { $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id , backup_cg => $fallback_id} ); + $result = "Backup Cachegroup $fallback_id DELETED from cachegroup $cache_id fallback list"; } elsif (defined ($cache_id)) { $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id} ); + $result = "Fallback list for Cachegroup $cache_id DELETED"; } elsif (defined ($fallback_id)) { + $result = "Cachegroup $fallback_id DELETED from all the configured fallback lists"; $rs_backups = $self->db->resultset('CachegroupFallback')->search( { backup_cg => $fallback_id} ); + } else { + return $self->alert("Invalid input"); } if ( ($rs_backups->count > 0) ) { my $del_records = $rs_backups->delete(); if ($del_records) { - &log( $self, "Backup configuration DELETED", "APICHANGE"); - return $self->success_message("Backup configuration DELETED"); + &log( $self, $result, "APICHANGE"); + return $self->success( $result ); } else { return $self->alert( "Backup configuration DELETE Failed!." ); } } else { - $self->app->log->error("No backup Cachegroups found"); + $self->app->log->error( "No backup Cachegroups found" ); return $self->not_found(); } } diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index b981aa5712..61f43bac77 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -49,7 +49,7 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') for rows.Next() { cachegroup := "" - primaryCacheID := "" + primaryCacheID := 0 ttype := "" latlon := tc.CRConfigLatitudeLongitude{} if err := rows.Scan(&cachegroup, &primaryCacheID, &ttype, &latlon.Lat, &latlon.Lon); err != nil { From 14588089d340ba1285732d56164a82d600340a68 Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Thu, 7 Jun 2018 12:42:38 -0400 Subject: [PATCH 7/8] Rawlins comment - parsing fallback to closest and make sure null is handled --- lib/go-tc/v13/cachegroups.go | 2 ++ .../traffic_ops_golang/crconfig/edgelocations.go | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/go-tc/v13/cachegroups.go b/lib/go-tc/v13/cachegroups.go index ee203fcd5c..1c0f550185 100644 --- a/lib/go-tc/v13/cachegroups.go +++ b/lib/go-tc/v13/cachegroups.go @@ -37,6 +37,7 @@ type CacheGroup struct { ParentCachegroupID int `json:"parentCachegroupId" db:"parent_cachegroup_id"` SecondaryParentName string `json:"secondaryParentCachegroupName"` SecondaryParentCachegroupID int `json:"secondaryParentCachegroupId" db:"secondary_parent_cachegroup_id"` + FallbackToClosest bool `json:"fallbackToClosest" db:"fallback_to_closest` Type string `json:"typeName" db:"type_name"` // aliased to type_name to disambiguate struct scans due to join on 'type' table TypeID int `json:"typeId" db:"type_id"` // aliased to type_id to disambiguate struct scans due join on 'type' table LastUpdated tc.TimeNoMod `json:"lastUpdated" db:"last_updated"` @@ -52,6 +53,7 @@ type CacheGroupNullable struct { ParentCachegroupID *int `json:"parentCachegroupId" db:"parent_cachegroup_id"` SecondaryParentName *string `json:"secondaryParentCachegroupName"` SecondaryParentCachegroupID *int `json:"secondaryParentCachegroupId" db:"secondary_parent_cachegroup_id"` + FallbackToClosest *bool `json:"fallbackToClosest" db:"fallback_to_closest"` Type *string `json:"typeName" db:"type_name"` // aliased to type_name to disambiguate struct scans due to join on 'type' table TypeID *int `json:"typeId" db:"type_id"` // aliased to type_id to disambiguate struct scans due join on 'type' table LastUpdated *tc.TimeNoMod `json:"lastUpdated" db:"last_updated"` diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index 61f43bac77..ab9918e6b8 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -32,7 +32,7 @@ func makeLocations(cdn string, db *sql.DB) (map[string]tc.CRConfigLatitudeLongit // TODO test whether it's faster to do a single query, joining lat/lon into servers q := ` -select cg.name, cg.id, t.name as type, cg.latitude, cg.longitude from cachegroup as cg +select cg.name, cg.id, t.name as type, cg.latitude, cg.longitude, cg.fallback_to_closest from cachegroup as cg inner join server as s on s.cachegroup = cg.id inner join type as t on t.id = s.type inner join status as st ON st.id = s.status @@ -51,8 +51,9 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') cachegroup := "" primaryCacheID := 0 ttype := "" + var fallbackToClosest *bool latlon := tc.CRConfigLatitudeLongitude{} - if err := rows.Scan(&cachegroup, &primaryCacheID, &ttype, &latlon.Lat, &latlon.Lon); err != nil { + if err := rows.Scan(&cachegroup, &primaryCacheID, &ttype, &latlon.Lat, &latlon.Lon, &fallbackToClosest); err != nil { return nil, nil, errors.New("Error scanning cachegroup: " + err.Error()) } if ttype == RouterTypeName { @@ -69,6 +70,14 @@ and cachegroup_fallbacks.primary_cg = $1 } defer dbRows.Close() + + if fallbackToClosest == nil { + fallbackToClosest = new(bool) + *fallbackToClosest = true + + } + latlon.BackupLocations.FallbackToClosest = *fallbackToClosest + index := 0 for dbRows.Next() { backupName := "" From 4174ac1dbbeaa5c92541863aa80d87cda7f0eaa3 Mon Sep 17 00:00:00 2001 From: Vijayanand Subramanian Date: Tue, 12 Jun 2018 20:40:48 -0400 Subject: [PATCH 8/8] Rawlins comment on using group by to enforce order which is not handled properly in GOLang implementation --- traffic_ops/traffic_ops_golang/crconfig/edgelocations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index ab9918e6b8..7f9cb96420 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -61,7 +61,7 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') } else { q := `select cachegroup.name from cachegroup_fallbacks join cachegroup on cachegroup_fallbacks.backup_cg = cachegroup.id -and cachegroup_fallbacks.primary_cg = $1 +and cachegroup_fallbacks.primary_cg = $1 order by cachegroup_fallbacks.set_order ` dbRows, err := db.Query(q, primaryCacheID)