OpenStack port spec issue backport to v3.25

networking-calico/.testr.conf

@@ -2,10 +2,9 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-0} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             ${PYTHON:-python} -m coverage run -a -m nose2 -v -s networking_calico/tests
-             ${PYTHON:-python} -m coverage run -a -m nose2 -v networking_calico.plugins.ml2.drivers.calico.test.test_election
-             ${PYTHON:-python} -m coverage run -a -m nose2 -v networking_calico.plugins.ml2.drivers.calico.test.test_plugin_etcd
-             ${PYTHON:-python} -m coverage run -a -m nose2 -v networking_calico.plugins.ml2.drivers.calico.test.test_compaction
+	     sh -c '\
+             ${PYTHON:-python} -m coverage run -a -m subunit.run discover -t . networking_calico/tests $LISTOPT $IDOPTION && \
+             ${PYTHON:-python} -m coverage run -a -m subunit.run discover -t . networking_calico/plugins/ml2/drivers/calico/test $LISTOPT $IDOPTION'
 
 test_id_option=--load-list $IDFILE
 test_list_option=--list

networking-calico/Dockerfile

@@ -4,4 +4,4 @@ FROM python:3.8
 
 COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd
 
-RUN pip3 install tox
+RUN pip3 install tox==3.25.1

networking-calico/Makefile

@@ -4,6 +4,10 @@ include ../metadata.mk
 # TODO: Release
 ###############################################################################
 
+export BUILDKIT_PROGRESS=plain
+
 tox:
+	curl -L https://releases.openstack.org/constraints/upper/ussuri -o upper-constraints-ussuri.txt
+	sed -i '/etcd3gw/d' upper-constraints-ussuri.txt
 	docker build -t networking-calico-test .
-	docker run -it --user `id -u`:`id -g` -v `pwd`:/code -w /code -e HOME=/code --rm networking-calico-test tox
+	docker run -it --user `id -u`:`id -g` -v `pwd`:/code -w /code -e HOME=/code -e PIP_CONSTRAINT=/code/upper-constraints-ussuri.txt --rm networking-calico-test tox

networking-calico/debian/control

@@ -2,7 +2,7 @@ Source: networking-calico
 Section: net
 Priority: optional
 Maintainer: Neil Jerram <[email protected]>
-Build-Depends: debhelper (>= 8.0.0), dh-systemd, dh-python, python3-all, python3-setuptools
+Build-Depends: debhelper (>= 8.0.0), base-files (>= 11.1~) | dh-systemd, dh-python, python3-all, python3-setuptools
 Standards-Version: 3.9.4
 
 Package: calico-compute

networking-calico/devstack/bootstrap.sh

@@ -16,6 +16,11 @@
 
 set -ex
 
+sudo pip uninstall -y setuptools
+sudo rm -rf /usr/local/lib/python3.8/dist-packages/setuptools-*.dist-info
+sudo find / -name "*setuptools*" || true
+sudo pip list || true
+
 #------------------------------------------------------------------------------
 # IMPORTANT - Review before use!
 #
@@ -96,15 +101,9 @@ sudo sysctl -w net.ipv6.conf.all.forwarding=1
 
 # Clone the DevStack repository (if not already present).
 test -e devstack || \
-    git clone ${DEVSTACK_REPO:-https://opendev.org/openstack/devstack}
+    git clone -b ${DEVSTACK_BRANCH:-master} ${DEVSTACK_REPO:-https://github.com/openstack/devstack} --depth=1
 cd devstack
 
-# If DEVSTACK_BRANCH has been specified, check out that branch.  (Otherwise we
-# use DevStack's master branch.)
-if [ -n "$DEVSTACK_BRANCH" ]; then
-    git checkout ${DEVSTACK_BRANCH}
-fi
-
 # Prepare DevStack config.
 cat > local.conf <<EOF
 [[local|localrc]]
@@ -121,7 +120,17 @@ disable_service horizon
 LOGFILE=stack.log
 LOG_COLOR=False
 
-TEMPEST_BRANCH=29.0.0
+TEMPEST_BRANCH=29.1.0
+
+# We clone from GitHub because we commonly used to hit GnuTLS errors when git cloning OpenStack
+# repos from opendev.org (which is the default server), for example:
+#
+# Cloning into 'devstack'...
+# remote: Enumerating objects: 28788, done.
+# remote: Counting objects: 100% (28788/28788), done.
+# remote: Compressing objects: 100% (9847/9847), done.
+# error: RPC failed; curl 56 GnuTLS recv error (-9): A TLS packet with unexpected length was received.
+GIT_BASE=https://github.com
 
 EOF
 
@@ -154,11 +163,16 @@ ls -la /opt/stack
 
 # Stack!
 sudo -u stack -H -E bash -x <<'EOF'
-
-set
 cd /opt/stack/devstack
 ./stack.sh
+EOF
 
+# We use a fresh `sudo -u stack -H -E bash ...` invocation here, because with
+# OpenStack Yoga it appears there is something in the stack.sh setup that
+# closes stdin, and that means that bash doesn't read any further commands from
+# stdin after the exit of the ./stack.sh line.
+sudo -u stack -H -E bash -x <<'EOF'
+cd /opt/stack/devstack
 if ! ${TEMPEST:-false}; then
     if [ x${SERVICE_HOST:-$HOSTNAME} = x$HOSTNAME ]; then
         # We're not running Tempest tests, and we're on the controller node.

networking-calico/networking_calico/plugins/ml2/drivers/calico/policy.py

@@ -209,15 +209,17 @@ def _neutron_rule_to_etcd_rule(rule):
         entity_rule['nets'] = [rule['remote_ip_prefix']]
     LOG.debug("=> Entity rule %s" % entity_rule)
 
+    if port_spec is not None:
+        if rule['direction'] == 'ingress':
+            etcd_rule['destination'] = {'ports': port_spec}
+        else:
+            entity_rule['ports'] = port_spec
+
     # Store in source or destination field of the overall rule.
     if entity_rule:
         if rule['direction'] == 'ingress':
             etcd_rule['source'] = entity_rule
-            if port_spec is not None:
-                etcd_rule['destination'] = {'ports': port_spec}
         else:
-            if port_spec is not None:
-                entity_rule['ports'] = port_spec
             etcd_rule['destination'] = entity_rule
 
     LOG.debug("=> %s Calico rule %s" % (rule['direction'], etcd_rule))

networking-calico/networking_calico/plugins/ml2/drivers/calico/test/test_plugin_etcd.py

@@ -1203,6 +1203,63 @@ def test_neutron_rule_to_etcd_rule_protocol_any(self):
                 'ipVersion': 4,
             })
 
+    def test_sg_rule_ingress_no_remote_ip_prefix(self):
+        # SG ingress rule with ports but no remote IP prefix
+        self.assertNeutronToEtcd(_neutron_rule_from_dict({
+            "protocol": "tcp",
+            "port_range_min": 25,
+            "port_range_max": 34,
+        }), {
+            'action': 'Allow',
+            'destination': {'ports': ['25:34']},
+            'ipVersion': 4,
+            'protocol': 'TCP',
+        })
+
+    def test_sg_rule_egress_no_remote_ip_prefix(self):
+        # SG egress rule with ports but no remote IP prefix
+        self.assertNeutronToEtcd(_neutron_rule_from_dict({
+            "direction": "egress",
+            "protocol": "tcp",
+            "port_range_min": 25,
+            "port_range_max": 34,
+        }), {
+            'action': 'Allow',
+            'destination': {'ports': ['25:34']},
+            'ipVersion': 4,
+            'protocol': 'TCP',
+        })
+
+    def test_sg_rule_ingress_with_remote_ip_prefix(self):
+        # SG ingress rule with ports and remote IP prefix
+        self.assertNeutronToEtcd(_neutron_rule_from_dict({
+            "protocol": "tcp",
+            "remote_ip_prefix": "1.2.3.0/24",
+            "port_range_min": 25,
+            "port_range_max": 34,
+        }), {
+            'action': 'Allow',
+            'destination': {'ports': ['25:34']},
+            'ipVersion': 4,
+            'protocol': 'TCP',
+            'source': {'nets': ['1.2.3.0/24']},
+        })
+
+    def test_sg_rule_egress_with_remote_ip_prefix(self):
+        # SG egress rule with ports and remote IP prefix
+        self.assertNeutronToEtcd(_neutron_rule_from_dict({
+            "direction": "egress",
+            "protocol": "tcp",
+            "remote_ip_prefix": "1.2.3.0/24",
+            "port_range_min": 25,
+            "port_range_max": 34,
+        }), {
+            'action': 'Allow',
+            'destination': {'nets': ['1.2.3.0/24'], 'ports': ['25:34']},
+            'ipVersion': 4,
+            'protocol': 'TCP',
+        })
+
     def test_not_master_does_not_resync(self):
         """Test that a driver that is not master does not resync."""
         # Initialize the state early to put the elector in place, then override
@@ -1783,16 +1840,16 @@ def _cancel():
             mock.call("hostname", "ep1", {"status": "up"}, priority="low"),
         ], any_order=True)
 
-        # Resync after deleting the Felix status.  We should see the other
-        # endpoint reported with status None.
+        # Resync after deleting the Felix status.  This does not affect the
+        # status of ep1.
         del self.etcd_data[felix_status_key]
         self.driver.on_felix_alive.reset_mock()
         self.driver.on_port_status_changed.reset_mock()
         self.clientv3.watch_prefix.return_value = _iterator(), _cancel
         self.watcher.start()
         self.driver.on_felix_alive.assert_not_called()
         self.driver.on_port_status_changed.assert_has_calls([
-            mock.call("hostname", "ep1", None, priority="low"),
+            mock.call("hostname", "ep1", {"status": "up"}, priority="low"),
         ], any_order=True)
 
         # Resync with some follow-on events; checks that the priority goes

networking-calico/networking_calico/tests/test_dhcp_agent.py

@@ -83,95 +83,101 @@ def test_mainline(self):
         # Create the DHCP agent.
         agent = CalicoDhcpAgent()
 
+        # Mock the MTU watcher.
+        agent.etcd.mtu_watcher = mock.Mock()
+        agent.etcd.mtu_watcher.get_mtu.return_value = 1500
+
+        # Mock the dnsmasq updater.
+        agent.etcd.dnsmasq_updater = mock.Mock()
+
         # Check that running it invokes the etcd watcher loop.
         with mock.patch.object(agent, 'etcd') as etcdobj:
             agent.run()
             etcdobj.start.assert_called_with()
 
         # Notify initial snapshot (empty).
-        with mock.patch.object(agent, 'call_driver') as call_driver:
-            snapshot_data = agent.etcd._pre_snapshot_hook()
-            agent.etcd._post_snapshot_hook(snapshot_data)
-            call_driver.assert_not_called()
-
-        with mock.patch.object(agent, 'call_driver') as call_driver:
-            # Notify subnets.
-            agent.etcd.subnet_watcher.on_subnet_set(
-                EtcdResponse(value=json.dumps({
-                    'cidr': '10.28.0.0/24',
-                    'gateway_ip': '10.28.0.1',
-                    'host_routes': []
-                })),
-                'v4subnet-1'
-            )
-            agent.etcd.subnet_watcher.on_subnet_set(
-                EtcdResponse(value=json.dumps({
-                    'cidr': '2001:db8:1::/80',
-                    'gateway_ip': '2001:db8:1::1'
-                })),
-                'v6subnet-1'
-            )
-            agent.etcd.subnet_watcher.on_subnet_set(
-                EtcdResponse(value=json.dumps({
-                    'cidr': '10.29.0.0/24',
-                    'gateway_ip': '10.29.0.1',
-                    'host_routes': [{'destination': '11.11.0.0/16',
-                                     'nexthop': '10.65.0.1'}]
-                })),
-                'v4subnet-2'
-            )
-
-            # Notify an endpoint.
-            agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
-                'interfaceName': 'tap1234',
-                'mac': 'fe:16:65:12:33:44',
-                'profiles': ['profile-1'],
-                'ipNetworks': ['10.28.0.2/32', '2001:db8:1::2/128'],
-                'ipv4Gateway': '10.28.0.1',
-                'ipv6Gateway': '2001:db8:1::1'
-            }})),
-                make_endpoint_name('endpoint-1')
-            )
-
-            # Check DHCP driver was asked to restart.
-            call_driver.assert_called_with('restart', mock.ANY)
-
-            # Notify another endpoint (using the same subnets).
-            agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
-                'interfaceName': 'tap5678',
-                'mac': 'fe:16:65:12:33:55',
-                'profiles': ['profile-1'],
-                'ipNetworks': ['10.28.0.3/32', '2001:db8:1::3/128'],
-                'ipv4Gateway': '10.28.0.1',
-                'ipv6Gateway': '2001:db8:1::1',
-                'fqdn': 'calico-vm17.datcon.co.uk'
-            }})),
-                make_endpoint_name('endpoint-2')
-            )
-
-            # Check DHCP driver was asked to restart.
-            call_driver.assert_called_with('restart', mock.ANY)
-
-            # Notify deletion of the first endpoint.
-            agent.etcd.on_endpoint_delete(None,
-                                          make_endpoint_name('endpoint-1'))
-
-            # Check DHCP driver was asked to reload allocations.
-            call_driver.assert_called_with('restart', mock.ANY)
-
-            # Notify another endpoint using a new subnet.
-            agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
-                'interfaceName': 'tapABCD',
-                'mac': 'fe:16:65:12:33:66',
-                'profiles': ['profile-1'],
-                'ipNetworks': ['10.29.0.3/32'],
-                'ipv4Gateway': '10.29.0.1',
-            }})),
-                make_endpoint_name('endpoint-3')
-            )
+        snapshot_data = agent.etcd._pre_snapshot_hook()
+        agent.etcd._post_snapshot_hook(snapshot_data)
+        agent.etcd.dnsmasq_updater.update_network.assert_not_called()
+
+        # Notify subnets.
+        agent.etcd.subnet_watcher.on_subnet_set(
+            EtcdResponse(value=json.dumps({
+                'cidr': '10.28.0.0/24',
+                'gateway_ip': '10.28.0.1',
+                'host_routes': []
+            })),
+            'v4subnet-1'
+        )
+        agent.etcd.subnet_watcher.on_subnet_set(
+            EtcdResponse(value=json.dumps({
+                'cidr': '2001:db8:1::/80',
+                'gateway_ip': '2001:db8:1::1'
+            })),
+            'v6subnet-1'
+        )
+        agent.etcd.subnet_watcher.on_subnet_set(
+            EtcdResponse(value=json.dumps({
+                'cidr': '10.29.0.0/24',
+                'gateway_ip': '10.29.0.1',
+                'host_routes': [{'destination': '11.11.0.0/16',
+                                 'nexthop': '10.65.0.1'}]
+            })),
+            'v4subnet-2'
+        )
 
-            # Check DHCP driver was asked to restart.
-            call_driver.assert_called_with('restart', mock.ANY)
+        # Notify an endpoint.
+        agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
+            'interfaceName': 'tap1234',
+            'mac': 'fe:16:65:12:33:44',
+            'profiles': ['profile-1'],
+            'ipNetworks': ['10.28.0.2/32', '2001:db8:1::2/128'],
+            'ipv4Gateway': '10.28.0.1',
+            'ipv6Gateway': '2001:db8:1::1'
+        }})),
+                                   make_endpoint_name('endpoint-1'))
+
+        # Check DHCP driver was asked to restart.
+        agent.etcd.dnsmasq_updater.update_network.assert_called()
+        agent.etcd.dnsmasq_updater.update_network.reset_mock()
+
+        # Notify another endpoint (using the same subnets).
+        agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
+            'interfaceName': 'tap5678',
+            'mac': 'fe:16:65:12:33:55',
+            'profiles': ['profile-1'],
+            'ipNetworks': ['10.28.0.3/32', '2001:db8:1::3/128'],
+            'ipv4Gateway': '10.28.0.1',
+            'ipv6Gateway': '2001:db8:1::1',
+            'fqdn': 'calico-vm17.datcon.co.uk'
+        }})),
+                                   make_endpoint_name('endpoint-2'))
+
+        # Check DHCP driver was asked to restart.
+        agent.etcd.dnsmasq_updater.update_network.assert_called()
+        agent.etcd.dnsmasq_updater.update_network.reset_mock()
+
+        # Notify deletion of the first endpoint.
+        agent.etcd.on_endpoint_delete(None,
+                                      make_endpoint_name('endpoint-1'))
+
+        # Check DHCP driver was asked to reload allocations.
+        agent.etcd.dnsmasq_updater.update_network.assert_called()
+        agent.etcd.dnsmasq_updater.update_network.reset_mock()
+
+        # Notify another endpoint using a new subnet.
+        agent.etcd.on_endpoint_set(EtcdResponse(value=json.dumps({'spec': {
+            'interfaceName': 'tapABCD',
+            'mac': 'fe:16:65:12:33:66',
+            'profiles': ['profile-1'],
+            'ipNetworks': ['10.29.0.3/32'],
+            'ipv4Gateway': '10.29.0.1',
+        }})),
+                                   make_endpoint_name('endpoint-3'))
+
+        # Check DHCP driver was asked to restart.
+        agent.etcd.dnsmasq_updater.update_network.assert_called()
+        agent.etcd.dnsmasq_updater.update_network.reset_mock()
 
     def test_initial_snapshot(self):
         # Create the DHCP agent.