Changes to make vostools work with the new cavern and vault ser…

…vices (CADC-12912) (#215)

.github/workflows/cibuild.yml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
         package: [vos]
     steps:
     - name: Checkout code

vos/setup.cfg

@@ -21,9 +21,11 @@ omit = */tests/*
 [tool:pytest]
 minversion = 2.2
 norecursedirs = build docs/_build
-doctest_plus = enabled
 testpaths = vos
 
+[flake8]
+max-line-length = 120
+
 [pep8]
 # E101 - mix of tabs and spaces
 # W191 - use of tabs
@@ -54,7 +56,7 @@ install_requires =
     aenum
 
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.4
+version = 3.5
 
 [options.extras_require]
 test =

vos/setup.py

@@ -4,7 +4,6 @@
 import glob
 import os
 import sys
-import imp
 from setuptools.command.test import test as TestCommand
 from setuptools import find_packages
 
@@ -128,7 +127,7 @@ def run_tests(self):
       use_2to3=False,
       setup_requires=['pytest-runner'],
       entry_points=entry_points,
-      python_requires='!=3.0.*, !=3.1.*, !=3.2.*, !=3.3, <4',
+      python_requires='>=3.7, <4',
       packages=find_packages(),
       package_data={PACKAGENAME: ['data/*', 'tests/data/*', '*/data/*', '*/tests/data/*']},
       classifiers=[

vos/tox.ini

@@ -5,7 +5,7 @@ name = vos
 
 [tox]
 envlist =
-    py{37,38,39,310,311}
+    py{37,38,39,310,311,312}
 requires =
     pip >= 19.3.1
 

vos/vos/commands/vchmod.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2022.                            (c) 2022.
+#  (c) 2024.                            (c) 2024.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -71,7 +71,7 @@
 """
 from ..vos import Client
 from ..vos import CADC_GMS_PREFIX
-from ..commonparser import CommonParser, set_logging_level_from_args,\
+from ..commonparser import CommonParser, set_logging_level_from_args, \
     URI_DESCRIPTION
 from ..commonparser import exit_on_exception
 import logging
@@ -190,20 +190,23 @@ def vchmod():
                         vospace_token=opt.token,
                         insecure=opt.insecure)
         node = client.get_node(opt.node)
-        if opt.recursive:
-            node.props.clear()
-            node.clear_properties()
-            # del node.node.findall(vos.Node.PROPERTIES)[0:]
+        node.props.clear()
+        node.clear_properties()
         if 'readgroup' in props:
             node.chrgrp(props['readgroup'])
         if 'writegroup' in props:
             node.chwgrp(props['writegroup'])
         if 'ispublic' in props:
             node.set_public(props['ispublic'])
         logging.debug("Node: {0}".format(node))
-        status = client.update(node, opt.recursive)
-        if status:
-            sys.exit(status)
+        successes, failures = client.update(node, opt.recursive)
+        if opt.recursive:
+            if failures:
+                logging.error('WARN. updated count: {}, failed count: {}\n'.
+                              format(successes, failures))
+                sys.exit(-1)
+            else:
+                logging.info('DONE. updated count: {}\n'.format(successes))
     except Exception as ex:
         exit_on_exception(ex)
 

vos/vos/commands/vcp.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2022.                            (c) 2022.
+#  (c) 2024.                            (c) 2024.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -69,7 +69,7 @@
 """copy files vospace to local or local to VOSpace"""
 from .. import md5_cache
 from .. import vos
-from ..commonparser import CommonParser, set_logging_level_from_args,\
+from ..commonparser import CommonParser, set_logging_level_from_args, \
     exit_on_exception, URI_DESCRIPTION
 
 try:
@@ -324,19 +324,6 @@ def copy(source_name, destination_name, exclude=None, include=None,
                         if ans != 'y':
                             raise Exception("File exists")
 
-                if not access(os.path.dirname(destination_name), os.F_OK):
-                    raise OSError(errno.EEXIST,
-                                  "vcp: ContainerNode %s does not exist" %
-                                  os.path.dirname(
-                                      destination_name))
-
-                if not isdir(os.path.dirname(destination_name)) and not islink(
-                        os.path.dirname(destination_name)):
-                    raise OSError(errno.ENOTDIR,
-                                  "vcp: %s is not a ContainerNode or LinkNode"
-                                  % os.path.dirname(
-                                      destination_name))
-
                 skip = False
                 if exclude is not None:
                     for thisIgnore in exclude.split(','):

vos/vos/commands/vrm.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2022.                            (c) 2022.
+#  (c) 2024.                            (c) 2024.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -68,6 +68,8 @@
 
 """remove a vospace data node, fails if container node or node is locked."""
 import logging
+import sys
+
 from ..commonparser import set_logging_level_from_args, exit_on_exception, \
     CommonParser, URI_DESCRIPTION
 from .. import vos
@@ -81,8 +83,12 @@
 
 def vrm():
     parser = CommonParser(description=DESCRIPTION)
+    parser.add_argument(
+        "-R", "--recursive", action="store_true",
+        help="Delete a file or directory even if it's not empty.",
+        default=False)
     parser.add_argument('node',
-                        help='dataNode or linkNode to delete from VOSpace',
+                        help='file, link or possibly directory to delete from VOSpace',
                         nargs='+')
 
     args = parser.parse_args()
@@ -97,17 +103,27 @@ def vrm():
             if not client.is_remote_file(node):
                 raise Exception(
                     '{} is not a valid VOSpace handle'.format(node))
-            if not node.endswith('/'):
-                if client.get_node(node).islink():
-                    logging.info('deleting link {}'.format(node))
-                    client.delete(node)
-                elif client.isfile(node):
-                    logging.info('deleting {}'.format(node))
-                    client.delete(node)
-            elif client.isdir(node):
-                raise Exception('{} is a directory'.format(node))
+            if args.recursive:
+                successes, failures = client.recursive_delete(node)
+                if failures:
+                    logging.error('WARN. deleted count: {}, failed count: '
+                                  '{}\n'.format(successes, failures))
+                    sys.exit(-1)
+                else:
+                    logging.info(
+                        'DONE. deleted count: {}\n'.format(successes))
             else:
-                raise Exception('{} is not a directory'.format(node))
+                if not node.endswith('/'):
+                    if client.get_node(node).islink():
+                        logging.info('deleting link {}'.format(node))
+                        client.delete(node)
+                    elif client.isfile(node):
+                        logging.info('deleting {}'.format(node))
+                        client.delete(node)
+                elif client.isdir(node):
+                    raise Exception('{} is a directory'.format(node))
+                else:
+                    raise Exception('{} is not a directory'.format(node))
 
     except Exception as ex:
         exit_on_exception(ex)

vos/vos/commands/vtag.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2022.                            (c) 2022.
+#  (c) 2024.                            (c) 2024.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -71,7 +71,9 @@
 
 The tag system is meant to allow tags, in addition to the standard node
 properties. """
+import logging
 import pprint
+import sys
 from ..commonparser import CommonParser, set_logging_level_from_args, \
     exit_on_exception, URI_DESCRIPTION
 from .. import vos
@@ -107,14 +109,15 @@ def vtag():
         nargs="*")
     parser.add_option('--remove', action="store_true",
                       help='remove the listed property')
+    parser.add_option('-R', '--recursive', action="store_true",
+                      help='perform the operation recursively on all the descendants')
 
-    opt = parser.parse_args()
-    args = opt
+    args = parser.parse_args()
     set_logging_level_from_args(args)
 
     # the node should be the first argument, the rest should contain
     # the key/val pairs
-    node = args.node
+    node_arg = args.node
 
     props = []
     if args.remove:
@@ -128,35 +131,53 @@ def vtag():
 
     try:
         client = vos.Client(
-            vospace_certfile=opt.certfile,
-            vospace_token=opt.token,
-            insecure=opt.insecure)
-        node = client.get_node(node)
+            vospace_certfile=args.certfile,
+            vospace_token=args.token,
+            insecure=args.insecure)
+        node = client.get_node(node_arg)
         if len(props) == 0:
             # print all properties
             pprint.pprint(node.props)
-
-        changed = False
-        for prop in props:
-            prop = prop.split('=')
-            if len(prop) == 1:
-                # get one property
-                pprint.pprint(node.props.get(prop[0], None))
-            elif len(prop) == 2:
-                # update properties
-                key, value = prop
+        if args.recursive:
+            node.props.clear()
+            node.clear_properties()
+            for prop in props:
+                key, value = prop.split('=')
                 if len(value) == 0:
                     value = None
-                if value != node.props.get(key, None):
-                    node.props[key] = value
-                    changed = True
-            else:
-                raise ValueError(
-                    "Illegal keyword of value character ('=') used: %s" % (
-                        '='.join(prop)))
-
-        if changed:
-            client.add_props(node)
+                node.props[key] = value
+            successes, failures = client.add_props(node, recursive=True)
+            if args.recursive:
+                if failures:
+                    logging.error(
+                        'WARN. updated count: {}, failed count: {}\n'.
+                        format(successes, failures))
+                    sys.exit(-1)
+                else:
+                    logging.info(
+                        'DONE. updated count: {}\n'.format(successes))
+        else:
+            changed = False
+            for prop in props:
+                prop = prop.split('=')
+                if len(prop) == 1:
+                    # get one property
+                    pprint.pprint(node.props.get(prop[0], None))
+                elif len(prop) == 2:
+                    # update properties
+                    key, value = prop
+                    if len(value) == 0:
+                        value = None
+                    if value != node.props.get(key, None):
+                        node.props[key] = value
+                        changed = True
+                else:
+                    raise ValueError(
+                        "Illegal keyword of value character ('=') used: %s" % (
+                            '='.join(prop)))
+
+            if changed:
+                client.add_props(node)
     except Exception as ex:
         exit_on_exception(ex)
 

vos/vos/tests/test_config.py

@@ -65,12 +65,12 @@
 #
 # ***********************************************************************
 #
-
+import unittest
 # Test the vosconfig functionality
 from unittest.mock import Mock, patch
 import warnings
 
-from vos import vosconfig, Connection
+from vos import vosconfig
 import pytest
 import tempfile
 import os
@@ -80,29 +80,30 @@
 skipTests = False
 
 
+# @unittest.skip('Functionality no longer used')
 @patch('vos.vos.os.path.exists', Mock())
 def test_update_config():
     # Cause all warnings to always be triggered.
     warnings.simplefilter("always")
-    with patch('vos.vos.open') as open_mock:
+    with patch('builtins.open') as open_mock:
         old_content = 'blah'
         new_config_mock = Mock()
         open_mock.return_value.read.return_value = old_content
         open_mock.return_value.write = new_config_mock
         vosconfig._update_config()
-    assert new_config_mock.called_once_with(old_content)
+    new_config_mock.assert_called_once_with(old_content)
 
     # test rewrite vospace resource in config file
     new_config_mock.reset_mock()
     # Cause all warnings to always be triggered.
     warnings.simplefilter("always")
-    with patch('vos.vos.open') as open_mock:
+    with patch('builtins.open') as open_mock:
         old_content = 'blah\nresourceID=ivo://cadc.nrc.ca/vospace\nfoo'
         new_content = Mock()
         open_mock.return_value.read.return_value = old_content
         open_mock.return_value.write = new_content
         vosconfig._update_config()
-    assert new_config_mock.called_once_with(old_content.replace(
+    new_content.assert_called_once_with(old_content.replace(
         'vospace', 'vault'))
 
     # test rewrite transfer protocol in config file
@@ -111,14 +112,14 @@ def test_update_config():
         "# transfer protocol configuration is no longer supported\n"
     # Cause all warnings to always be triggered.
     warnings.simplefilter("always")
-    with patch('vos.vos.open') as open_mock:
+    with patch('builtins.open') as open_mock:
         old_content = 'blah\nprotocol=http\nfoo'
         new_content = Mock()
         open_mock.return_value.read.return_value = old_content
         open_mock.return_value.write = new_content
         vosconfig._update_config()
-    assert new_config_mock.called_once_with(old_content.replace(
-        'protocol', '{}#protocol'.format(protocol_text)))
+    new_content.assert_called_once_with(old_content.replace(
+        'protocol', '{}# protocol'.format(protocol_text)))
 
 
 @patch('vos.vosconfig.os.path.isfile', Mock())