Datetime date from CLDR data

schema/check_generated_data.py

@@ -5,6 +5,7 @@
 import glob
 import json
 
+from jsonschema import Draft7Validator, ValidationError
 
 import logging
 import logging.config

schema/schema_validator.py

@@ -5,8 +5,8 @@
 
 import jsonschema.exceptions
 from jsonschema import validate
-from jsonschema import validate
 from jsonschema import exceptions
+from jsonschema import ValidationError
 
 import logging
 import logging.config

testdriver/testplan.py

@@ -272,11 +272,9 @@ def run_one_test_mode(self):
         try:
             result_dir = os.path.dirname(self.outputFilePath)
             if not os.path.isdir(result_dir):
-                os.makedirs(result_dir)
+                os.makedirs(result_dir, exist_ok=True)
         except BaseException as error:
-            sys.stderr.write('!!!%s:  Cannot create directory %sfor report file %s' %
-                             (error, result_dir, self.outputFilePath))
-            return None
+            logging.error('testplay.py: %s for %s / %s', error, result_dir, self.outputFilePath)
 
         # Create results file
         try:

testgen/generators/datetime_fmt.py

@@ -1,8 +1,12 @@
 # -*- coding: utf-8 -*-
+from datetime import datetime, timezone
+# import pytz
+
 import os
 import json
 import re
 import logging
+import math
 import subprocess
 from generators.base import DataGenerator
 
@@ -12,6 +16,87 @@ class DateTimeFmtGenerator(DataGenerator):
     json_test = {"test_type": "datetime_fmt"}
     json_verify = {"test_type": "datetime_fmt"}
 
+    def generate_datetime_data_from_cldr(self, dt_json_path, run_limit=-1):
+        # Get CLDR-derived date time json file and parse it
+        # Optionally sample from it if run_limit > 1
+        with open(dt_json_path, 'r', encoding="UTF-8") as dt_json_file:
+            try:
+                json_data = json.load(dt_json_file)
+            except json.JSONDecodeError as err:
+                return None
+
+            test_cases = []
+            verify_cases = []
+
+            test_obj = {
+                'Test scenario': 'datetime_fmt',
+                'test_type': 'datetime_fmt',
+                'description': 'date/time format test data generated by Node',
+                'icuVersion': self.icu_version,
+                'cldrVersion': '??'
+            }
+
+            test_cases = []
+            verify_cases = []
+            verify_obj = {
+                'test_type': 'datetime_fmt',
+                'description': 'date/time format test data generated by Node',
+                'icuVersion': self.icu_version,
+                'cldrVersion': '??'
+            }
+            # Get each entry and assemble the test data and verify data.
+            label_num = 0
+            desired_width = math.ceil(math.log10(len(json_data)))  # Based the size of json_data
+            for test_item in json_data:
+                label_str = str(label_num).rjust(desired_width, "0")
+                # Construct options
+                options = {}
+                # TODO: Generate input string with "Z" and compute tz_offset_secs
+                # TODO: Generate hash of the data without the label
+                raw_input = test_item['input']
+                start_index = raw_input.find('[')
+                tz_str = raw_input[start_index+1:-1]
+                #3z_pattern = re.compile(r'(\[\s\\]*\])')
+                #tz_name = tz_pattern.search(raw_input)
+                raw_time = datetime.fromisoformat(raw_input[0:start_index])
+
+                dt = datetime.fromisoformat(test_item['input'])
+                if dt.tzinfo is None:
+                    tzinfo = tt.tzinfo
+                    dt = dt.replace(tzinfo=timezone.utc)
+                new_test = {
+                    "locale": test_item['locale'],
+                    "input_string": test_item['input'],
+                            "options": options
+                }
+                # TODO: Generate hash of the data without the label
+                new_test['tz_offset_secs'] = 0  # COMPUTE THIS FOR THE TIMEZONE
+                new_test['label'] = label_str
+
+                new_verify = {"label": label_str,
+                              "verify": test_item['expected']
+                }
+                test_cases.append(new_test)
+                verify_cases.append(new_verify)
+                label_num += 1
+
+            # Save output as: datetime_fmt_test.json and datetime_fmt_verify.json
+            test_obj['tests'] = test_cases
+            verify_obj['verifications'] = verify_cases
+            # Create the hex hash values
+            self.generateTestHashValues(test_obj)
+
+            base_path = ''
+            dt_test_path = os.path.join(base_path, 'datetime_fmt_test.json')
+            dt_verify_path = os.path.join(base_path, 'datetime_fmt_verify.json')
+
+            try:
+                self.saveJsonFile(dt_test_path, test_obj, indent=2)
+                self.saveJsonFile(dt_verify_path, verify_obj, indent=2)
+            except BaseException as err:
+                logging.error('!!! %s: Failure to save file %s', err, )
+                return None
+
     def process_test_data(self):
         # Use NOde JS to create the .json files
         icu_nvm_versions = {
@@ -22,11 +107,18 @@ def process_test_data(self):
             'icu71': '18.7.0',
         }
 
+        # Update to check for datetime.json which has been generated from CLDR data.
+        dt_json_path = os.path.join('.', self.icu_version, 'datetime.json')
+        if os.path.exists(dt_json_path):
+            result = self.generate_datetime_data_from_cldr(dt_json_path, self.run_limit)
+            return result
+
+        # OK, there's no CLDR-based JSON data available.
         run_list = [
-            ['source ~/.nvm/nvm.sh; nvm install 21.6.0; nvm use 21.6.0 --silent'],
-            ['node generators/datetime_gen.js'],
-            ['mv datetime_fmt*.json icu74']
-        ]
+                ['source ~/.nvm/nvm.sh; nvm install 21.6.0; nvm use 21.6.0 --silent'],
+                ['node generators/datetime_gen.js'],
+                ['mv datetime_fmt*.json icu74']
+            ]
 
         if self.icu_version not in icu_nvm_versions:
             logging.warning('Generating datetime data not configured for icu version %s', self.icu_version)
@@ -39,7 +131,7 @@ def process_test_data(self):
             nvm_version, nvm_version, '-run_limit', self.run_limit)
 
         logging.debug('Running this command: %s', generate_command)
-        result = result = subprocess.run(generate_command, shell=True)
+        result = subprocess.run(generate_command, shell=True)
 
         # Move results to the right directory
         mv_command = 'mv datetime_fmt*.json %s' % self.icu_version