youcat: use DelegatingStorageManager and improve availability checks

cadc-tap-tmp/build.gradle

@@ -15,7 +15,7 @@ apply from: '../opencadc.gradle'
 sourceCompatibility = 1.8
 
 group = 'org.opencadc'
-version = '1.1.4'
+version = '1.1.5'
 
 description = 'OpenCADC TAP-1.1 temporary on-disk upload storage library'
 def git_url = 'https://github.com/opencadc/tap'

cadc-tap-tmp/src/main/java/org/opencadc/tap/tmp/DelegatingStorageManager.java

@@ -109,6 +109,10 @@ public DelegatingStorageManager() {
             throw new InvalidConfigException("unknown implementation " + IMPL_KEY + "=" + cname);
         }
     }
+
+    public void check() throws Exception {
+        impl.check();
+    }
 
     @Override
     public void setJob(Job job) {

cadc-tap-tmp/src/main/java/org/opencadc/tap/tmp/HttpStorageManager.java

@@ -70,6 +70,7 @@
 import ca.nrc.cadc.auth.AuthMethod;
 import ca.nrc.cadc.auth.RunnableAction;
 import ca.nrc.cadc.auth.SSLUtil;
+import ca.nrc.cadc.auth.X509CertificateChain;
 import ca.nrc.cadc.cred.client.CredUtil;
 import ca.nrc.cadc.dali.tables.TableWriter;
 import ca.nrc.cadc.io.ByteLimitExceededException;
@@ -108,8 +109,11 @@
 import java.net.URL;
 import java.nio.charset.Charset;
 import java.security.AccessControlException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
 import java.security.cert.CertificateExpiredException;
 import java.security.cert.CertificateNotYetValidException;
+import java.security.spec.InvalidKeySpecException;
 import java.sql.ResultSet;
 import javax.security.auth.Subject;
 import org.apache.log4j.Logger;
@@ -155,6 +159,14 @@ private void init(MultiValuedProperties props) {
         log.debug("cert file: " + absCertFile);
         this.certFile = new File(absCertFile);
     }
+
+    @Override
+    public void check() throws Exception {
+        // validate certificate
+        X509CertificateChain cert = SSLUtil.readPemCertificateAndKey(certFile);
+        cert.getChain()[0].checkValidity();
+        // TODO: check baseURL? 
+    }
 
 
     @Override

cadc-tap-tmp/src/main/java/org/opencadc/tap/tmp/StorageManager.java

@@ -77,4 +77,12 @@
  */
 public interface StorageManager extends ResultStore, UWSInlineContentHandler {
     static final String CONFIG = "cadc-tap-tmp.properties";
+
+    /**
+     * Perform any checks that the implementation should be able to function if
+     * called.
+     * 
+     * @throws Exception if the implementation will probably fail to function
+     */
+    public void check() throws Exception;
 }

cadc-tap-tmp/src/main/java/org/opencadc/tap/tmp/TempStorageManager.java

@@ -114,6 +114,19 @@ public TempStorageManager() {
         this.baseDir = new File(props.getFirstPropertyValue(TempStorageInitAction.BASE_DIR_KEY));
     }
 
+    @Override
+    public void check() throws Exception {
+        // check baseDir
+        if (baseDir.isDirectory() && baseDir.canRead() && baseDir.canWrite()) {
+            return;
+        }
+        throw new IOException("cannot use baseDir: " + baseDir 
+                + " -- reason: isDir=" + baseDir.isDirectory()
+                + " readable=" + baseDir.canRead()
+                + " writable=" + baseDir.canWrite());
+    }
+
+
     // used by TempStorageGetAction
     public File getStoredFile(String filename) {
         return getDestFile(filename);

youcat/README.md

@@ -71,7 +71,7 @@ See <a href="https://github.com/opencadc/reg/tree/master/cadc-registry">cadc-reg
 
 This service uses the `cadc-tap-tmp` library for temporary storage needed by core TAP features 
 (async results, tap_upload tables). It is currently hard-coded (PluginFactory.properties) to 
-use the HttpStorageManager and persist to an external URL (HTTP PUT).
+use the DelegatingStorageManager.
 
 This storage _is not_ used for content upload of user-created tables: that content is streamed 
 directly into the database server.

youcat/VERSION

@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor
 # build version tag: timestamp
-VER=0.5.3
+VER=0.5.4
 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
 unset VER

youcat/build.gradle

@@ -41,8 +41,8 @@ dependencies {
     compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)'
     compile 'org.opencadc:cadc-registry:[1.7.2,)'
     compile 'org.opencadc:cadc-gms:[1.0.4,2.0)'
+    compile 'org.opencadc:cadc-tap-tmp:[1.1.5,)'
 
-    runtime 'org.opencadc:cadc-tap-tmp:[1.1,)'
     runtime 'org.opencadc:cadc-vos:[1.1,2.0)'
     runtime 'org.opencadc:cadc-access-control-identity:[1.1.0,)'
 

youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java

@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2019.                            (c) 2019.
+*  (c) 2023.                            (c) 2023.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -74,18 +74,27 @@
 import ca.nrc.cadc.reg.client.RegistryClient;
 import ca.nrc.cadc.rest.RestAction;
 import ca.nrc.cadc.tap.schema.InitDatabaseTS;
+import ca.nrc.cadc.util.InvalidConfigException;
 import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS;
 import ca.nrc.cadc.vosi.Availability;
 import ca.nrc.cadc.vosi.AvailabilityPlugin;
+import ca.nrc.cadc.vosi.avail.CheckCertificate;
 import ca.nrc.cadc.vosi.avail.CheckDataSource;
 import ca.nrc.cadc.vosi.avail.CheckException;
 import ca.nrc.cadc.vosi.avail.CheckResource;
 import ca.nrc.cadc.vosi.avail.CheckWebService;
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.spec.InvalidKeySpecException;
+import java.util.NoSuchElementException;
 import javax.sql.DataSource;
 import org.apache.log4j.Logger;
+import org.opencadc.tap.tmp.DelegatingStorageManager;
+import org.opencadc.tap.tmp.StorageManager;
 
 /**
  *
@@ -98,7 +107,7 @@ public class CatalogTapService implements AvailabilityPlugin {
     private static final String TAPUSER_TEST = "select schema_name from tap_schema.schemas11 where schema_name='tap_schema'";
     private static final String UWS_TEST = "select jobID from uws.Job limit 1";
 
-    private static File SERVOPS_CERT = new File(System.getProperty("user.home") + "/.ssl/cadcproxy.pem");
+    private static final File AAI_PEM_FILE = new File(System.getProperty("user.home") + "/.ssl/cadcproxy.pem");
     private static File TMPOPS_CERT = new File(System.getProperty("user.home") + "/.ssl/tmpops.pem");
 
     private static final URI TMP_STORAGE_WS = URI.create("ivo://cadc.nrc.ca/cadc/minoc");
@@ -147,38 +156,70 @@ public Availability getStatus() {
             cr.check();
             // TODO: check that DS_TAPUSER can create/drop TAP_UPLOAD tables
 
-            // create/drop in all user schemas
-            //File cert = TODO;
-            //CheckCertificate checkCert = new CheckCertificate(cert);
-            //checkCert.check();
-
             // check other services we depend on
             RegistryClient reg = new RegistryClient();
-            URL url;
-            CheckResource checkResource;
-
             LocalAuthority localAuthority = new LocalAuthority();
 
-            URI credURI = localAuthority.getServiceURI(Standards.CRED_PROXY_10.toString());
-            url = reg.getServiceURL(credURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
-            checkResource = new CheckWebService(url);
-            checkResource.check();
-
-            URI usersURI = localAuthority.getServiceURI(Standards.UMS_USERS_01.toString());
-            url = reg.getServiceURL(usersURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
-            checkResource = new CheckWebService(url);
-            checkResource.check();
-
-            URI groupsURI = localAuthority.getServiceURI(Standards.GMS_SEARCH_10.toString());
-            if (!groupsURI.equals(usersURI)) {
-                url = reg.getServiceURL(groupsURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
-                checkResource = new CheckWebService(url);
-                checkResource.check();
+            URI credURI = null;
+            try {
+                credURI = localAuthority.getServiceURI(Standards.CRED_PROXY_10.toString());
+                URL url = reg.getServiceURL(credURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
+                if (url != null) {
+                    CheckResource checkResource = new CheckWebService(url);
+                    checkResource.check();
+                } else {
+                    log.debug("check skipped: " + credURI + " does not provide " + Standards.VOSI_AVAILABILITY);
+                }
+            } catch (NoSuchElementException ex) {
+                log.debug("not configured: " + Standards.CRED_PROXY_10);
+            }
+
+            URI usersURI = null;
+            try {
+                usersURI = localAuthority.getServiceURI(Standards.UMS_USERS_01.toString());
+                URL url = reg.getServiceURL(credURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
+                if (url != null) {
+                    CheckResource checkResource = new CheckWebService(url);
+                    checkResource.check();
+                } else {
+                    log.debug("check skipped: " + usersURI + " does not provide " + Standards.VOSI_AVAILABILITY);
+                }
+            } catch (NoSuchElementException ex) {
+                log.debug("not configured: " + Standards.UMS_USERS_01);
+            }
+
+            URI groupsURI = null;
+            try {
+                groupsURI = localAuthority.getServiceURI(Standards.GMS_SEARCH_10.toString());
+                if (!groupsURI.equals(usersURI)) {
+                    URL url = reg.getServiceURL(groupsURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
+                    if (url != null) {
+                        CheckResource checkResource = new CheckWebService(url);
+                        checkResource.check();
+                    } else {
+                        log.debug("check skipped: " + groupsURI + " does not provide " + Standards.VOSI_AVAILABILITY);
+                    }
+                }
+            } catch (NoSuchElementException ex) {
+                log.debug("not configured: " + Standards.GMS_SEARCH_10);
+            }
+
+            if (credURI != null || usersURI != null) {
+                if (AAI_PEM_FILE.exists() && AAI_PEM_FILE.canRead()) {
+                    // check for a certificate needed to perform network A&A ops
+                    CheckCertificate checkCert = new CheckCertificate(AAI_PEM_FILE);
+                    checkCert.check();
+                } else {
+                    log.debug("AAI cert not found or unreadable");
+                }
             }
 
-            url = reg.getServiceURL(TMP_STORAGE_WS, Standards.VOSI_AVAILABILITY, AuthMethod.ANON);
-            checkResource = new CheckWebService(url);
-            checkResource.check();
+            try {
+                StorageManager store = new DelegatingStorageManager();
+                store.check();
+            } catch (Exception ex) {
+                throw new CheckException("cadc-tap-tmp check: " + ex, ex);
+            }
 
         } catch (CheckException ce) {
             // tests determined that the resource is not working

youcat/src/main/resources/PluginFactory.properties

@@ -18,7 +18,7 @@ ca.nrc.cadc.tap.UploadManager = org.opencadc.youcat.tap.UploadManagerImpl
 
 ca.nrc.cadc.tap.writer.format.FormatFactory = org.opencadc.youcat.tap.FormatFactoryImpl
 
-ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.HttpStorageManager
+ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.DelegatingStorageManager
 
 ca.nrc.cadc.tap.schema.TapSchemaDAO = org.opencadc.youcat.tap.TapSchemaDAOImpl