Merge branch 'develop' into 3621-update-metadata-namespace

doc/release-notes/6656-file-uploads.md

@@ -0,0 +1 @@
+new JVM option: dataverse.files.uploads

doc/sphinx-guides/source/installation/config.rst

@@ -699,6 +699,26 @@ Once you have configured a trusted remote store, you can point your users to the
 
     ===========================================  ==================  ==========================================================================  ===================
 
+.. _temporary-file-storage:
+
+Temporary Upload File Storage
++++++++++++++++++++++++++++++
+
+When uploading files via the API or Web UI, you need to be aware that multiple steps are involved to enable
+features like ingest processing, transfer to a permanent storage, checking for duplicates, unzipping etc.
+
+All of these processes are triggered after finishing transfers over the wire and moving the data into a temporary
+(configurable) location on disk at :ref:`${dataverse.files.directory} <dataverse.files.directory>`\ ``/temp``.
+
+Before being moved there,
+
+- JSF Web UI uploads are stored at :ref:`${dataverse.files.uploads} <dataverse.files.uploads>`, defaulting to
+  ``/usr/local/payara5/glassfish/domains/domain1/uploads`` folder in a standard installation. This place is
+  configurable and might be set to a separate disk volume where stale uploads are purged periodically.
+- API uploads are stored at the system's temporary files location indicated by the Java system property
+  ``java.io.tmpdir``, defaulting to ``/tmp`` on Linux. If this location is backed by a `tmpfs <https://www.kernel.org/doc/html/latest/filesystems/tmpfs.html>`_
+  on your machine, large file uploads via API will cause RAM and/or swap usage bursts. You might want to point this to
+  a different location, restrict maximum size of it, and monitor for stale uploads.
 
 
 .. _Branding Your Installation:
@@ -1043,6 +1063,14 @@ On a new Dataverse installation, users may select from the following licenses or
 
 (Note that existing Dataverse installations which are upgraded from 5.9 or previous will only offer CC0 1.0, added automatically during the upgrade to version 5.10.)
 
+If the Dataverse Installation supports multiple languages, the license name/description translations should be added to the ``License`` properties files. (See :ref:`i18n` for more on properties files and internationalization in general.)
+To create the key, the license name has to be converted to lowercase, replace space with underscore.
+
+Example::
+
+  license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Dedication.
+  license.cc0_1.0.name=CC0 1.0
+
 You have a lot of control over which licenses and terms are available. You can remove licenses and add new ones. You can decide which license is the default. You can remove "Custom Dataset Terms" as a option. You can remove all licenses and make "Custom Dataset Terms" the only option.
 
 Before making changes, you are encouraged to read the :ref:`license-terms` section of the User Guide about why CC0 is the default and what the "Custom Dataset Terms" option allows.
@@ -1434,11 +1462,27 @@ Note that it's also possible to use the ``dataverse.fqdn`` as a variable, if you
 
 We are absolutely aware that it's confusing to have both ``dataverse.fqdn`` and ``dataverse.siteUrl``. https://github.com/IQSS/dataverse/issues/6636 is about resolving this confusion.
 
+.. _dataverse.files.directory:
+
 dataverse.files.directory
 +++++++++++++++++++++++++
 
 This is how you configure the path Dataverse uses for temporary files. (File store specific dataverse.files.\<id\>.directory options set the permanent data storage locations.)
 
+.. _dataverse.files.uploads:
+
+dataverse.files.uploads
++++++++++++++++++++++++
+
+Configure a folder to store the incoming file stream during uploads (before transfering to `${dataverse.files.directory}/temp`).
+Please also see :ref:`temporary-file-storage` for more details.
+You can use an absolute path or a relative, which is relative to the application server domain directory.
+
+Defaults to ``./uploads``, which resolves to ``/usr/local/payara5/glassfish/domains/domain1/uploads`` in a default
+installation.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``.
+
 dataverse.auth.password-reset-timeout-in-minutes
 ++++++++++++++++++++++++++++++++++++++++++++++++
 

src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java

@@ -25,12 +25,8 @@
 import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 import java.util.logging.Logger;
-import java.util.Base64;
-import java.util.HashMap;
-import java.util.Map;
 import javax.imageio.ImageIO;
 import org.apache.commons.io.IOUtils;
 import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
@@ -43,6 +39,7 @@
 import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.EnumUtils;
 
 public class DatasetUtil {
 
@@ -459,7 +456,7 @@ public static List<DatasetField> getDatasetSummaryFields(DatasetVersion datasetV
     }
 
     public static boolean isRsyncAppropriateStorageDriver(Dataset dataset){
-        // ToDo - rsync was written before multiple store support and currently is hardcoded to use the DataAccess.S3 store. 
+        // ToDo - rsync was written before multiple store support and currently is hardcoded to use the DataAccess.S3 store.
         // When those restrictions are lifted/rsync can be configured per store, this test should check that setting
         // instead of testing for the 's3" store,
         //This method is used by both the dataset and edit files page so one change here
@@ -551,7 +548,7 @@ public static License getLicense(DatasetVersion dsv) {
 
     public static String getLicenseName(DatasetVersion dsv) {
         License license = DatasetUtil.getLicense(dsv);
-        return license != null ? license.getName()
+        return license != null ? getLocalizedLicenseDetails(license,"NAME")
                 : BundleUtil.getStringFromBundle("license.custom");
     }
 
@@ -577,7 +574,30 @@ public static String getLicenseIcon(DatasetVersion dsv) {
 
     public static String getLicenseDescription(DatasetVersion dsv) {
         License license = DatasetUtil.getLicense(dsv);
-        return license != null ? license.getShortDescription() : BundleUtil.getStringFromBundle("license.custom.description");
+        return license != null ? getLocalizedLicenseDetails(license,"DESCRIPTION") : BundleUtil.getStringFromBundle("license.custom.description");
+    }
+
+    public enum LicenseOption {
+        NAME, DESCRIPTION
+    };
+
+    public static String getLocalizedLicenseDetails(License license,String keyPart) {
+        String licenseName = license.getName();
+        String localizedLicenseValue =  "" ;
+        try {
+            if (EnumUtils.isValidEnum(LicenseOption.class, keyPart ) ){
+                String key = "license." + licenseName.toLowerCase().replace(" ", "_") + "." + keyPart.toLowerCase();
+                localizedLicenseValue = BundleUtil.getStringFromPropertyFile(key, "License");
+            }
+        }
+        catch (Exception e) {
+            localizedLicenseValue = licenseName;
+        }
+
+        if (localizedLicenseValue == null) {
+            localizedLicenseValue = licenseName ;
+        }
+        return localizedLicenseValue;
     }
 
     public static String getLocaleExternalStatus(String status) {

src/main/java/propertyFiles/License.properties

@@ -0,0 +1,4 @@
+license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Dedication. 
+license.cc_by_4.0.description=Creative Commons Attribution 4.0 International License.
+license.cc0_1.0.name=CC0 1.0
+license.cc_by_4.0.name=CC-BY 4.0

src/main/webapp/WEB-INF/glassfish-web.xml

@@ -8,9 +8,15 @@
       <description>Keep a copy of the generated servlet class' java code.</description>
     </property>
   </jsp-config>
+  <parameter-encoding default-charset="UTF-8"/>
+  <!-- Find a list of properties here: https://docs.oracle.com/cd/E19798-01/821-1750/beayb/index.html -->
   <property name="alternatedocroot_1" value="from=/guides/* dir=./docroot"/>
   <property name="alternatedocroot_2" value="from=/dataexplore/* dir=./docroot"/>
   <property name="alternatedocroot_logos" value="from=/logos/* dir=./docroot"/>
   <property name="alternatedocroot_sitemap" value="from=/sitemap/* dir=./docroot"/>
-  <parameter-encoding default-charset="UTF-8"/>
+  <!--
+    This folder is not only holding compiled JSP pages but also the place where file streams are stored
+    during uploads. As Dataverse does not use any JSP, there will only be uploads stored here.
+  -->
+  <property name="tempdir" value="${MPCONFIG=dataverse.files.uploads:./uploads}"/>
 </glassfish-web-app>

src/main/webapp/dataset-license-terms.xhtml

@@ -46,7 +46,7 @@
                                    <p class="help-block"><h:outputText value="#{bundle['file.dataFilesTab.terms.list.license.edit.description']}" escape="false"/></p>
                                    <p:selectOneMenu id="licenses" value="#{termsOfUseAndAccess.license}" style="width: 40%" converter="licenseConverter">
                                        <f:selectItems value="#{licenseServiceBean.listAllActive()}"
-                                                   var="license" itemLabel="#{license.name}" itemValue="#{license}"/>
+                                                   var="license" itemLabel="#{DatasetUtil:getLocalizedLicenseDetails(license, 'NAME')}" itemValue="#{license}"/>
                                        <c:if test="#{settingsWrapper.customLicenseAllowed}">
                                            <f:selectItem itemLabel="#{bundle['license.custom']}" itemValue="#{null}"/>
                                        </c:if>
@@ -55,8 +55,8 @@
                                </ui:fragment>
                                <ui:fragment rendered="#{!empty termsOfUseAndAccess.license}">
                                    <p>
-                                       <img src="#{termsOfUseAndAccess.license.iconUrl}" title="#{termsOfUseAndAccess.license.shortDescription}"  style="display:none"  onload="this.style.display='inline'" />
-                                       <a href="#{termsOfUseAndAccess.license.uri}" title="#{termsOfUseAndAccess.license.shortDescription}" target="_blank">#{termsOfUseAndAccess.license.name}</a>
+                                       <img src="#{termsOfUseAndAccess.license.iconUrl}" title="#{DatasetUtil:getLocalizedLicenseDetails(termsOfUseAndAccess.license, 'DESCRIPTION')}"  style="display:none"  onload="this.style.display='inline'" />
+                                       <a href="#{termsOfUseAndAccess.license.uri}" title="#{DatasetUtil:getLocalizedLicenseDetails(termsOfUseAndAccess.license,'DESCRIPTION')}" target="_blank">#{DatasetUtil:getLocalizedLicenseDetails(termsOfUseAndAccess.license,'NAME')}</a>
                                    </p>
                                </ui:fragment>
                            </div>

src/main/webapp/datasetLicenseInfoFragment.xhtml

@@ -30,12 +30,12 @@ xmlns:jsf="http://xmlns.jcp.org/jsf">
         </div>
         <ui:fragment rendered="#{!empty DatasetPage.workingVersion.termsOfUseAndAccess.license}">
             <div class="form-group"
-                jsf:rendered="#{!empty DatasetPage.workingVersion.termsOfUseAndAccess.license.shortDescription}">
+                jsf:rendered="#{!empty DatasetUtil:getLocalizedLicenseDetails(DatasetPage.workingVersion.termsOfUseAndAccess.license,'DESCRIPTION')} }">
                 <label for="metadata_Terms" class="col-sm-3 control-label">
                     #{bundle['dataset.publish.terms.description']}
                 </label>
                 <div class="col-sm-9">
-                    <h:outputText value="#{DatasetPage.workingVersion.termsOfUseAndAccess.license.shortDescription}" />
+                    <h:outputText value="#{DatasetUtil:getLocalizedLicenseDetails(DatasetPage.workingVersion.termsOfUseAndAccess.license,'DESCRIPTION')}" />
                 </div>
             </div>
         </ui:fragment>
@@ -121,4 +121,4 @@ xmlns:jsf="http://xmlns.jcp.org/jsf">
         </ui:fragment>
     </div>
     <!-- END license info -->
-</ui:composition>
+</ui:composition>