diff --git a/CMakeLists.txt b/CMakeLists.txt index 2125b93ab4..0d5524db55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,11 +140,15 @@ ENDIF() # Check for libtiff ################################################################################ -option(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED "Disable TIFF support (strongly discouraged !)" OFF) -mark_as_advanced(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) -if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) +option(DISABLE_TIFF "Disable TIFF support" OFF) +mark_as_advanced(DISABLE_TIFF) +if(DISABLE_TIFF) + message(WARNING "TIFF support has been disabled and will result in the inability to read some grids") +else() find_package(TIFF REQUIRED) - if(NOT TIFF_FOUND) + if(TIFF_FOUND) + boost_report_value(TIFF_FOUND) + else() message(SEND_ERROR "libtiff dependency not found!") endif() add_definitions(-DTIFF_ENABLED) diff --git a/cmake/ProjTest.cmake b/cmake/ProjTest.cmake index 6abfbcd721..a016cd9a05 100644 --- a/cmake/ProjTest.cmake +++ b/cmake/ProjTest.cmake @@ -28,10 +28,10 @@ function(proj_add_test_script_sh SH_NAME BIN_USE) ) if(MSVC) set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") else() set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") endif() endif() @@ -51,10 +51,10 @@ function(proj_add_gie_test TESTNAME TESTCASE) if(MSVC) set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") else() set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") endif() diff --git a/docs/source/operations/transformations/deformation.rst b/docs/source/operations/transformations/deformation.rst index b45c5e2bc2..aefda3a6d8 100644 --- a/docs/source/operations/transformations/deformation.rst +++ b/docs/source/operations/transformations/deformation.rst @@ -92,7 +92,7 @@ Parameters .. option:: +xy_grids= - Comma-separated list of grids to load. If a grid is prefixed by an `@` the + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the grid is considered optional and PROJ will the not complain if the grid is not available. diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst index c49b7592f1..3053703285 100644 --- a/docs/source/operations/transformations/xyzgridshift.rst +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -21,16 +21,16 @@ translation values from a grid. The grid is referenced against either the 2D geographic CRS corresponding to the input (or sometimes output) CRS. This method is described (in French) in :cite:`NTF_88` -and as EPSG operation methode code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 +and as EPSG operation method code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 France geocentric interpolation). The translation in the grids are added to the input coordinates in the forward direction, and subtracted in the reverse direction. -By default (if grid_ref=input_crs), in the forward direction, the input coordinates +By default (if ``grid_ref=input_crs``), in the forward direction, the input coordinates are converted to their geographic equivalent to directly read and interpolate from the grid. In the reverse direction, an iterative method is used to be able to find the grid locations to read. -If grid_ref=output_crs is used, then the reverse strategy is applied: iterative +If ``grid_ref=output_crs`` is used, then the reverse strategy is applied: iterative method in the forward direction, and direct read in the reverse direction. Example @@ -62,7 +62,7 @@ Required .. option:: +grids= - Comma-separated list of grids to load. If a grid is prefixed by an `@` the + Comma-separated list of grids to load. If a grid is prefixed by an ``@`` the grid is considered optional and PROJ will the not complain if the grid is not available. diff --git a/docs/source/references.bib b/docs/source/references.bib index 8e527ea133..912929c3ed 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -231,7 +231,7 @@ @InProceedings{LambersKolb2012 } @TechReport{NTF_88, - Title = {Grille de parametres de transformation de coordonnees - GR3DF97A - Notice d'utilisation}, + Title = {Grille de parametres de transformation de coordonnees - {GR3DF97A} - Notice d'utilisation}, Author = {IGN}, Institution = {Service de Geodesie et Nivellement, Institut Geographique National}, Year = {1997}, diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index ea02fd4bea..3684a948f9 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -23,21 +23,34 @@ The following paths are checked in order: - For transformation grids that have an explict relative or absolute path, the directory specified in the grid filename. + - Path resolved by the callback function set with the :c:func:`proj_context_set_file_finder`. If it is set, the next tests will not be run. + - Path(s) set with the :c:func:`proj_context_set_search_paths`. If set, the next tests will not be run. + +- The PROJ user writable directory, which is : + + * on Windows, ${LOCALAPPDATA}/proj + * on MacOSX, ${HOME}/Library/Logs/proj + * on other platforms (Linux), ${XDG_DATA_HOME}/proj if :envvar:`XDG_DATA_HOME` + is defined. Else ${HOME}/.local/share/proj + - Path(s) set with by the environment variable :envvar:`PROJ_LIB`. On Linux/MacOSX/Unix, use ``:`` to separate paths. On Windows, ``;`` + - On Windows, the *..\\share\\proj\\* and its contents are found automatically at run-time if the installation respects the build structure. That is, the binaries and proj.dll are installed under *..\\bin\\*, and resource files are in *..\\share\\proj\\*. + - A path built into PROJ as its resource installation directory (whose value is $(pkgdatadir)), for builds using the Makefile build system. Note, however, that since this is a hard-wired path setting, it only works if the whole PROJ installation is not moved somewhere else. + - The current directory When networking capabilities are enabled, either by API with the @@ -77,6 +90,12 @@ Its default content is: ; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. cdn_endpoint = https://cdn.proj.org + cache_enabled = on + + cache_size_MB = 100 + + cache_ttl_sec = 86400 + Transformation grids ------------------------------------------------------------------------------- diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 24ae4a4526..31a74e9ae1 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -71,6 +71,6 @@ done by setting the variable with no content:: .. versionadded:: 7.0.0 Define the endpoint of the CDN storage. Normally defined through the proj.ini - configuration file locale in PROJ_LIB. + configuration file locale in :envvar:`PROJ_LIB`. Alternatively, the :c:func:`proj_context_set_url_endpoint` function can be used. diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index 90a1d3cf87..d10f3bc197 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -708,6 +708,7 @@ pj_cleanup_lock pj_clear_initcache pj_compare_datums pj_context_get_grid_cache_filename(projCtx_t*) +pj_context_get_user_writable_directory(projCtx_t*, bool) pj_context_is_network_enabled(projCtx_t*) pj_ctx_alloc pj_ctx_fclose @@ -930,6 +931,7 @@ proj_cs_get_axis_info proj_cs_get_type proj_destroy proj_dmstor +proj_download_file proj_ellipsoid_get_parameters proj_errno proj_errno_reset @@ -969,6 +971,7 @@ proj_int_list_destroy proj_is_crs proj_is_deprecated proj_is_derived_crs +proj_is_download_needed proj_is_equivalent_to proj_list_angular_units proj_list_destroy diff --git a/src/4D_api.cpp b/src/4D_api.cpp index cee8262ee2..9107723d53 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -48,6 +48,7 @@ #include #include "geodesic.h" #include "grids.hpp" +#include "filemanager.hpp" #include "proj/common.hpp" #include "proj/coordinateoperation.hpp" @@ -1450,6 +1451,16 @@ PJ_INFO proj_info (void) { /* build search path string */ auto ctx = pj_get_default_ctx(); if (!ctx || ctx->search_paths.empty()) { + // Env var mostly for testing purposes and being independent from + // an existing installation + const char* ignoreUserWritableDirectory = + getenv("PROJ_IGNORE_USER_WRITABLE_DIRECTORY"); + if( ignoreUserWritableDirectory == nullptr || + ignoreUserWritableDirectory[0] == '\0' ) { + buf = path_append(buf, + pj_context_get_user_writable_directory(ctx, false).c_str(), + &buf_size); + } const char *envPROJ_LIB = getenv("PROJ_LIB"); buf = path_append(buf, envPROJ_LIB, &buf_size); #ifdef PROJ_LIB diff --git a/src/Makefile.am b/src/Makefile.am index d29eb97661..3966750943 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -196,9 +196,9 @@ libproj_la_SOURCES = \ release.cpp gauss.cpp \ fileapi.cpp \ \ - apply_gridshift.cpp datums.cpp datum_set.cpp transform.cpp \ + datums.cpp datum_set.cpp transform.cpp \ geocent.cpp geocent.h utils.cpp \ - jniproj.cpp mutex.cpp initcache.cpp apply_vgridshift.cpp geodesic.c \ + jniproj.cpp mutex.cpp initcache.cpp geodesic.c \ strtod.cpp \ \ 4D_api.cpp pipeline.cpp \ diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp deleted file mode 100644 index 4ef86fc0fc..0000000000 --- a/src/apply_gridshift.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply datum shifts based on grid shift files (normally NAD27 to - * NAD83 or the reverse). This module is responsible for keeping - * a list of loaded grids, and calling with each one that is - * allowed for a given datum (expressed as the nadgrids= parameter). - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2000, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef FROM_PROJ_CPP -#define FROM_PROJ_CPP -#endif - -#include -#include -#include - -#include - -#include "proj.h" -#include "proj_internal.h" -#include "proj/internal/internal.hpp" -#include "grids.hpp" - -NS_PROJ_START - -// --------------------------------------------------------------------------- - -static const HorizontalShiftGrid* findGrid(const ListOfHGrids& grids, - PJ_LP input) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - return grid; - } - return nullptr; -} - -// --------------------------------------------------------------------------- - -static ListOfHGrids getListOfGridSets(PJ_CONTEXT* ctx, const char* grids) -{ - ListOfHGrids list; - auto listOfGrids = internal::split(std::string(grids), ','); - for( const auto& grid: listOfGrids ) - { - const char* gridname = grid.c_str(); - bool canFail = false; - if( gridname[0] == '@' ) - { - canFail = true; - gridname ++; - } - auto gridSet = HorizontalShiftGridSet::open(ctx, gridname); - if( !gridSet ) - { - if( !canFail ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return {}; - } - } - else - { - list.emplace_back(std::move(gridSet)); - } - } - return list; -} - - -/**********************************************/ -ListOfHGrids proj_hgrid_init(PJ* P, const char *gridkey) { -/********************************************** - - Initizalize and populate list of horizontal - grids. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - std::string key("s"); - key += gridkey; - const char* grids = pj_param(P->ctx, P->params, key.c_str()).s; - if( grids == nullptr ) - return {}; - - return getListOfGridSets(P->ctx, grids); -} - -typedef struct { pj_int32 lam, phi; } ILP; - -static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid, bool compensateNTConvention) { - PJ_LP val, frct; - ILP indx; - int in; - - const auto& extent = grid->extentAndRes(); - t.lam /= extent.resLon; - indx.lam = isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); - t.phi /= extent.resLat; - indx.phi = isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); - - frct.lam = t.lam - indx.lam; - frct.phi = t.phi - indx.phi; - val.lam = val.phi = HUGE_VAL; - if (indx.lam < 0) { - if (indx.lam == -1 && frct.lam > 0.99999999999) { - ++indx.lam; - frct.lam = 0.; - } else - return val; - } else if ((in = indx.lam + 1) >= grid->width()) { - if (in == grid->width() && frct.lam < 1e-11) { - --indx.lam; - frct.lam = 1.; - } else - return val; - } - if (indx.phi < 0) { - if (indx.phi == -1 && frct.phi > 0.99999999999) { - ++indx.phi; - frct.phi = 0.; - } else - return val; - } else if ((in = indx.phi + 1) >= grid->height()) { - if (in == grid->height() && frct.phi < 1e-11) { - --indx.phi; - frct.phi = 1.; - } else - return val; - } - - float f00Lon = 0, f00Lat = 0; - float f10Lon = 0, f10Lat = 0; - float f01Lon = 0, f01Lat = 0; - float f11Lon = 0, f11Lat = 0; - if( !grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, f00Lat) || - !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, f10Lat) || - !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, f01Lat) || - !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, f11Lon, f11Lat) ) - { - return val; - } - - double m10 = frct.lam; - double m11 = m10; - double m01 = 1. - frct.lam; - double m00 = m01; - m11 *= frct.phi; - m01 *= frct.phi; - frct.phi = 1. - frct.phi; - m00 *= frct.phi; - m10 *= frct.phi; - val.lam = m00 * f00Lon + m10 * f10Lon + - m01 * f01Lon + m11 * f11Lon; - val.phi = m00 * f00Lat + m10 * f10Lat + - m01 * f01Lat + m11 * f11Lat; - return val; -} - - -#define MAX_ITERATIONS 10 -#define TOL 1e-12 - -static -PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* grid, - const ListOfHGrids& grids) { - PJ_LP t, tb,del, dif; - int i = MAX_ITERATIONS; - const double toltol = TOL*TOL; - - if (in.lam == HUGE_VAL) - return in; - - /* normalize input to ll origin */ - tb = in; - const auto* extent = &(grid->extentAndRes()); - tb.lam -= extent->westLon; - tb.phi -= extent->southLat; - - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - - t = nad_intr (tb, grid, true); - if (t.lam == HUGE_VAL) - return t; - - if (!inverse) { - in.lam += t.lam; - in.phi += t.phi; - return in; - } - - t.lam = tb.lam - t.lam; - t.phi = tb.phi - t.phi; - - do { - del = nad_intr(t, grid, true); - - /* We can possibly go outside of the initial guessed grid, so try */ - /* to fetch a new grid into which iterate... */ - if (del.lam == HUGE_VAL) - { - PJ_LP lp; - lp.lam = t.lam + extent->westLon; - lp.phi = t.phi + extent->southLat; - auto newGrid = findGrid(grids, lp); - if( newGrid == nullptr || newGrid == grid || newGrid->isNullGrid() ) - break; - pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", - grid->name().c_str(), - newGrid->name().c_str()); - grid = newGrid; - extent = &(grid->extentAndRes()); - t.lam = lp.lam - extent->westLon; - t.phi = lp.phi - extent->southLat; - tb = in; - tb.lam -= extent->westLon; - tb.phi -= extent->southLat; - tb.lam = adjlon (tb.lam - M_PI) + M_PI; - dif.lam = std::numeric_limits::max(); - dif.phi = std::numeric_limits::max(); - continue; - } - - dif.lam = t.lam + del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; - t.lam -= dif.lam; - t.phi -= dif.phi; - - } while (--i && (dif.lam*dif.lam + dif.phi*dif.phi > toltol)); /* prob. slightly faster than hypot() */ - - if (i==0) { - /* If we had access to a context, this should go through pj_log, and we should set ctx->errno */ - if (getenv ("PROJ_DEBUG")) - fprintf( stderr, "Inverse grid shift iterator failed to converge.\n" ); - t.lam = t.phi = HUGE_VAL; - return t; - } - - /* and again: pj_log and ctx->errno */ - if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) - fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - - in.lam = adjlon (t.lam + extent->westLon); - in.phi = t.phi + extent->southLat; - return in; -} - -/********************************************/ -/* proj_hgrid_value() */ -/* */ -/* Return coordinate offset in grid */ -/********************************************/ -PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids& grids, PJ_LP lp) { - PJ_LP out = proj_coord_error().lp; - - const auto grid = findGrid(grids, lp); - if( !grid ) { - pj_ctx_set_errno( P->ctx, PJD_ERR_GRID_AREA ); - return out; - } - - /* normalize input to ll origin */ - const auto& extent = grid->extentAndRes(); - lp.lam -= extent.westLon; - lp.phi -= extent.southLat; - - lp.lam = adjlon(lp.lam - M_PI) + M_PI; - - out = nad_intr(lp, grid, false); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { - pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); - } - - return out; -} - -static -PJ_LP proj_hgrid_apply_internal(PJ_CONTEXT* ctx, - PJ_LP lp, - PJ_DIRECTION direction, - const ListOfHGrids& grids) -{ - PJ_LP out; - - out.lam = HUGE_VAL; - out.phi = HUGE_VAL; - - const auto grid = findGrid(grids, lp); - if( !grid ) { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return out; - } - if( grid->isNullGrid() ) - { - return lp; - } - - int inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(ctx, lp, inverse, grid, grids); - - if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) - pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); - - return out; -} - -PJ_LP proj_hgrid_apply(PJ *P, const ListOfHGrids& grids, PJ_LP lp, PJ_DIRECTION direction) { - return proj_hgrid_apply_internal(P->ctx, lp, direction, grids); -} - - -NS_PROJ_END - -/************************************************************************/ -/* pj_apply_gridshift() */ -/* */ -/* This is the externally callable interface - part of the */ -/* public API - though it is not used internally any more and I */ -/* doubt it is used by any other applications. But we preserve */ -/* it to honour our public api. */ -/************************************************************************/ - -int pj_apply_gridshift( projCtx ctx, const char *nadgrids, int inverse, - long point_count, int point_offset, - double *x, double *y, double * /*z */ ) - -{ - auto hgrids = NS_PROJ::getListOfGridSets(ctx, nadgrids); - if( hgrids.empty() ) - { - pj_ctx_set_errno( ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return 1; - } - - - for( long i = 0; i < point_count; i++ ) - { - PJ_LP input; - - long io = i * point_offset; - input.phi = y[io]; - input.lam = x[io]; - - auto output = proj_hgrid_apply_internal(ctx, input, inverse ? PJ_INV : PJ_FWD, hgrids); - - if ( output.lam != HUGE_VAL ) - { - y[io] = output.phi; - x[io] = output.lam; - } - else - { - if( ctx->debug_level >= PJ_LOG_DEBUG_MAJOR ) - { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "pj_apply_gridshift(): failed to find a grid shift table for\n" - " location (%.7fdW,%.7fdN)", - x[io] * RAD_TO_DEG, - y[io] * RAD_TO_DEG ); - } - } - } - - return 0; -} diff --git a/src/apply_vgridshift.cpp b/src/apply_vgridshift.cpp deleted file mode 100644 index b0136e5cf0..0000000000 --- a/src/apply_vgridshift.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/****************************************************************************** - * Project: PROJ.4 - * Purpose: Apply vertical datum shifts based on grid shift files, normally - * geoid grids mapping WGS84 to NAVD88 or something similar. - * Author: Frank Warmerdam, warmerdam@pobox.com - * - ****************************************************************************** - * Copyright (c) 2010, Frank Warmerdam - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef FROM_PROJ_CPP -#define FROM_PROJ_CPP -#endif - -#include -#include -#include - -#include -#include "proj_internal.h" -#include "proj/internal/internal.hpp" -#include "grids.hpp" - -NS_PROJ_START - -static double read_vgrid_value(const ListOfVGrids& grids, PJ_LP input, double vmultiplier) { - - /* do not deal with NaN coordinates */ - /* cppcheck-suppress duplicateExpression */ - if( isnan(input.phi) || isnan(input.lam) ) - { - return HUGE_VAL; - } - - const VerticalShiftGrid* grid = nullptr; - for( const auto& gridset: grids ) - { - grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - break; - } - if( !grid ) - { - return HUGE_VAL; - } - - const auto& extent = grid->extentAndRes(); - - /* Interpolation a location within the grid */ - double grid_x = (input.lam - extent.westLon) / extent.resLon; - if( extent.fullWorldLongitude() ) { - // The first fmod goes to ]-lim, lim[ range - // So we add lim again to be in ]0, 2*lim[ and fmod again - grid_x = fmod( - fmod(grid_x + grid->width(), grid->width()) + grid->width(), - grid->width()); - } - double grid_y = (input.phi - extent.southLat) / extent.resLat; - int grid_ix = static_cast(lround(floor(grid_x))); - assert(grid_ix >= 0 && grid_ix < grid->width()); - int grid_iy = static_cast(lround(floor(grid_y))); - assert(grid_iy >= 0 && grid_iy < grid->height()); - grid_x -= grid_ix; - grid_y -= grid_iy; - - int grid_ix2 = grid_ix + 1; - if( grid_ix2 >= grid->width() ) { - if( extent.fullWorldLongitude() ) { - grid_ix2 = 0; - } else { - grid_ix2 = grid->width() - 1; - } - } - int grid_iy2 = grid_iy + 1; - if( grid_iy2 >= grid->height() ) - grid_iy2 = grid->height() - 1; - - float value_a = 0; - float value_b = 0; - float value_c = 0; - float value_d = 0; - if( !grid->valueAt(grid_ix, grid_iy, value_a) || - !grid->valueAt(grid_ix2, grid_iy, value_b) || - !grid->valueAt(grid_ix, grid_iy2, value_c) || - !grid->valueAt(grid_ix2, grid_iy2, value_d) ) - { - return HUGE_VAL; - } - - double total_weight = 0.0; - int n_weights = 0; - double value = 0.0f; - - if( !grid->isNodata(value_a, vmultiplier) ) - { - double weight = (1.0-grid_x) * (1.0-grid_y); - value += value_a * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_b, vmultiplier) ) - { - double weight = (grid_x) * (1.0-grid_y); - value += value_b * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_c, vmultiplier) ) - { - double weight = (1.0-grid_x) * (grid_y); - value += value_c * weight; - total_weight += weight; - n_weights ++; - } - if( !grid->isNodata(value_d, vmultiplier) ) - { - double weight = (grid_x) * (grid_y); - value += value_d * weight; - total_weight += weight; - n_weights ++; - } - if( n_weights == 0 ) - value = HUGE_VAL; - else if( n_weights != 4 ) - value /= total_weight; - - return value * vmultiplier; -} - -/**********************************************/ -ListOfVGrids proj_vgrid_init(PJ* P, const char *gridkey) { -/********************************************** - - Initizalize and populate gridlist. - - Takes a PJ-object and the plus-parameter - name that is used in the proj-string to - specify the grids to load, e.g. "+grids". - The + should be left out here. - - Returns the number of loaded grids. - -***********************************************/ - - std::string key("s"); - key += gridkey; - const char* gridnames = pj_param(P->ctx, P->params, key.c_str()).s; - if( gridnames == nullptr ) - return {}; - - auto listOfGridNames = internal::split(std::string(gridnames), ','); - ListOfVGrids grids; - for( const auto& gridnameStr: listOfGridNames ) - { - const char* gridname = gridnameStr.c_str(); - bool canFail = false; - if( gridname[0] == '@' ) - { - canFail = true; - gridname ++; - } - auto gridSet = VerticalShiftGridSet::open(P->ctx, gridname); - if( !gridSet ) - { - if( !canFail ) - { - pj_ctx_set_errno( P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID ); - return {}; - } - } - else - { - grids.emplace_back(std::move(gridSet)); - } - } - - return grids; -} - -/***********************************************/ -double proj_vgrid_value(PJ *P, const ListOfVGrids& grids, PJ_LP lp, double vmultiplier){ -/*********************************************** - - Read grid value at position lp in grids loaded - with proj_grid_init. - - Returns the grid value of the given coordinate. - -************************************************/ - - double value; - - value = read_vgrid_value(grids, lp, vmultiplier); - proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam*RAD_TO_DEG, lp.phi*RAD_TO_DEG, value); - - return value; -} - -NS_PROJ_END diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 68910a9470..9acea83e85 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -135,6 +135,9 @@ class FileStdio : public File { unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -198,6 +201,9 @@ class FileLegacyAdapter : public File { unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + // We may lie, but the real use case is only for network files + bool hasChanged() const override { return false; } + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -462,6 +468,13 @@ static const char *cache_db_structure_sql = " lastModified TEXT," " etag TEXT" ");" + "CREATE TABLE downloaded_file_properties(" + " url TEXT PRIMARY KEY NOT NULL," + " lastChecked TIMESTAMP NOT NULL," + " fileSize INTEGER NOT NULL," + " lastModified TEXT," + " etag TEXT" + ");" "CREATE TABLE chunk_data(" " id INTEGER PRIMARY KEY AUTOINCREMENT CHECK (id > 0)," " data BLOB NOT NULL" @@ -672,6 +685,7 @@ void DiskChunkCache::closeAndUnlink() { if (hDB_) { sqlite3_exec(hDB_, "COMMIT", nullptr, nullptr, nullptr); sqlite3_close(hDB_); + hDB_ = nullptr; } if (vfs_) { vfs_->raw()->xDelete(vfs_->raw(), path_.c_str(), 0); @@ -1336,8 +1350,9 @@ class NetworkFile : public File { unsigned long long m_pos = 0; size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; - unsigned long long m_filesize; + FileProperties m_props; proj_network_close_cbk_type m_closeCbk; + bool m_hasChanged = false; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -1346,9 +1361,9 @@ class NetworkFile : public File { NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, unsigned long long lastDownloadOffset, - unsigned long long filesize) + const FileProperties &props) : File(url), m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_lastDownloadedOffset(lastDownloadOffset), m_props(props), m_closeCbk(ctx->networking.close) {} public: @@ -1358,18 +1373,51 @@ class NetworkFile : public File { bool seek(unsigned long long offset, int whence) override; unsigned long long tell() override; void reassign_context(PJ_CONTEXT *ctx) override; + bool hasChanged() const override { return m_hasChanged; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); + + static bool get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props); }; // --------------------------------------------------------------------------- +bool NetworkFile::get_props_from_headers(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + FileProperties &props) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + props.size = std::stoull(slash + 1); + + const char *lastModified = ctx->networking.get_header_value( + ctx, handle, "Last-Modified", ctx->networking.user_data); + if (lastModified) + props.lastModified = lastModified; + + const char *etag = ctx->networking.get_header_value( + ctx, handle, "ETag", ctx->networking.user_data); + if (etag) + props.etag = etag; + + return true; + } + } + return false; +} + +// --------------------------------------------------------------------------- + std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { FileProperties props; if (gNetworkChunkCache.get(ctx, filename, 0, props)) { return std::unique_ptr(new NetworkFile( ctx, filename, nullptr, - std::numeric_limits::max(), props.size)); + std::numeric_limits::max(), props)); } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; @@ -1386,40 +1434,18 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { errorBuffer.c_str()); } - unsigned long long filesize = 0; + bool ok = false; if (handle) { - const char *contentRange = ctx->networking.get_header_value( - ctx, handle, "Content-Range", ctx->networking.user_data); - if (contentRange) { - const char *slash = strchr(contentRange, '/'); - if (slash) { - filesize = std::stoull(slash + 1); - - props.size = filesize; - - const char *lastModified = ctx->networking.get_header_value( - ctx, handle, "Last-Modified", - ctx->networking.user_data); - if (lastModified) - props.lastModified = lastModified; - - const char *etag = ctx->networking.get_header_value( - ctx, handle, "ETag", ctx->networking.user_data); - if (etag) - props.etag = etag; - - gNetworkFileProperties.insert(ctx, filename, props); - } - } - if (filesize != 0) { + if (get_props_from_headers(ctx, handle, props)) { + ok = true; + gNetworkFileProperties.insert(ctx, filename, props); gNetworkChunkCache.insert(ctx, filename, 0, std::move(buffer)); } } return std::unique_ptr( - handle != nullptr && filesize != 0 - ? new NetworkFile(ctx, filename, handle, size_read, filesize) - : nullptr); + ok ? new NetworkFile(ctx, filename, handle, size_read, props) + : nullptr); } } @@ -1504,6 +1530,20 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { } return 0; } + + if (!m_hasChanged) { + FileProperties props; + if (get_props_from_headers(m_ctx, m_handle, props)) { + if (props.size != m_props.size || + props.lastModified != m_props.lastModified || + props.etag != m_props.etag) { + gNetworkFileProperties.insert(m_ctx, m_url, props); + gNetworkChunkCache.clearMemoryCache(); + m_hasChanged = true; + } + } + } + region.resize(nRead); m_lastDownloadedOffset = offsetToDownload + nRead; @@ -1546,7 +1586,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; - m_pos = m_filesize; + m_pos = m_props.size; } return true; } @@ -1870,6 +1910,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, auto hCurlHandle = handle->m_handle; double oldDelay = MIN_RETRY_DELAY_MS; + std::string headers; std::string body; char szBuffer[128]; @@ -1879,6 +1920,12 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, while (true) { curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + headers.clear(); + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, + pj_curl_write_func); + body.clear(); body.reserve(size_to_read); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); @@ -1929,6 +1976,8 @@ static size_t pj_curl_read_range(PJ_CONTEXT *ctx, if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); } + handle->m_headers = std::move(headers); + return std::min(size_to_read, body.size()); } @@ -2252,50 +2301,468 @@ static void CreateDirectory(const std::string &path) { // --------------------------------------------------------------------------- -std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { - pj_load_ini(ctx); - if (!ctx->gridChunkCache.filename.empty()) { - return ctx->gridChunkCache.filename; +std::string pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, + bool create) { + if (ctx->user_writable_directory.empty()) { + // For testing purposes only + const char *env_var_PROJ_USER_WRITABLE_DIRECTORY = + getenv("PROJ_USER_WRITABLE_DIRECTORY"); + if (env_var_PROJ_USER_WRITABLE_DIRECTORY && + env_var_PROJ_USER_WRITABLE_DIRECTORY[0] != '\0') { + ctx->user_writable_directory = env_var_PROJ_USER_WRITABLE_DIRECTORY; + } } - std::string path; + if (ctx->user_writable_directory.empty()) { + std::string path; #ifdef _WIN32 - std::wstring wPath; - wPath.resize(MAX_PATH); - if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, &wPath[0]) == - S_OK) { - wPath.resize(wcslen(wPath.data())); - path = WStringToUTF8(wPath); - } else { - const char *local_app_data = getenv("LOCALAPPDATA"); - if (!local_app_data) { - local_app_data = getenv("TEMP"); + std::wstring wPath; + wPath.resize(MAX_PATH); + if (SHGetFolderPathW(nullptr, CSIDL_LOCAL_APPDATA, nullptr, 0, + &wPath[0]) == S_OK) { + wPath.resize(wcslen(wPath.data())); + path = WStringToUTF8(wPath); + } else { + const char *local_app_data = getenv("LOCALAPPDATA"); if (!local_app_data) { - local_app_data = "c:/users"; + local_app_data = getenv("TEMP"); + if (!local_app_data) { + local_app_data = "c:/users"; + } } + path = local_app_data; } - path = local_app_data; - } #else - const char *xdg_data_home = getenv("XDG_DATA_HOME"); - if (xdg_data_home != nullptr) { - path = xdg_data_home; - } else { - const char *home = getenv("HOME"); - if (home) { + const char *xdg_data_home = getenv("XDG_DATA_HOME"); + if (xdg_data_home != nullptr) { + path = xdg_data_home; + } else { + const char *home = getenv("HOME"); + if (home) { #if defined(__MACH__) && defined(__APPLE__) - path = std::string(home) + "/Library/Logs"; + path = std::string(home) + "/Library/Logs"; #else - path = std::string(home) + "/.local/share"; + path = std::string(home) + "/.local/share"; #endif - } else { - path = "/tmp"; + } else { + path = "/tmp"; + } } - } #endif - path += "/proj"; - CreateDirectory(path); + path += "/proj"; + ctx->user_writable_directory = path; + } + if (create) { + CreateDirectory(ctx->user_writable_directory); + } + return ctx->user_writable_directory; +} + +// --------------------------------------------------------------------------- + +std::string pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx) { + pj_load_ini(ctx); + if (!ctx->gridChunkCache.filename.empty()) { + return ctx->gridChunkCache.filename; + } + const std::string path(pj_context_get_user_writable_directory(ctx, true)); ctx->gridChunkCache.filename = path + "/cache.db"; return ctx->gridChunkCache.filename; } +// --------------------------------------------------------------------------- + +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +#else +static const char dir_chars[] = "/"; +#endif + +static bool is_tilde_slash(const char *name) { + return *name == '~' && strchr(dir_chars, name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) { + return strchr(dir_chars, *name) || + (*name == '.' && strchr(dir_chars, name[1])) || + (!strncmp(name, "..", 2) && strchr(dir_chars, name[2])) || + (name[0] != '\0' && name[1] == ':' && strchr(dir_chars, name[2])); +} + +static std::string build_url(PJ_CONTEXT *ctx, const char *name) { + if (!is_tilde_slash(name) && !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && !starts_with(name, "https://")) { + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if (!remote_file.empty()) { + if (remote_file.back() != '/') { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if (pos + 4 == remote_file.size()) { + remote_file = remote_file.substr(0, pos) + ".tif"; + } else { + // For example for resource files like 'alaska' + remote_file += ".tif"; + } + } + return remote_file; + } + return name; +} + //! @endcond + +// --------------------------------------------------------------------------- + +/** Return if a file must be downloaded or is already available in the + * PROJ user-writable directory. + * + * The file will be determinted to have to be downloaded if it does not exist + * yet in the user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @return TRUE if the file must be downloaded with proj_download_file() + * @since 7.0 + */ + +int proj_is_download_needed(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename( + pj_context_get_user_writable_directory(ctx, false) + filename); + + auto f = NS_PROJ::FileManager::open(ctx, localFilename.c_str()); + if (!f) { + return true; + } + f.reset(); + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return true; + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_ROW) { + return true; + } + + NS_PROJ::FileProperties cachedProps; + cachedProps.lastChecked = stmt->getInt64(); + cachedProps.size = stmt->getInt64(); + const char *lastModified = stmt->getText(); + cachedProps.lastModified = lastModified ? lastModified : std::string(); + const char *etag = stmt->getText(); + cachedProps.etag = etag ? etag : std::string(); + + if (!ignore_ttl_setting) { + const auto ttl = NS_PROJ::pj_context_get_grid_cache_ttl(ctx); + if (ttl > 0) { + time_t curTime; + time(&curTime); + if (curTime > cachedProps.lastChecked + ttl) { + + unsigned char dummy; + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, 1, &dummy, &size_read, + errorBuffer.size(), &errorBuffer[0], + ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + return false; + } + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, + props)) { + ctx->networking.close(ctx, handle, + ctx->networking.user_data); + return false; + } + ctx->networking.close(ctx, handle, ctx->networking.user_data); + + if (props.size != cachedProps.size || + props.lastModified != cachedProps.lastModified || + props.etag != cachedProps.etag) { + return true; + } + + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(curTime); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + auto hDB = diskCache->handle(); + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + } + } + + return false; +} + +// --------------------------------------------------------------------------- + +/** Download a file in the PROJ user-writable directory. + * + * The file will only be downloaded if it does not exist yet in the + * user-writable directory, or if it is determined that a more recent + * version exists. To determine if a more recent version exists, PROJ will + * use the "downloaded_file_properties" table of its grid cache database. + * Consequently files manually placed in the user-writable + * directory without using this function would be considered as + * non-existing/obsolete and would be unconditionnaly downloaded again. + * + * This function can only be used if networking is enabled, and either + * the default curl network API or a custom one have been installed. + * + * @param ctx PROJ context, or NULL + * @param url_or_filename URL or filename (without directory component) + * @param ignore_ttl_setting If set to FALSE, PROJ will only check the + * recentness of an already downloaded file, if + * the delay between the last time it has been + * verified and the current time exceeds the TTL + * setting. This can save network accesses. + * If set to TRUE, PROJ will unconditionnally + * check from the server the recentness of the file. + * @param progress_cbk Progress callback, or NULL. + * The passed percentage is in the [0, 1] range. + * The progress callback must return TRUE + * if download must be continued. + * @param user_data User data to provide to the progress callback, or NULL + * @return TRUE if the download was successful (or not needed) + * @since 7.0 + */ + +int proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!pj_context_is_network_enabled(ctx)) { + pj_log(ctx, PJ_LOG_ERROR, "Networking capabilities are not enabled"); + return false; + } + if (!proj_is_download_needed(ctx, url_or_filename, ignore_ttl_setting)) { + return true; + } + + const auto url(build_url(ctx, url_or_filename)); + const char *filename = strrchr(url.c_str(), '/'); + if (filename == nullptr) + return false; + const auto localFilename(pj_context_get_user_writable_directory(ctx, true) + + filename); + +#ifdef _WIN32 + const int nPID = GetCurrentProcessId(); +#else + const int nPID = getpid(); +#endif + char szUniqueSuffix[128]; + snprintf(szUniqueSuffix, sizeof(szUniqueSuffix), "%d_%p", nPID, &url); + const auto localFilenameTmp(localFilename + szUniqueSuffix); + FILE *f = fopen(localFilenameTmp.c_str(), "wb"); + if (!f) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot create %s", localFilenameTmp.c_str()); + return false; + } + + constexpr size_t FULL_FILE_CHUNK_SIZE = 1024 * 1024; + std::vector buffer(FULL_FILE_CHUNK_SIZE); + // For testing purposes only + const char *env_var_PROJ_FULL_FILE_CHUNK_SIZE = + getenv("PROJ_FULL_FILE_CHUNK_SIZE"); + if (env_var_PROJ_FULL_FILE_CHUNK_SIZE && + env_var_PROJ_FULL_FILE_CHUNK_SIZE[0] != '\0') { + buffer.resize(atoi(env_var_PROJ_FULL_FILE_CHUNK_SIZE)); + } + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + auto handle = ctx->networking.open( + ctx, url.c_str(), 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", url.c_str(), + errorBuffer.c_str()); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + time_t curTime; + time(&curTime); + NS_PROJ::FileProperties props; + if (!NS_PROJ::NetworkFile::get_props_from_headers(ctx, handle, props)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + if (size_read < + std::min(static_cast(buffer.size()), props.size)) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + if (fwrite(buffer.data(), size_read, 1, f) != 1) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + unsigned long long totalDownloaded = size_read; + while (totalDownloaded < props.size) { + if (totalDownloaded + buffer.size() > props.size) { + buffer.resize(static_cast(props.size - totalDownloaded)); + } + errorBuffer.resize(1024); + size_read = ctx->networking.read_range( + ctx, handle, totalDownloaded, buffer.size(), &buffer[0], + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + + if (size_read < buffer.size()) { + pj_log(ctx, PJ_LOG_ERROR, "Did not get as many bytes as expected"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + if (fwrite(buffer.data(), size_read, 1, f) != 1) { + pj_log(ctx, PJ_LOG_ERROR, "Write error"); + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + + totalDownloaded += size_read; + if (progress_cbk && + !progress_cbk(ctx, double(totalDownloaded) / props.size, + user_data)) { + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + unlink(localFilenameTmp.c_str()); + return false; + } + } + + ctx->networking.close(ctx, handle, ctx->networking.user_data); + fclose(f); + + unlink(localFilename.c_str()); + if (rename(localFilenameTmp.c_str(), localFilename.c_str()) != 0) { + pj_log(ctx, PJ_LOG_ERROR, "Cannot rename %s to %s", + localFilenameTmp.c_str(), localFilename.c_str()); + return false; + } + + auto diskCache = NS_PROJ::DiskChunkCache::open(ctx); + if (!diskCache) + return false; + auto stmt = + diskCache->prepare("SELECT lastChecked, fileSize, lastModified, etag " + "FROM downloaded_file_properties WHERE url = ?"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + + props.lastChecked = curTime; + auto hDB = diskCache->handle(); + + if (stmt->execute() == SQLITE_ROW) { + stmt = diskCache->prepare( + "UPDATE downloaded_file_properties SET lastChecked = ?, " + "fileSize = ?, lastModified = ?, etag = ? " + "WHERE url = ?"); + if (!stmt) + return false; + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + stmt->bindText(url.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } else { + stmt = diskCache->prepare( + "INSERT INTO downloaded_file_properties (url, lastChecked, " + "fileSize, lastModified, etag) VALUES " + "(?,?,?,?,?)"); + if (!stmt) + return false; + stmt->bindText(url.c_str()); + stmt->bindInt64(props.lastChecked); + stmt->bindInt64(props.size); + if (props.lastModified.empty()) + stmt->bindNull(); + else + stmt->bindText(props.lastModified.c_str()); + if (props.etag.empty()) + stmt->bindNull(); + else + stmt->bindText(props.etag.c_str()); + if (stmt->execute() != SQLITE_DONE) { + pj_log(ctx, PJ_LOG_ERROR, "%s", sqlite3_errmsg(hDB)); + return false; + } + } + return true; +} diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 993048a79f..9793267c80 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -33,10 +33,10 @@ #include "proj.h" #include "proj/util.hpp" -NS_PROJ_START - //! @cond Doxygen_Suppress +NS_PROJ_START + class File; class FileManager { @@ -69,12 +69,13 @@ class File { virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; virtual void reassign_context(PJ_CONTEXT *ctx) = 0; + virtual bool hasChanged() const = 0; const std::string &name() const { return name_; } }; -//! @endcond Doxygen_Suppress - NS_PROJ_END +//! @endcond Doxygen_Suppress + #endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/grids.cpp b/src/grids.cpp index 5a99106bdb..3007fedc78 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -140,6 +140,7 @@ class NullVerticalShiftGrid : public VerticalShiftGrid { bool valueAt(int, int, float &out) const override; bool isNodata(float, double) const override { return false; } void reassign_context(PJ_CONTEXT *) override {} + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -177,6 +178,8 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -373,6 +376,7 @@ class GTiffGrid : public Grid { PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset TIFF *m_hTIFF; // owned by the belonging GTiffDataset BlockCache &m_cache; // owned by the belonging GTiffDataset + File *m_fp; // owned by the belonging GTiffDataset uint32 m_ifdIdx; TIFFDataType m_dt; uint16 m_samplesPerPixel; @@ -402,9 +406,9 @@ class GTiffGrid : public Grid { uint32 offsetInBlock, uint16 sample) const; public: - GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, - const std::string &nameIn, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); ~GTiffGrid() override; @@ -420,17 +424,19 @@ class GTiffGrid : public Grid { uint32 subfileType() const { return m_subfileType; } void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- -GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, File *fp, uint32 ifdIdx, const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn) : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), - m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), + m_cache(cache), m_fp(fp), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), m_tiled(TIFFIsTiled(hTIFF) != 0) { @@ -1048,8 +1054,8 @@ std::unique_ptr GTiffDataset::nextGrid() { } auto ret = std::unique_ptr(new GTiffGrid( - m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent, - dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ctx, m_hTIFF, m_cache, m_fp.get(), m_ifdIdx, m_filename, width, + height, extent, dt, samplesPerPixel, planarConfig, vRes < 0)); m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); @@ -1058,10 +1064,12 @@ std::unique_ptr GTiffDataset::nextGrid() { // --------------------------------------------------------------------------- -class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { +class GTiffVGridShiftSet : public VerticalShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffVGridShiftSet() override; @@ -1072,7 +1080,26 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { void reassign_context(PJ_CONTEXT *ctx) override { VerticalShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -1172,6 +1199,8 @@ class GTiffVGrid : public VerticalShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1221,14 +1250,14 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffVGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } uint16 idxSample = 0; std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -1364,6 +1393,19 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool VerticalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const VerticalShiftGrid *VerticalShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { @@ -1435,6 +1477,8 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { float &latShift) const override; void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -1482,6 +1526,8 @@ class NTv1Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1603,6 +1649,8 @@ class CTable2Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1737,6 +1785,8 @@ class NTv2Grid : public HorizontalShiftGrid { m_ctx = ctx; m_fp->reassign_context(ctx); } + + bool hasChanged() const override { return m_fp->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -1790,6 +1840,11 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } + if (memcmp(header + 56, "SECONDS", 7) != 0) { + pj_log(ctx, PJ_LOG_ERROR, "Only GS_TYPE=SECONDS is supported"); + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return nullptr; + } const bool must_swap = (header[8] == 11) ? !IS_LSB : IS_LSB; if (must_swap) { @@ -1906,10 +1961,12 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- -class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { +class GTiffHGridShiftSet : public HorizontalShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffHGridShiftSet() override; @@ -1920,7 +1977,26 @@ class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { void reassign_context(PJ_CONTEXT *ctx) override { HorizontalShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -1954,6 +2030,8 @@ class GTiffHGrid : public HorizontalShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -2024,7 +2102,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffHGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } @@ -2037,7 +2115,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -2269,6 +2347,19 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool HorizontalShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const HorizontalShiftGrid *HorizontalShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { @@ -2317,11 +2408,12 @@ void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { #ifdef TIFF_ENABLED // --------------------------------------------------------------------------- -class GTiffGenericGridShiftSet : public GenericShiftGridSet, - public GTiffDataset { +class GTiffGenericGridShiftSet : public GenericShiftGridSet { + + std::unique_ptr m_GTiffDataset; GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) - : GTiffDataset(ctx, std::move(fp)) {} + : m_GTiffDataset(new GTiffDataset(ctx, std::move(fp))) {} public: ~GTiffGenericGridShiftSet() override; @@ -2332,7 +2424,26 @@ class GTiffGenericGridShiftSet : public GenericShiftGridSet, void reassign_context(PJ_CONTEXT *ctx) override { GenericShiftGridSet::reassign_context(ctx); - GTiffDataset::reassign_context(ctx); + if (m_GTiffDataset) { + m_GTiffDataset->reassign_context(ctx); + } + } + + bool reopen(PJ_CONTEXT *ctx) override { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + m_grids.clear(); + m_GTiffDataset.reset(); + auto fp = FileManager::open_resource_file(ctx, m_name.c_str()); + if (!fp) { + return false; + } + auto newGS = open(ctx, std::move(fp), m_name); + if (newGS) { + m_grids = std::move(newGS->m_grids); + m_GTiffDataset = std::move(newGS->m_GTiffDataset); + } + return !m_grids.empty(); } }; @@ -2375,6 +2486,8 @@ class GTiffGenericGrid : public GenericShiftGrid { void reassign_context(PJ_CONTEXT *ctx) override { m_grid->reassign_context(ctx); } + + bool hasChanged() const override { return m_grid->hasChanged(); } }; // --------------------------------------------------------------------------- @@ -2446,6 +2559,8 @@ class NullGenericShiftGrid : public GenericShiftGrid { } void reassign_context(PJ_CONTEXT *) override {} + + bool hasChanged() const override { return false; } }; // --------------------------------------------------------------------------- @@ -2466,13 +2581,13 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, new GTiffGenericGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; - if (!set->openTIFF(filename)) { + if (!set->m_GTiffDataset->openTIFF(filename)) { return nullptr; } std::map mapGrids; for (int ifd = 0;; ++ifd) { - auto grid = set->nextGrid(); + auto grid = set->m_GTiffDataset->nextGrid(); if (!grid) { if (ifd == 0) { return nullptr; @@ -2574,6 +2689,19 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { // --------------------------------------------------------------------------- +bool GenericShiftGridSet::reopen(PJ_CONTEXT *ctx) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Grid %s has changed. Re-loading it", + m_name.c_str()); + auto newGS = open(ctx, m_name); + m_grids.clear(); + if (newGS) { + m_grids = std::move(newGS->m_grids); + } + return !m_grids.empty(); +} + +// --------------------------------------------------------------------------- + const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { for (const auto &child : m_children) { const auto &extentChild = child->extentAndRes(); @@ -2614,7 +2742,7 @@ void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *gridkey) { std::string key("s"); key += gridkey; const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; @@ -2644,4 +2772,615 @@ ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { return grids; } +// --------------------------------------------------------------------------- + +static const HorizontalShiftGrid * +findGrid(const ListOfHGrids &grids, const PJ_LP &input, + HorizontalShiftGridSet *&gridSetOut) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + gridSetOut = gridset.get(); + return grid; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static ListOfHGrids getListOfGridSets(PJ_CONTEXT *ctx, const char *grids) { + ListOfHGrids list; + auto listOfGrids = internal::split(std::string(grids), ','); + for (const auto &grid : listOfGrids) { + const char *gridname = grid.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = HorizontalShiftGridSet::open(ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + list.emplace_back(std::move(gridSet)); + } + } + return list; +} + +/**********************************************/ +ListOfHGrids pj_hgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate list of horizontal + grids. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *grids = pj_param(P->ctx, P->params, key.c_str()).s; + if (grids == nullptr) + return {}; + + return getListOfGridSets(P->ctx, grids); +} + +// --------------------------------------------------------------------------- + +typedef struct { pj_int32 lam, phi; } ILP; + +static PJ_LP pj_hgrid_interpolate(PJ_LP t, const HorizontalShiftGrid *grid, + bool compensateNTConvention) { + PJ_LP val, frct; + ILP indx; + int in; + + const auto &extent = grid->extentAndRes(); + t.lam /= extent.resLon; + indx.lam = std::isnan(t.lam) ? 0 : (pj_int32)lround(floor(t.lam)); + t.phi /= extent.resLat; + indx.phi = std::isnan(t.phi) ? 0 : (pj_int32)lround(floor(t.phi)); + + frct.lam = t.lam - indx.lam; + frct.phi = t.phi - indx.phi; + val.lam = val.phi = HUGE_VAL; + if (indx.lam < 0) { + if (indx.lam == -1 && frct.lam > 0.99999999999) { + ++indx.lam; + frct.lam = 0.; + } else + return val; + } else if ((in = indx.lam + 1) >= grid->width()) { + if (in == grid->width() && frct.lam < 1e-11) { + --indx.lam; + frct.lam = 1.; + } else + return val; + } + if (indx.phi < 0) { + if (indx.phi == -1 && frct.phi > 0.99999999999) { + ++indx.phi; + frct.phi = 0.; + } else + return val; + } else if ((in = indx.phi + 1) >= grid->height()) { + if (in == grid->height() && frct.phi < 1e-11) { + --indx.phi; + frct.phi = 1.; + } else + return val; + } + + float f00Lon = 0, f00Lat = 0; + float f10Lon = 0, f10Lat = 0; + float f01Lon = 0, f01Lat = 0; + float f11Lon = 0, f11Lat = 0; + if (!grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, + f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, + f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, + f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, + f11Lon, f11Lat)) { + return val; + } + + double m10 = frct.lam; + double m11 = m10; + double m01 = 1. - frct.lam; + double m00 = m01; + m11 *= frct.phi; + m01 *= frct.phi; + frct.phi = 1. - frct.phi; + m00 *= frct.phi; + m10 *= frct.phi; + val.lam = m00 * f00Lon + m10 * f10Lon + m01 * f01Lon + m11 * f11Lon; + val.phi = m00 * f00Lat + m10 * f10Lat + m01 * f01Lat + m11 * f11Lat; + return val; +} + +// --------------------------------------------------------------------------- + +#define MAX_ITERATIONS 10 +#define TOL 1e-12 + +static PJ_LP pj_hgrid_apply_internal(projCtx ctx, PJ_LP in, + PJ_DIRECTION direction, + const HorizontalShiftGrid *grid, + HorizontalShiftGridSet *gridset, + const ListOfHGrids &grids, + bool &shouldRetry) { + PJ_LP t, tb, del, dif; + int i = MAX_ITERATIONS; + const double toltol = TOL * TOL; + + shouldRetry = false; + if (in.lam == HUGE_VAL) + return in; + + /* normalize input to ll origin */ + tb = in; + const auto *extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + + tb.lam = adjlon(tb.lam - M_PI) + M_PI; + + t = pj_hgrid_interpolate(tb, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } + if (t.lam == HUGE_VAL) + return t; + + if (direction == PJ_FWD) { + in.lam += t.lam; + in.phi += t.phi; + return in; + } + + t.lam = tb.lam - t.lam; + t.phi = tb.phi - t.phi; + + do { + del = pj_hgrid_interpolate(t, grid, true); + if (grid->hasChanged()) { + shouldRetry = gridset->reopen(ctx); + return t; + } + + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ + if (del.lam == HUGE_VAL) { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp, gridset); + if (newGrid == nullptr || newGrid == grid || newGrid->isNullGrid()) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + tb.lam = adjlon(tb.lam - M_PI) + M_PI; + dif.lam = std::numeric_limits::max(); + dif.phi = std::numeric_limits::max(); + continue; + } + + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; + t.lam -= dif.lam; + t.phi -= dif.phi; + + } while (--i && (dif.lam * dif.lam + dif.phi * dif.phi > + toltol)); /* prob. slightly faster than hypot() */ + + if (i == 0) { + /* If we had access to a context, this should go through pj_log, and we + * should set ctx->errno */ + if (getenv("PROJ_DEBUG")) + fprintf(stderr, + "Inverse grid shift iterator failed to converge.\n"); + t.lam = t.phi = HUGE_VAL; + return t; + } + + /* and again: pj_log and ctx->errno */ + if (del.lam == HUGE_VAL && getenv("PROJ_DEBUG")) + fprintf(stderr, "Inverse grid shift iteration failed, presumably at " + "grid edge.\nUsing first approximation.\n"); + + in.lam = adjlon(t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; + return in; +} + +// --------------------------------------------------------------------------- + +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction) { + PJ_LP out; + + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + + while (true) { + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); + if (!grid) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return out; + } + if (grid->isNullGrid()) { + return lp; + } + + bool shouldRetry = false; + out = pj_hgrid_apply_internal(ctx, lp, direction, grid, gridset, grids, + shouldRetry); + if (!shouldRetry) { + break; + } + } + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) + pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); + + return out; +} + +/********************************************/ +/* proj_hgrid_value() */ +/* */ +/* Return coordinate offset in grid */ +/********************************************/ +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp) { + PJ_LP out = proj_coord_error().lp; + + HorizontalShiftGridSet *gridset = nullptr; + const auto grid = findGrid(grids, lp, gridset); + if (!grid) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + return out; + } + + /* normalize input to ll origin */ + const auto &extent = grid->extentAndRes(); + lp.lam -= extent.westLon; + lp.phi -= extent.southLat; + + lp.lam = adjlon(lp.lam - M_PI) + M_PI; + + out = pj_hgrid_interpolate(lp, grid, false); + if (grid->hasChanged()) { + if (gridset->reopen(P->ctx)) { + return pj_hgrid_value(P, grids, lp); + } + out.lam = HUGE_VAL; + out.phi = HUGE_VAL; + } + + if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { + pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); + } + + return out; +} + +// --------------------------------------------------------------------------- + +static double read_vgrid_value(PJ_CONTEXT *ctx, const ListOfVGrids &grids, + const PJ_LP &input, const double vmultiplier) { + + /* do not deal with NaN coordinates */ + /* cppcheck-suppress duplicateExpression */ + if (std::isnan(input.phi) || std::isnan(input.lam)) { + return HUGE_VAL; + } + + VerticalShiftGridSet *curGridset = nullptr; + const VerticalShiftGrid *grid = nullptr; + for (const auto &gridset : grids) { + grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + curGridset = gridset.get(); + break; + } + } + if (!grid) { + return HUGE_VAL; + } + + const auto &extent = grid->extentAndRes(); + + /* Interpolation a location within the grid */ + double grid_x = (input.lam - extent.westLon) / extent.resLon; + if (extent.fullWorldLongitude()) { + // The first fmod goes to ]-lim, lim[ range + // So we add lim again to be in ]0, 2*lim[ and fmod again + grid_x = + fmod(fmod(grid_x + grid->width(), grid->width()) + grid->width(), + grid->width()); + } + double grid_y = (input.phi - extent.southLat) / extent.resLat; + int grid_ix = static_cast(lround(floor(grid_x))); + assert(grid_ix >= 0 && grid_ix < grid->width()); + int grid_iy = static_cast(lround(floor(grid_y))); + assert(grid_iy >= 0 && grid_iy < grid->height()); + grid_x -= grid_ix; + grid_y -= grid_iy; + + int grid_ix2 = grid_ix + 1; + if (grid_ix2 >= grid->width()) { + if (extent.fullWorldLongitude()) { + grid_ix2 = 0; + } else { + grid_ix2 = grid->width() - 1; + } + } + int grid_iy2 = grid_iy + 1; + if (grid_iy2 >= grid->height()) + grid_iy2 = grid->height() - 1; + + float value_a = 0; + float value_b = 0; + float value_c = 0; + float value_d = 0; + bool error = (!grid->valueAt(grid_ix, grid_iy, value_a) || + !grid->valueAt(grid_ix2, grid_iy, value_b) || + !grid->valueAt(grid_ix, grid_iy2, value_c) || + !grid->valueAt(grid_ix2, grid_iy2, value_d)); + if (grid->hasChanged()) { + if (curGridset->reopen(ctx)) { + return read_vgrid_value(ctx, grids, input, vmultiplier); + } + error = true; + } + + if (error) { + return HUGE_VAL; + } + + double total_weight = 0.0; + int n_weights = 0; + double value = 0.0f; + + if (!grid->isNodata(value_a, vmultiplier)) { + double weight = (1.0 - grid_x) * (1.0 - grid_y); + value += value_a * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_b, vmultiplier)) { + double weight = (grid_x) * (1.0 - grid_y); + value += value_b * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_c, vmultiplier)) { + double weight = (1.0 - grid_x) * (grid_y); + value += value_c * weight; + total_weight += weight; + n_weights++; + } + if (!grid->isNodata(value_d, vmultiplier)) { + double weight = (grid_x) * (grid_y); + value += value_d * weight; + total_weight += weight; + n_weights++; + } + if (n_weights == 0) + value = HUGE_VAL; + else if (n_weights != 4) + value /= total_weight; + + return value * vmultiplier; +} + +/**********************************************/ +ListOfVGrids pj_vgrid_init(PJ *P, const char *gridkey) { + /********************************************** + + Initizalize and populate gridlist. + + Takes a PJ-object and the plus-parameter + name that is used in the proj-string to + specify the grids to load, e.g. "+grids". + The + should be left out here. + + Returns the number of loaded grids. + + ***********************************************/ + + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfVGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = VerticalShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + +/***********************************************/ +double pj_vgrid_value(PJ *P, const ListOfVGrids &grids, PJ_LP lp, + double vmultiplier) { + /*********************************************** + + Read grid value at position lp in grids loaded + with proj_grid_init. + + Returns the grid value of the given coordinate. + + ************************************************/ + + double value; + + value = read_vgrid_value(P->ctx, grids, lp, vmultiplier); + proj_log_trace(P, "proj_vgrid_value: (%f, %f) = %f", lp.lam * RAD_TO_DEG, + lp.phi * RAD_TO_DEG, value); + + return value; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut) { + for (const auto &gridset : grids) { + auto grid = gridset->gridAt(input.lam, input.phi); + if (grid) { + gridSetOut = gridset.get(); + return grid; + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry) { + must_retry = false; + + const auto &extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast(grid_x); + int iy = static_cast(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1 = 0.0f, dy1 = 0.0f, dz1 = 0.0f; + float dx2 = 0.0f, dy2 = 0.0f, dz2 = 0.0f; + float dx3 = 0.0f, dy3 = 0.0f, dz3 = 0.0f; + float dx4 = 0.0f, dy4 = 0.0f, dz4 = 0.0f; + bool error = (!grid->valueAt(ix, iy, idx1, dx1) || + !grid->valueAt(ix, iy, idx2, dy1) || + !grid->valueAt(ix, iy, idx3, dz1) || + !grid->valueAt(ix2, iy, idx1, dx2) || + !grid->valueAt(ix2, iy, idx2, dy2) || + !grid->valueAt(ix2, iy, idx3, dz2) || + !grid->valueAt(ix, iy2, idx1, dx3) || + !grid->valueAt(ix, iy2, idx2, dy3) || + !grid->valueAt(ix, iy2, idx3, dz3) || + !grid->valueAt(ix2, iy2, idx1, dx4) || + !grid->valueAt(ix2, iy2, idx2, dy4) || + !grid->valueAt(ix2, iy2, idx3, dz4)); + if (grid->hasChanged()) { + must_retry = true; + return false; + } + if (error) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + + v1 = m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4; + v2 = m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4; + v3 = m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4; + return true; +} + NS_PROJ_END + +/************************************************************************/ +/* pj_apply_gridshift() */ +/* */ +/* This is the externally callable interface - part of the */ +/* public API - though it is not used internally any more and I */ +/* doubt it is used by any other applications. But we preserve */ +/* it to honour our public api. */ +/************************************************************************/ + +int pj_apply_gridshift(projCtx ctx, const char *nadgrids, int inverse, + long point_count, int point_offset, double *x, double *y, + double * /*z */) + +{ + auto hgrids = NS_PROJ::getListOfGridSets(ctx, nadgrids); + if (hgrids.empty()) { + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return 1; + } + + for (long i = 0; i < point_count; i++) { + PJ_LP input; + + long io = i * point_offset; + input.phi = y[io]; + input.lam = x[io]; + + auto output = + pj_hgrid_apply(ctx, hgrids, input, inverse ? PJ_INV : PJ_FWD); + + if (output.lam != HUGE_VAL) { + y[io] = output.phi; + x[io] = output.lam; + } else { + if (ctx->debug_level >= PJ_LOG_DEBUG_MAJOR) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "pj_apply_gridshift(): failed to find a grid shift " + "table for\n" + " location (%.7fdW,%.7fdN)", + x[io] * RAD_TO_DEG, y[io] * RAD_TO_DEG); + } + } + } + + return 0; +} diff --git a/src/grids.hpp b/src/grids.hpp index aa852ef6f9..6b6ee0d2f1 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -70,6 +70,7 @@ class Grid { const std::string &name() const { return m_name; } virtual bool isNullGrid() const { return false; } + virtual bool hasChanged() const = 0; }; // --------------------------------------------------------------------------- @@ -116,6 +117,7 @@ class VerticalShiftGridSet { const VerticalShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -162,6 +164,7 @@ class HorizontalShiftGridSet { const HorizontalShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -217,6 +220,7 @@ class GenericShiftGridSet { const GenericShiftGrid *gridAt(double lon, double lat) const; virtual void reassign_context(PJ_CONTEXT *ctx); + virtual bool reopen(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -225,15 +229,24 @@ typedef std::vector> ListOfHGrids; typedef std::vector> ListOfVGrids; typedef std::vector> ListOfGenericGrids; -ListOfVGrids proj_vgrid_init(PJ *P, const char *grids); -ListOfHGrids proj_hgrid_init(PJ *P, const char *grids); -ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *grids); - -double proj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, - double vmultiplier); -PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids &, PJ_LP lp); -PJ_LP proj_hgrid_apply(PJ *P, const ListOfHGrids &, PJ_LP lp, - PJ_DIRECTION direction); +ListOfVGrids pj_vgrid_init(PJ *P, const char *grids); +ListOfHGrids pj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids pj_generic_grid_init(PJ *P, const char *grids); + +PJ_LP pj_hgrid_value(PJ *P, const ListOfHGrids &grids, PJ_LP lp); +double pj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, + double vmultiplier); +PJ_LP pj_hgrid_apply(PJ_CONTEXT *ctx, const ListOfHGrids &grids, PJ_LP lp, + PJ_DIRECTION direction); + +const GenericShiftGrid *pj_find_generic_grid(const ListOfGenericGrids &grids, + const PJ_LP &input, + GenericShiftGridSet *&gridSetOut); +bool pj_bilinear_interpolation_three_samples(const GenericShiftGrid *grid, + const PJ_LP &lp, int idx1, + int idx2, int idx3, double &v1, + double &v2, double &v3, + bool &must_retry); NS_PROJ_END diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index 18c9102166..fdb5943474 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -220,8 +220,6 @@ set(SRC_LIBPROJ_CORE 4D_api.cpp aasincos.cpp adjlon.cpp - apply_gridshift.cpp - apply_vgridshift.cpp auth.cpp ctx.cpp datum_set.cpp diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 23ee79a0a3..ae387281a0 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -224,6 +224,16 @@ static bool is_rel_or_absolute_filename(const char *name) || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])); } +static bool ignoreUserWritableDirectory() +{ + // Env var mostly for testing purposes and being independent from + // an existing installation + const char* envVarIgnoreUserWritableDirectory = + getenv("PROJ_IGNORE_USER_WRITABLE_DIRECTORY"); + return envVarIgnoreUserWritableDirectory != nullptr && + envVarIgnoreUserWritableDirectory[0] != '\0'; +} + static void* pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, void* (*open_file)(projCtx, const char*, const char*), @@ -279,6 +289,17 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, break; } } + + else if( !ignoreUserWritableDirectory() && + (fid = open_file(ctx, + (pj_context_get_user_writable_directory(ctx, false) + + DIR_CHAR + name).c_str(), mode)) != nullptr ) { + fname = pj_context_get_user_writable_directory(ctx, false); + fname += DIR_CHAR; + fname += name; + sysname = fname.c_str(); + } + /* if is environment PROJ_LIB defined */ else if ((sysname = getenv("PROJ_LIB")) != nullptr) { auto paths = NS_PROJ::internal::split(std::string(sysname), dirSeparator); @@ -381,6 +402,23 @@ std::unique_ptr NS_PROJ::FileManager::open_resource_file( "Using %s", remote_file.c_str() ); pj_ctx_set_errno( ctx, 0 ); } + } else { + // For example for resource files like 'alaska' + auto remote_file_tif = remote_file + ".tif"; + file = open(ctx, remote_file_tif.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file_tif.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } else { + // Init files + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } + } } } } diff --git a/src/proj.h b/src/proj.h index 33057617bb..8add29acc5 100644 --- a/src/proj.h +++ b/src/proj.h @@ -401,6 +401,9 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( * * Read size_to_read bytes from handle, starting at offset, into * buffer. + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type callback. * * error_string_max_size should be the maximum size that can be written into * the out_error_string buffer (including terminating nul character). @@ -440,6 +443,15 @@ void PROJ_DLL proj_grid_cache_set_ttl(PJ_CONTEXT* ctx, int ttl_seconds); void PROJ_DLL proj_grid_cache_clear(PJ_CONTEXT* ctx); +int PROJ_DLL proj_is_download_needed(PJ_CONTEXT* ctx, + const char* url_or_filename, + int ignore_ttl_setting); +int PROJ_DLL proj_download_file(PJ_CONTEXT *ctx, const char *url_or_filename, + int ignore_ttl_setting, + int (*progress_cbk)(PJ_CONTEXT *, double pct, + void *user_data), + void *user_data); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 63c5355149..ce7b9d74af 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -709,6 +709,7 @@ struct projCtx_t { bool iniFileLoaded = false; std::string endpoint{}; + std::string user_writable_directory{}; projGridChunkCache gridChunkCache{}; int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() @@ -844,8 +845,9 @@ std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); void pj_load_ini(PJ_CONTEXT* ctx); -// For testing purposes +// Exported for testing purposes only std::string PROJ_DLL pj_context_get_grid_cache_filename(PJ_CONTEXT *ctx); +std::string PROJ_DLL pj_context_get_user_writable_directory(PJ_CONTEXT *ctx, bool create); /* classic public API */ #include "proj_api.h" diff --git a/src/sqlite3.cpp b/src/sqlite3.cpp index 0c89c0b9a7..90e73c2a8e 100644 --- a/src/sqlite3.cpp +++ b/src/sqlite3.cpp @@ -25,8 +25,17 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + #include "sqlite3.hpp" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + #include #include #include // std::ostringstream @@ -182,4 +191,4 @@ SQLiteStatement::SQLiteStatement(sqlite3_stmt *hStmtIn) : hStmt(hStmtIn) {} // --------------------------------------------------------------------------- -NS_PROJ_END \ No newline at end of file +NS_PROJ_END diff --git a/src/transform.cpp b/src/transform.cpp index 020d62ea9a..811e2a6ac4 100644 --- a/src/transform.cpp +++ b/src/transform.cpp @@ -451,7 +451,7 @@ static int pj_apply_vgridshift( PJ *defn, if( defn->vgrids_legacy == nullptr ) { defn->vgrids_legacy = new ListOfVGrids; - auto vgrids = proj_vgrid_init(defn, "geoidgrids"); + auto vgrids = pj_vgrid_init(defn, "geoidgrids"); if( vgrids.empty() ) return 0; *static_cast(defn->vgrids_legacy) = std::move(vgrids); @@ -470,7 +470,7 @@ static int pj_apply_vgridshift( PJ *defn, input.phi = y[io]; input.lam = x[io]; - value = proj_vgrid_value(defn, *static_cast(defn->vgrids_legacy), input, 1.0); + value = pj_vgrid_value(defn, *static_cast(defn->vgrids_legacy), input, 1.0); if( inverse ) z[io] -= value; @@ -896,7 +896,7 @@ int pj_apply_gridshift_2( PJ *defn, int inverse, if( defn->hgrids_legacy == nullptr ) { defn->hgrids_legacy = new ListOfHGrids; - auto hgrids = proj_hgrid_init(defn, "nadgrids"); + auto hgrids = pj_hgrid_init(defn, "nadgrids"); if( hgrids.empty() ) return 0; *static_cast(defn->hgrids_legacy) = std::move(hgrids); @@ -914,7 +914,7 @@ int pj_apply_gridshift_2( PJ *defn, int inverse, input.phi = y[io]; input.lam = x[io]; - auto output = proj_hgrid_apply(defn, *static_cast(defn->hgrids_legacy), input, inverse ? PJ_INV : PJ_FWD); + auto output = pj_hgrid_apply(defn->ctx, *static_cast(defn->hgrids_legacy), input, inverse ? PJ_INV : PJ_FWD); if ( output.lam != HUGE_VAL ) { diff --git a/src/transformations/deformation.cpp b/src/transformations/deformation.cpp index eb10982690..8aee50c944 100644 --- a/src/transformations/deformation.cpp +++ b/src/transformations/deformation.cpp @@ -80,20 +80,6 @@ struct deformationData { // --------------------------------------------------------------------------- -static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - return grid; - } - return nullptr; -} - -// --------------------------------------------------------------------------- - static bool get_grid_values(PJ* P, deformationData* Q, const PJ_LP& lp, @@ -101,7 +87,8 @@ static bool get_grid_values(PJ* P, double& vy, double& vz) { - auto grid = findGrid(Q->grids, lp); + GenericShiftGridSet* gridset = nullptr; + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -136,57 +123,20 @@ static bool get_grid_values(PJ* P, return false; } - const auto& extent = grid->extentAndRes(); - double grid_x = (lp.lam - extent.westLon) / extent.resLon; - double grid_y = (lp.phi - extent.southLat) / extent.resLat; - int ix = static_cast(grid_x); - int iy = static_cast(grid_y); - int ix2 = std::min(ix + 1, grid->width() - 1); - int iy2 = std::min(iy + 1, grid->height() - 1); - - float dx1, dy1, dz1; - if( !grid->valueAt(ix, iy, sampleE, dx1) || - !grid->valueAt(ix, iy, sampleN, dy1) || - !grid->valueAt(ix, iy, sampleU, dz1) ) { - return false; - } - - float dx2, dy2, dz2; - if( !grid->valueAt(ix2, iy, sampleE, dx2) || - !grid->valueAt(ix2, iy, sampleN, dy2) || - !grid->valueAt(ix2, iy, sampleU, dz2) ) { - return false; - } - - float dx3, dy3, dz3; - if( !grid->valueAt(ix, iy2, sampleE, dx3) || - !grid->valueAt(ix, iy2, sampleN, dy3) || - !grid->valueAt(ix, iy2, sampleU, dz3) ) { - return false; - } - - float dx4, dy4, dz4; - if( !grid->valueAt(ix2, iy2, sampleE, dx4) || - !grid->valueAt(ix2, iy2, sampleN, dy4) || - !grid->valueAt(ix2, iy2, sampleU, dz4) ) { + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleE, sampleN, sampleU, + vx, vy, vz, + must_retry) ) + { + if( must_retry ) + return get_grid_values( P, Q, lp, vx, vy, vz); return false; } - - double frct_lam = grid_x - ix; - double frct_phi = grid_y - iy; - double m10 = frct_lam; - double m11 = m10; - double m01 = 1. - frct_lam; - double m00 = m01; - m11 *= frct_phi; - m01 *= frct_phi; - frct_phi = 1. - frct_phi; - m00 *= frct_phi; - m10 *= frct_phi; // divide by 1000 to get m/year - vx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) / 1000; - vy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) / 1000; - vz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) / 1000; + vx /= 1000; + vy /= 1000; + vz /= 1000; return true; } @@ -226,8 +176,8 @@ static PJ_XYZ get_grid_shift(PJ* P, const PJ_XYZ& cartesian) { } else { - shift.lp = proj_hgrid_value(P, Q->hgrids, geodetic.lp); - shift.enu.u = proj_vgrid_value(P, Q->vgrids, geodetic.lp, 1.0); + shift.lp = pj_hgrid_value(P, Q->hgrids, geodetic.lp); + shift.enu.u = pj_vgrid_value(P, Q->vgrids, geodetic.lp, 1.0); if (proj_errno(P) == PJD_ERR_GRID_AREA) proj_log_debug(P, "deformation: coordinate (%.3f, %.3f) outside deformation model", @@ -425,7 +375,7 @@ PJ *TRANSFORMATION(deformation,1) { if( has_grids ) { - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "deformation: could not find required grid(s)."); @@ -434,13 +384,13 @@ PJ *TRANSFORMATION(deformation,1) { } else { - Q->hgrids = proj_hgrid_init(P, "xy_grids"); + Q->hgrids = pj_hgrid_init(P, "xy_grids"); if (proj_errno(P)) { proj_log_error(P, "deformation: could not find requested xy_grid(s)."); return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); } - Q->vgrids = proj_vgrid_init(P, "z_grids"); + Q->vgrids = pj_vgrid_init(P, "z_grids"); if (proj_errno(P)) { proj_log_error(P, "deformation: could not find requested z_grid(s)."); return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index 24da4dde30..122a7ab253 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -28,7 +28,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); if ( proj_errno(P) ) { return proj_coord_error().xyz; } @@ -37,7 +37,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, Q->grids, point.lp, PJ_FWD); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_FWD); } return point.xyz; @@ -51,7 +51,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); if ( proj_errno(P) ) { return proj_coord_error().lpz; } @@ -60,7 +60,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.lp = proj_hgrid_apply(P, Q->grids, point.lp, PJ_INV); + point.lp = pj_hgrid_apply(P->ctx, Q->grids, point.lp, PJ_INV); } return point.lpz; @@ -165,7 +165,7 @@ PJ *TRANSFORMATION(hgridshift,0) { Q->defer_grid_opening = true; } else { - Q->grids = proj_hgrid_init(P, "grids"); + Q->grids = pj_hgrid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "hgridshift: could not find required grid(s)."); diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index 3e7a015e74..121b795a27 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -56,7 +56,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); deal_with_vertcon_gtx_hack(P); if ( proj_errno(P) ) { return proj_coord_error().xyz; @@ -66,7 +66,7 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z += proj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); + point.xyz.z += pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.xyz; @@ -80,7 +80,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); deal_with_vertcon_gtx_hack(P); if ( proj_errno(P) ) { return proj_coord_error().lpz; @@ -90,7 +90,7 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ - point.xyz.z -= proj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); + point.xyz.z -= pj_vgrid_value(P, Q->grids, point.lp, Q->forward_multiplier); } return point.lpz; @@ -193,7 +193,7 @@ PJ *TRANSFORMATION(vgridshift,0) { } else { /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - Q->grids = proj_vgrid_init(P, "grids"); + Q->grids = pj_vgrid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index a76f32551f..e1c7651833 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -51,21 +51,6 @@ struct xyzgridshiftData { }; } // anonymous namespace - -// --------------------------------------------------------------------------- - -static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, - const PJ_LP& input) -{ - for( const auto& gridset: grids ) - { - auto grid = gridset->gridAt(input.lam, input.phi); - if( grid ) - return grid; - } - return nullptr; -} - // --------------------------------------------------------------------------- static bool get_grid_values(PJ* P, @@ -77,13 +62,14 @@ static bool get_grid_values(PJ* P, { if ( Q->defer_grid_opening ) { Q->defer_grid_opening = false; - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); if ( proj_errno(P) ) { return false; } } - auto grid = findGrid(Q->grids, lp); + GenericShiftGridSet* gridset = nullptr; + auto grid = pj_find_generic_grid(Q->grids, lp, gridset); if( !grid ) { return false; } @@ -118,56 +104,20 @@ static bool get_grid_values(PJ* P, return false; } - const auto& extent = grid->extentAndRes(); - double grid_x = (lp.lam - extent.westLon) / extent.resLon; - double grid_y = (lp.phi - extent.southLat) / extent.resLat; - int ix = static_cast(grid_x); - int iy = static_cast(grid_y); - int ix2 = std::min(ix + 1, grid->width() - 1); - int iy2 = std::min(iy + 1, grid->height() - 1); - - float dx1, dy1, dz1; - if( !grid->valueAt(ix, iy, sampleX, dx1) || - !grid->valueAt(ix, iy, sampleY, dy1) || - !grid->valueAt(ix, iy, sampleZ, dz1) ) { - return false; - } - - float dx2, dy2, dz2; - if( !grid->valueAt(ix2, iy, sampleX, dx2) || - !grid->valueAt(ix2, iy, sampleY, dy2) || - !grid->valueAt(ix2, iy, sampleZ, dz2) ) { - return false; - } - - float dx3, dy3, dz3; - if( !grid->valueAt(ix, iy2, sampleX, dx3) || - !grid->valueAt(ix, iy2, sampleY, dy3) || - !grid->valueAt(ix, iy2, sampleZ, dz3) ) { - return false; - } - - float dx4, dy4, dz4; - if( !grid->valueAt(ix2, iy2, sampleX, dx4) || - !grid->valueAt(ix2, iy2, sampleY, dy4) || - !grid->valueAt(ix2, iy2, sampleZ, dz4) ) { + bool must_retry = false; + if( !pj_bilinear_interpolation_three_samples(grid, lp, + sampleX, sampleY, sampleZ, + dx, dy, dz, + must_retry) ) + { + if( must_retry ) + return get_grid_values( P, Q, lp, dx, dy, dz); return false; } - double frct_lam = grid_x - ix; - double frct_phi = grid_y - iy; - double m10 = frct_lam; - double m11 = m10; - double m01 = 1. - frct_lam; - double m00 = m01; - m11 *= frct_phi; - m01 *= frct_phi; - frct_phi = 1. - frct_phi; - m00 *= frct_phi; - m10 *= frct_phi; - dx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) * Q->multiplier; - dy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) * Q->multiplier; - dz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) * Q->multiplier; + dx *= Q->multiplier; + dy *= Q->multiplier; + dz *= Q->multiplier; return true; } @@ -341,7 +291,7 @@ PJ *TRANSFORMATION(xyzgridshift,0) { Q->defer_grid_opening = true; } else { - Q->grids = proj_generic_grid_init(P, "grids"); + Q->grids = pj_generic_grid_init(P, "grids"); /* Was gridlist compiled properly? */ if ( proj_errno(P) ) { proj_log_error(P, "xyzgridshift: could not find required grid(s)."); diff --git a/test/cli/Makefile.am b/test/cli/Makefile.am index 758352c6c1..253b85d835 100644 --- a/test/cli/Makefile.am +++ b/test/cli/Makefile.am @@ -27,7 +27,7 @@ EXTRA_DIST = pj_out27.dist pj_out83.dist td_out.dist \ CMakeLists.txt testprojinfo-check: - PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTPROJINFO) $(PROJINFOEXE) test27-check: $(TEST27) $(PROJEXE) @@ -36,24 +36,24 @@ test83-check: $(TEST83) $(PROJEXE) testvarious-check: - PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTVARIOUS) $(CS2CSEXE) testdatumfile-check: @if [ -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat -a -f $(PROJ_LIB)/MD -a -f $(PROJ_LIB)/ntf_r93.gsb -a -f $(PROJ_LIB)/egm96_15.gtx ]; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTDATUMFILE) $(CS2CSEXE) ; \ fi testign-check: @if [ -f $(PROJ_LIB)/ntf_r93.gsb ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTIGN) $(CS2CSEXE) ; \ fi testntv2-check: @if [ -f $(PROJ_LIB)/ntv2_0.gsb -a -f $(PROJ_LIB)/conus -a -f $(PROJ_LIB)/ntv1_can.dat ] ; then \ - PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTNTV2) $(CS2CSEXE) ; \ fi testcct-check: - PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(TESTCCT) $(CCTEXE) check-local: testprojinfo-check test27-check test83-check testvarious-check testdatumfile-check testign-check testntv2-check testcct-check diff --git a/test/gie/Makefile.am b/test/gie/Makefile.am index ff333a1464..1539cb2eaf 100644 --- a/test/gie/Makefile.am +++ b/test/gie/Makefile.am @@ -15,33 +15,33 @@ EXTRA_DIST = 4D-API_cs2cs-style.gie \ PROJ_LIB ?= ../../data 4D-API-cs2cs-style: 4D-API_cs2cs-style.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< GDA: GDA.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< axisswap: axisswap.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< builtins: builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< deformation: deformation.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< ellipsoid: ellipsoid.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< more_builtins: more_builtins.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< unitconvert: unitconvert.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< DHDN_ETRS89: DHDN_ETRS89.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< geotiff_grids: geotiff_grids.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 geotiff_grids diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie index 920fcf28db..85c0cbe916 100644 --- a/test/gie/geotiff_grids.gie +++ b/test/gie/geotiff_grids.gie @@ -6,6 +6,10 @@ Test GeoTIFF grids +# Those first tests using +proj=vgridshift only test the capability of reading +# correctly a value from various formulations of GeoTIFF file, hence only the +# forward path is tested (reverse path is tested in other files) + ------------------------------------------------------------------------------- operation +proj=vgridshift +grids=tests/test_vgrid_pixelispoint.tif +multiplier=1 ------------------------------------------------------------------------------- diff --git a/test/gigs/Makefile.am b/test/gigs/Makefile.am index bea3be5966..564935502b 100644 --- a/test/gigs/Makefile.am +++ b/test/gigs/Makefile.am @@ -23,54 +23,54 @@ EXTRA_DIST = \ PROJ_LIB ?= ../../data 5101.1: 5101.1-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.2: 5101.2-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.3: 5101.3-jhs.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5101.4: 5101.4-jhs-etmerc.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5102.1: 5102.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.1: 5103.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.2: 5103.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5103.3: 5103.3.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5105.2: 5105.2.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5106: 5106.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5107: 5107.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5109: 5109.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5111.1: 5111.1.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5112: 5112.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5113: 5113.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5201: 5201.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< 5208: 5208.gie - PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< check-local: 5101.1 5101.2 5101.3 5101.4 5102.1 5103.1 5103.2 5103.3 5105.2 5106 5107 5109 5111.1 5112 5113 5201 5208 diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 35a6e4c1b7..e1eefcf057 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -71,7 +71,7 @@ target_link_libraries(proj_pj_transform_test ${PROJ_LIBRARIES}) add_test(NAME proj_pj_transform_test COMMAND proj_pj_transform_test) set_property(TEST proj_pj_transform_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_errno_string_test @@ -82,7 +82,7 @@ target_link_libraries(proj_errno_string_test ${PROJ_LIBRARIES}) add_test(NAME proj_errno_string_test COMMAND proj_errno_string_test) set_property(TEST proj_errno_string_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_angular_io_test main.cpp @@ -92,7 +92,7 @@ target_link_libraries(proj_angular_io_test ${PROJ_LIBRARIES}) add_test(NAME proj_angular_io_test COMMAND proj_angular_io_test) set_property(TEST proj_angular_io_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(proj_context_test main.cpp @@ -102,7 +102,7 @@ target_link_libraries(proj_context_test ${PROJ_LIBRARIES}) add_test(NAME proj_context_test COMMAND proj_context_test) set_property(TEST proj_context_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") if(MSVC AND BUILD_LIBPROJ_SHARED) # ph_phi2_test not compatible of a .dll build @@ -115,7 +115,7 @@ else() ${PROJ_LIBRARIES}) add_test(NAME pj_phi2_test COMMAND pj_phi2_test) set_property(TEST pj_phi2_test - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") endif() add_executable(proj_test_cpp_api @@ -135,7 +135,7 @@ target_link_libraries(proj_test_cpp_api ${SQLITE3_LIBRARY}) add_test(NAME proj_test_cpp_api COMMAND proj_test_cpp_api) set_property(TEST proj_test_cpp_api - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(gie_self_tests @@ -146,7 +146,7 @@ target_link_libraries(gie_self_tests ${PROJ_LIBRARIES}) add_test(NAME gie_self_tests COMMAND gie_self_tests) set_property(TEST gie_self_tests - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data") add_executable(test_network @@ -163,8 +163,8 @@ target_link_libraries(test_network add_test(NAME test_network COMMAND test_network) if(MSVC) set_property(TEST test_network - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") else() set_property(TEST test_network - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") + PROPERTY ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") endif() diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 422fe6877e..7ffb06aecc 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -23,7 +23,7 @@ pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ pj_transform_test-check: pj_transform_test - PROJ_LIB=$(PROJ_LIB) ./pj_transform_test + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./pj_transform_test pj_phi2_test_SOURCES = pj_phi2_test.cpp main.cpp pj_phi2_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ @@ -47,19 +47,19 @@ proj_context_test_SOURCES = proj_context_test.cpp main.cpp proj_context_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ proj_context_test-check: proj_context_test - ./proj_context_test + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES ./proj_context_test test_cpp_api_SOURCES = test_util.cpp test_common.cpp test_crs.cpp test_metadata.cpp test_io.cpp test_operation.cpp test_datum.cpp test_factory.cpp test_c_api.cpp main.cpp test_cpp_api_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ test_cpp_api-check: test_cpp_api - PROJ_LIB=$(PROJ_LIB) ./test_cpp_api + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./test_cpp_api gie_self_tests_SOURCES = gie_self_tests.cpp main.cpp gie_self_tests_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ gie_self_tests-check: gie_self_tests - PROJ_LIB=$(PROJ_LIB) ./gie_self_tests + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) ./gie_self_tests include_proj_h_from_c_SOURCES = include_proj_h_from_c.c @@ -68,6 +68,6 @@ test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @SQLITE3_LIBS@ @CURL_LIBS@ test_network-check: test_network - PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/gie_self_tests.cpp b/test/unit/gie_self_tests.cpp index a738db7552..fc0f074846 100644 --- a/test/unit/gie_self_tests.cpp +++ b/test/unit/gie_self_tests.cpp @@ -348,7 +348,9 @@ TEST(gie, info_functions) { /* proj_info() */ /* this one is difficult to test, since the output changes with the setup */ + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); info = proj_info(); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); if (info.version[0] != '\0') { char tmpstr[64]; @@ -360,6 +362,9 @@ TEST(gie, info_functions) { ASSERT_NE(std::string(info.searchpath), std::string()); } + ASSERT_TRUE(std::string(info.searchpath).find("/proj") != + std::string::npos); + /* proj_pj_info() */ { P = proj_create(PJ_DEFAULT_CTX, diff --git a/test/unit/proj_context_test.cpp b/test/unit/proj_context_test.cpp index 23c46f29c7..ec59590d78 100644 --- a/test/unit/proj_context_test.cpp +++ b/test/unit/proj_context_test.cpp @@ -40,7 +40,20 @@ namespace { -static std::string createTempDict(std::string &dirname) { +static bool createTmpFile(const std::string &filename) { + FILE *f = fopen(filename.c_str(), "wt"); + if (!f) + return false; + fprintf( + f, + " +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); + fclose(f); + return true; +} + +// --------------------------------------------------------------------------- + +static std::string createTempDict(std::string &dirname, const char *filename) { const char *temp_dir = getenv("TEMP"); if (!temp_dir) { temp_dir = getenv("TMP"); @@ -58,16 +71,9 @@ static std::string createTempDict(std::string &dirname) { std::string tmpFilename; tmpFilename = temp_dir; tmpFilename += DIR_CHAR; - tmpFilename += "temp_proj_dic"; + tmpFilename += filename; - FILE *f = fopen(tmpFilename.c_str(), "wt"); - if (!f) - return std::string(); - fprintf( - f, - " +proj=pipeline +step +proj=utm +zone=31 +ellps=GRS80\n"); - fclose(f); - return tmpFilename; + return createTmpFile(tmpFilename) ? tmpFilename : std::string(); } // --------------------------------------------------------------------------- @@ -85,7 +91,7 @@ static int MyUnlink(const std::string &filename) { TEST(proj_context, proj_context_set_file_finder) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic1"); if (filename.empty()) return; @@ -111,7 +117,7 @@ TEST(proj_context, proj_context_set_file_finder) { finderData.dirname = dirname; proj_context_set_file_finder(ctx, finder, &finderData); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic1:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -125,7 +131,7 @@ TEST(proj_context, proj_context_set_file_finder) { TEST(proj_context, proj_context_set_search_paths) { std::string dirname; - auto filename = createTempDict(dirname); + auto filename = createTempDict(dirname, "temp_proj_dic2"); if (filename.empty()) return; @@ -134,7 +140,7 @@ TEST(proj_context, proj_context_set_search_paths) { const char *path = dirname.c_str(); proj_context_set_search_paths(ctx, 1, &path); - auto P = proj_create(ctx, "+init=temp_proj_dic:MY_PIPELINE"); + auto P = proj_create(ctx, "+init=temp_proj_dic2:MY_PIPELINE"); EXPECT_NE(P, nullptr); proj_destroy(P); @@ -143,4 +149,32 @@ TEST(proj_context, proj_context_set_search_paths) { MyUnlink(filename); } +// --------------------------------------------------------------------------- + +TEST(proj_context, read_grid_from_user_writable_directory) { + + auto ctx = proj_context_create(); + auto path = pj_context_get_user_writable_directory(ctx, true); + EXPECT_TRUE(!path.empty()); + auto filename = path + DIR_CHAR + "temp_proj_dic3"; + EXPECT_TRUE(createTmpFile(filename)); + { + // Check that with PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES (set by + // calling script), we cannot find the file + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_EQ(P, nullptr); + proj_destroy(P); + } + { + // Cancel the effect of PROJ_IGNORE_USER_WRITABLE_DIRECTORY + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); + auto P = proj_create(ctx, "+init=temp_proj_dic3:MY_PIPELINE"); + EXPECT_NE(P, nullptr); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); + proj_destroy(P); + } + proj_context_destroy(ctx); + MyUnlink(filename); +} + } // namespace diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 2ec38e41df..4e66d8c516 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -38,6 +38,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #ifdef CURL_ENABLED #include #endif @@ -473,7 +479,30 @@ TEST(networking, custom) { } exchange.events.emplace_back(std::move(event)); } - + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; double lat = 49 / 180. * M_PI; @@ -500,6 +529,30 @@ TEST(networking, custom) { } exchange.events.emplace_back(std::move(event)); } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; @@ -743,6 +796,30 @@ TEST(networking, simul_read_range_error) { } exchange.events.emplace_back(std::move(event)); } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } { double lon = 2 / 180. * M_PI; @@ -793,6 +870,188 @@ TEST(networking, simul_read_range_error) { // --------------------------------------------------------------------------- +TEST(networking, simul_file_change_while_opened) { + auto ctx = proj_context_create(); + proj_grid_cache_set_enable(ctx, false); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/file_change_while_opened.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/file_change_while_opened.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 3; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes=0-16383/10000000"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Last-Modified"; + event->value = "some_date CHANGED!!!!"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "ETag"; + event->value = "some_etag"; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 3; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + #ifdef CURL_ENABLED TEST(networking, curl_hgridshift) { @@ -1305,6 +1564,131 @@ TEST(networking, cache_lock) { proj_context_destroy(ctx); } + +// --------------------------------------------------------------------------- + +TEST(networking, download_whole_files) { + if (!networkAccessOK) { + return; + } + + proj_cleanup(); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/ntf_r93.tif"); + rmdir("proj_test_tmp"); + + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast("PROJ_USER_WRITABLE_DIRECTORY=./proj_test_tmp")); + putenv(const_cast("PROJ_FULL_FILE_CHUNK_SIZE=30000")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + ASSERT_TRUE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + ASSERT_TRUE( + proj_download_file(ctx, "ntf_r93.gsb", false, nullptr, nullptr)); + + FILE *f = fopen("proj_test_tmp/ntf_r93.tif", "rb"); + ASSERT_NE(f, nullptr); + fseek(f, 0, SEEK_END); + ASSERT_EQ(ftell(f), 93581); + fclose(f); + + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Force lastChecked to the Epoch so that data is expired. + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0", -1, + &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // If we ignore TTL settings, then no network access will be done + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", true)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + // Check that the lastChecked timestamp is still 0 + sqlite3_stmt *hStmt = nullptr; + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + // Should recheck from the CDN, update last_checked and do nothing + ASSERT_FALSE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + { + sqlite3 *hDB = nullptr; + sqlite3_open_v2("proj_test_tmp/cache.db", &hDB, SQLITE_OPEN_READWRITE, + nullptr); + ASSERT_NE(hDB, nullptr); + sqlite3_stmt *hStmt = nullptr; + // Check that the lastChecked timestamp has been updated + sqlite3_prepare_v2(hDB, + "SELECT lastChecked FROM downloaded_file_properties", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_ROW); + ASSERT_NE(sqlite3_column_int64(hStmt, 0), 0); + sqlite3_finalize(hStmt); + hStmt = nullptr; + + // Now invalid lastModified. This should trigger a new download + sqlite3_prepare_v2( + hDB, "UPDATE downloaded_file_properties SET lastChecked = 0, " + "lastModified = 'foo'", + -1, &hStmt, nullptr); + ASSERT_NE(hStmt, nullptr); + ASSERT_EQ(sqlite3_step(hStmt), SQLITE_DONE); + sqlite3_finalize(hStmt); + sqlite3_close(hDB); + } + + ASSERT_TRUE(proj_is_download_needed(ctx, "ntf_r93.gsb", false)); + + // Redo download with a progress callback this time. + unlink("proj_test_tmp/ntf_r93.tif"); + + const auto cbk = [](PJ_CONTEXT *l_ctx, double pct, void *user_data) -> int { + auto vect = static_cast> *>( + user_data); + vect->push_back(std::pair(l_ctx, pct)); + return true; + }; + + std::vector> vectPct; + ASSERT_TRUE(proj_download_file(ctx, "ntf_r93.gsb", false, cbk, &vectPct)); + ASSERT_EQ(vectPct.size(), 3U); + ASSERT_EQ(vectPct.back().first, ctx); + ASSERT_EQ(vectPct.back().second, 1.0); + + proj_context_destroy(ctx); + putenv(const_cast("PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES")); + putenv(const_cast("PROJ_USER_WRITABLE_DIRECTORY=")); + putenv(const_cast("PROJ_FULL_FILE_CHUNK_SIZE=")); + unlink("proj_test_tmp/cache.db"); + unlink("proj_test_tmp/ntf_r93.tif"); + rmdir("proj_test_tmp"); +} + #endif } // namespace