Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SEDONA-193] Add Sedona NetCDF reader suite #1134

Merged
merged 35 commits into from
Dec 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
8b443d9
temp changes for netcdf
iGN5117 Aug 30, 2023
4f3dfd7
add test files
iGN5117 Aug 31, 2023
0d0b606
netcdf progress push
iGN5117 Sep 8, 2023
4fdd806
Add Sedona NetCDF suite core logic
iGN5117 Nov 30, 2023
94c17a6
Merge branch 'sedona_master' into develop_Nilesh_NetCDF
iGN5117 Nov 30, 2023
a6c5cbb
fix lint error
iGN5117 Nov 30, 2023
01349b2
add ucar to pom
iGN5117 Nov 30, 2023
7b4ef6d
add missing import
iGN5117 Nov 30, 2023
9618169
add another create function with properties
iGN5117 Nov 30, 2023
ee54841
Remove test for now
iGN5117 Nov 30, 2023
e127542
Remove netcdf test stuff from testbase for now
iGN5117 Nov 30, 2023
aa77c29
rename constant for global attributes
iGN5117 Dec 5, 2023
da77808
Fix bugs in netcdf
iGN5117 Dec 5, 2023
82bbf43
pulled from remote master
iGN5117 Dec 5, 2023
2c6388a
add sanity tests for testNetCDF and testRecordInfo
iGN5117 Dec 5, 2023
79e01db
Commented test cases for big HDF5 file, and cleaned up test code
iGN5117 Dec 5, 2023
22e83b0
Add test netcdf file
iGN5117 Dec 5, 2023
bdbfd3b
Change signature a bit
iGN5117 Dec 5, 2023
d376116
Add long form test case
iGN5117 Dec 5, 2023
69af41c
add scala test cases
iGN5117 Dec 5, 2023
43fc9b9
add scala test cases (2)
iGN5117 Dec 5, 2023
afb3fd6
Add documentation
iGN5117 Dec 5, 2023
d325a4e
Merge branch 'apache:master' into develop_Nilesh_NetCDF
iGN5117 Dec 5, 2023
7ba4f98
Renamed files to remove 'Sedona' from their name
iGN5117 Dec 6, 2023
70d7c77
Removed geotools netcdf reader dependency in pom
iGN5117 Dec 6, 2023
6c724b7
Removed explicit scope declaration for ucar in pom
iGN5117 Dec 6, 2023
382039c
Fixed accidental incorrect API name in docs
iGN5117 Dec 6, 2023
51ea10f
Removed HDF5 test and cleaned up comments
iGN5117 Dec 6, 2023
085d4f4
Part of renaming
iGN5117 Dec 6, 2023
b847040
Changed file to byte implementation and cleaned up comments
iGN5117 Dec 6, 2023
25ecd65
Optimized imports in SerNetCDFUtils
iGN5117 Dec 6, 2023
62ed3cb
Merge branch 'develop_Nilesh_NetCDF' of https://github.com/iGN5117/se…
iGN5117 Dec 6, 2023
432d315
Fixed EOF linting
iGN5117 Dec 6, 2023
12f0281
Add apache license and try to EOF linting error
iGN5117 Dec 7, 2023
ba22591
fix EOF lint in md file
iGN5117 Dec 7, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@
<version>${janino-version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>edu.ucar</groupId>
<artifactId>cdm-core</artifactId>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import org.apache.sedona.common.FunctionsGeoTools;
import org.apache.sedona.common.raster.inputstream.ByteArrayImageInputStream;
import org.apache.sedona.common.raster.netcdf.NetCdfReader;
import org.apache.sedona.common.utils.RasterUtils;
import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.coverage.grid.GridEnvelope2D;
Expand All @@ -28,6 +29,7 @@
import org.geotools.geometry.jts.JTS;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.geotools.process.vector.VectorToRasterProcess;
import org.geotools.referencing.CRS;
import org.geotools.referencing.crs.DefaultEngineeringCRS;
import org.geotools.referencing.operation.transform.AffineTransform2D;
import org.geotools.util.factory.Hints;
Expand All @@ -38,13 +40,15 @@
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.opengis.referencing.datum.PixelInCell;
import org.opengis.referencing.operation.MathTransform;
import ucar.nc2.NetcdfFile;
import ucar.nc2.NetcdfFiles;

import javax.media.jai.RasterFactory;
import java.awt.image.WritableRaster;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

public class RasterConstructors
{
Expand All @@ -58,6 +62,25 @@ public static GridCoverage2D fromGeoTiff(byte[] bytes) throws IOException {
return geoTiffReader.read(null);
}

public static GridCoverage2D fromNetCDF(byte[] bytes, String variableName, String lonDimensionName, String latDimensionName) throws IOException, FactoryException {
NetcdfFile netcdfFile = openNetCdfBytes(bytes);
return NetCdfReader.getRaster(netcdfFile, variableName, latDimensionName, lonDimensionName);
}

public static GridCoverage2D fromNetCDF(byte[] bytes, String recordVariableName) throws IOException, FactoryException {
NetcdfFile netcdfFile = openNetCdfBytes(bytes);
return NetCdfReader.getRaster(netcdfFile, recordVariableName);
}

public static String getRecordInfo(byte[] bytes) throws IOException {
NetcdfFile netcdfFile = openNetCdfBytes(bytes);
return NetCdfReader.getRecordInfo(netcdfFile);
}

private static NetcdfFile openNetCdfBytes(byte[] bytes) throws IOException {
return NetcdfFiles.openInMemory("", bytes);
}

/**
* Returns a raster that is converted from the geometry provided.
* @param geom The geometry to convert
Expand Down Expand Up @@ -310,7 +333,31 @@ public static GridCoverage2D makeEmptyRaster(int numBand, String bandDataType, i
new GridEnvelope2D(0, 0, widthInPixel, heightInPixel),
PixelInCell.CELL_CORNER,
transform, crs, null);
return RasterUtils.create(raster, gridGeometry, null);
return RasterUtils.create(raster, gridGeometry, null, null);
}

public static GridCoverage2D makeNonEmptyRaster(int numBand, int widthInPixel, int heightInPixel, double upperLeftX, double upperLeftY, double scaleX, double scaleY, double skewX, double skewY, int srid, double[][] data, Map<String, List<String>> properties, Double noDataValue, PixelInCell anchor) throws FactoryException {
CoordinateReferenceSystem crs;
if (srid == 0) {
crs = DefaultEngineeringCRS.GENERIC_2D;
} else {
// Create the CRS from the srid
// Longitude first, Latitude second
crs = CRS.decode("EPSG:" + srid, true);
}

// Create a new empty raster
WritableRaster raster = RasterFactory.createBandedRaster(5, widthInPixel, heightInPixel, numBand, null); //create a raster with double values
for (int i = 0; i < numBand; i++) {
raster.setSamples(0, 0, widthInPixel, heightInPixel, i, data[i]);
}
//raster.setPixels(0, 0, widthInPixel, heightInPixel, data);
MathTransform transform = new AffineTransform2D(scaleX, skewY, skewX, scaleY, upperLeftX, upperLeftY);
GridGeometry2D gridGeometry = new GridGeometry2D(
new GridEnvelope2D(0, 0, widthInPixel, heightInPixel),
anchor,
transform, crs, null);
return RasterUtils.create(raster, gridGeometry, null, noDataValue, properties);
}

public static GridCoverage2D makeNonEmptyRaster(int numBands, String bandDataType, int widthInPixel, int heightInPixel, double upperLeftX, double upperLeftY, double scaleX, double scaleY, double skewX, double skewY, int srid, double[][] rasterValues) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.sedona.common.raster.netcdf;

public class NetCdfConstants {
public static final String VALID_MIN = "valid_min";
public static final String VALID_MAX = "valid_max";
public static final String MISSING_VALUE = "missing_value";
public static final String VALID_RANGE = "valid_range";
public static final String SCALE_FACTOR = "scale_factor";
public static final String ADD_OFFSET = "add_offset";
public static final String LONG_NAME = "long_name";
public static final String UNITS = "units";
public static final String GLOBAL_ATTR_PREFIX = "GLOBAL_ATTRIBUTES";
public static final String VAR_ATTR_PREFIX = "VAR_";
public static final String BAND_ATTR_PREFIX = "BAND_";

public static final String INSUFFICIENT_DIMENSIONS_VARIABLE = "Provided variable has less than two dimensions";

public static final String NON_NUMERIC_VALUE = "An attribute expected to have numeric values does not have numeric value";

public static final String INVALID_LON_NAME = "Provided longitude variable short name is invalid";

public static final String INVALID_LAT_NAME = "Provided latitude variable short name is invalid";

public static final String INVALID_LAT_SHAPE = "Shape of latitude variable is supposed to be 1";

public static final String INVALID_LON_SHAPE = "Shape of longitude variable is supposed to be 1";

public static final String INVALID_RECORD_NAME = "Provided record variable short name is invalid";

public static final String COORD_VARIABLE_NOT_FOUND = "Corresponding coordinate variable not found for dimension %s";

public static final String COORD_IDX_NOT_FOUND = "Given record variable does not contain one of the latitude or longitude dimensions as the participating dimensions";
}
Loading
Loading