Method for image transformation matrix generation (more evualtion needed)

scripts/builtin/img_transform_linearized.dml

@@ -85,7 +85,8 @@ m_img_transform_linearized = function(Matrix[Double] img_in, Integer out_w, Inte
     ind= matrix(seq(1,ncol(xs),1),1,ncol(xs))
     z = table(xs, ind) 
     output = ys%*%z
-
+    print(nrow(img_in))
+    print(ncol(img_in))
     img_out = matrix(output, rows=nrow(img_in), cols=out_w*out_h)
   }
 }

scripts/builtin/img_transform_matrix.dml

@@ -0,0 +1,77 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Generates the z matrix for the new linearized image transformation function in order to apply
+# affine transformation to linearized images.
+#
+# INPUT:
+# -------------------------------------------------------------------------------------------
+# transMat      3x3 matrix for affine transformation (transformation Matrix)
+# dimMat        2x2 matrix containing the original size of the image in the first row as follows
+#               [1,1] original height; [1,2] original width
+#               and the target size of the new image in the second row as follows
+#               [2,1] target height; [2,2] target width
+#               (dimensionMatrix)
+# -------------------------------------------------------------------------------------------
+# OUTPUT:
+# ---------------------------------------------------------------------------------------
+# zMat          transformation matrix to be multiplied with for the linearized image
+#               transformation function (arbitrary naming)
+# isFillable    returns a boolean which indicates if cells need to be filled (with fillvalue),
+                in this case the image is extended, otherwise the original image is used
+# ---------------------------------------------------------------------------------------
+
+ m_img_transform_matrix = function(Matrix[Double] transMat, Integer orig_w, Integer orig_h, Integer out_w, Integer out_h)
+            return (Matrix[Double] zMat, Matrix[Double] isFillable){
+         #orig_w = as.scalar(dimMat[1,2])
+         #orig_h = as.scalar(dimMat[1,1])
+
+         #out_w = as.scalar(dimMat[2,2])
+         #out_h = as.scalar(dimMat[2,1])
+         T_inv = inv(transMat)
+
+         ## coordinates of output pixel-centers linearized in row-major order
+         coords = matrix(1, rows=3, cols=out_w*out_h)
+         coords[1,] = t((seq(0, out_w*out_h-1) %% out_w) + 0.5)
+         coords[2,] = t((seq(0, out_w*out_h-1) %/% out_w) + 0.5)
+
+         # compute sampling pixel indices
+         coords = floor(T_inv %*% coords) + 1
+
+         inx = t(coords[1,])
+         iny = t(coords[2,])
+
+         # any out-of-range pixels, if present, correspond to an extra pixel with fill_value at the end of the input
+         index_vector = (orig_w *(iny-1) + inx) * ((0<inx) & (inx<=orig_w) & (0<iny) & (iny<=orig_h))
+         index_vector = t(index_vector)
+         xs = ((index_vector == 0)*(orig_w*orig_h +1)) + index_vector
+
+         #if(min(index_vector) == 0){
+         #  ys=cbind(img_in, matrix(fill_value,nrow(img_in), 1))
+         #}else{
+         #  ys = img_in
+         #}
+
+         ind= matrix(seq(1,ncol(xs),1),1,ncol(xs))
+         z = table(xs, ind)
+         zMat = transMat
+         isFillable = as.logical(as.double(min(index_vector) == 0))
+ }

scripts/builtin/img_transform_test.dml

@@ -0,0 +1,52 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+m_img_transform_test = function(Matrix[Double] transMat, Matrix[Double] dimensionMatrix)
+            return (Matrix[Double] zMat, Matrix[Double] isFillable){
+            #zMat = transMat
+            #isFillable = FALSE
+            T_inv = inv(transMat)
+
+            orig_w = as.scalar(dimensionMatrix[1,1])
+            orig_h = as.scalar(dimensionMatrix[1,2])
+            out_w = as.scalar(dimensionMatrix[2,1])
+            out_h = as.scalar(dimensionMatrix[2,2])
+
+            ## coordinates of output pixel-centers linearized in row-major order
+            coords = matrix(1, rows=3, cols=out_w*out_h)
+            coords[1,] = t((seq(0, out_w*out_h-1) %% out_w) + 0.5)
+            coords[2,] = t((seq(0, out_w*out_h-1) %/% out_w) + 0.5)
+
+            # compute sampling pixel indices
+            coords = floor(T_inv %*% coords) + 1
+
+            inx = t(coords[1,])
+            iny = t(coords[2,])
+
+            # any out-of-range pixels, if present, correspond to an extra pixel with fill_value at the end of the input
+            index_vector = (orig_w *(iny-1) + inx) * ((0<inx) & (inx<=orig_w) & (0<iny) & (iny<=orig_h))
+            index_vector = t(index_vector)
+            xs = ((index_vector == 0)*(orig_w*orig_h +1)) + index_vector
+
+            ind= matrix(seq(1,ncol(xs),1),1,ncol(xs))
+            zMat = table(xs, ind)
+            isFillable = as.matrix(min(index_vector) == 0)
+}

src/main/java/org/apache/sysds/common/Builtins.java

@@ -168,6 +168,8 @@ public enum Builtins {
 	IMG_CROP_LINEARIZED("img_crop_linearized", true),
 	IMG_TRANSFORM("img_transform", true),
 	IMG_TRANSFORM_LINEARIZED("img_transform_linearized", true),
+	IMG_TRANSFORM_MATRIX("img_transform_matrix", false, ReturnType.MULTI_RETURN),
+	IMG_TRANSFORM_TEST("img_transform_test", true),
 	IMG_TRANSLATE("img_translate", true),
 	IMG_TRANSLATE_LINEARIZED("img_translate_linearized", true),
 	IMG_ROTATE("img_rotate", true),

src/main/java/org/apache/sysds/hops/FunctionOp.java

@@ -183,43 +183,41 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
 		if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
 			throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
 		else {
-			if ( getFunctionName().equalsIgnoreCase("qr") ) {
+			if (getFunctionName().equalsIgnoreCase("qr")) {
 				// upper-triangular and lower-triangular matrices
 				long outputH = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
 				long outputR = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
-				return outputH+outputR; 
-			}
-			else if ( getFunctionName().equalsIgnoreCase("lu") ) {
+				return outputH + outputR;
+			} else if (getFunctionName().equalsIgnoreCase("lu")) {
 				// upper-triangular and lower-triangular matrices
-				long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0/getOutputs().get(1).getDim2());
+				long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0 / getOutputs().get(1).getDim2());
 				long outputL = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
 				long outputU = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
-				return outputL+outputU+outputP; 
-			}
-			else if ( getFunctionName().equalsIgnoreCase("eigen") ) {
+				return outputL + outputU + outputP;
+			} else if (getFunctionName().equalsIgnoreCase("eigen")) {
 				long outputVectors = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputValues = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), 1, 1.0);
-				return outputVectors+outputValues; 
-			}
-			else if ( getFunctionName().equalsIgnoreCase("fft") ) {
+				return outputVectors + outputValues;
+			} else if (getFunctionName().equalsIgnoreCase("fft")) {
 				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputIm = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0);
-				return outputRe+outputIm;
-			}
-			else if ( getFunctionName().equalsIgnoreCase("ifft") ) {
+				return outputRe + outputIm;
+			} else if (getFunctionName().equalsIgnoreCase("ifft")) {
 				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputIm = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0);
-				return outputRe+outputIm;
-			}
-			else if ( getFunctionName().equalsIgnoreCase("fft_linearized") ) {
+				return outputRe + outputIm;
+			} else if (getFunctionName().equalsIgnoreCase("fft_linearized")) {
 				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputIm = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0);
-				return outputRe+outputIm;
-			}
-			else if ( getFunctionName().equalsIgnoreCase("ifft_linearized") ) {
+				return outputRe + outputIm;
+			} else if (getFunctionName().equalsIgnoreCase("ifft_linearized")) {
 				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputIm = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0);
-				return outputRe+outputIm;
+				return outputRe + outputIm;
+			} else if ( getFunctionName().equalsIgnoreCase("img_transform_matrix")) {
+				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
+				long outputIm = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0);
+				return outputRe + outputIm;
 			}
 			else if ( getFunctionName().equalsIgnoreCase("stft") ) {
 				long outputRe = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
@@ -294,9 +292,8 @@ else if ( getFunctionName().equalsIgnoreCase("fft_linearized") ) {
 			}
 			else if ( getFunctionName().equalsIgnoreCase("ifft_linearized") ) {
 				// 2 matrices of size same as the input
-				return 2*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
-			}
-			else if ( getFunctionName().equalsIgnoreCase("stft") ) {
+				return 2 * OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
+			} else if ( getFunctionName().equalsIgnoreCase("stft") ) {
 				// 2 matrices of size same as the input
 				return 2*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
 			}

src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java

@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 
+import javassist.expr.Expr;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.NotImplementedException;
@@ -672,6 +673,31 @@ else if(((ConstIdentifier) getThirdExpr().getOutput())
 
 			break;
 		}
+		//implement the calculation of the transformation matrix for affine transformation of images
+			case IMG_TRANSFORM_MATRIX:
+				//transformation matrix must be 3x3 matrix
+				Expression expressionOne_IMG = getFirstExpr();
+				//dimension matrix must be a 2x2 matrix
+				Expression expressionTwo_IMG = getSecondExpr();
+				checkMatrixFrameParam(expressionOne_IMG);
+				checkMatrixFrameParam(expressionTwo_IMG);
+
+				if ((expressionOne_IMG.getOutput().getDim1() != expressionOne_IMG.getOutput().getDim2()) && expressionOne_IMG.getOutput().getDim1() != 3) {
+					raiseValidateError("The first argument to " + _opcode + " must be a square 3x3 matrix.", false, LanguageErrorCodes.INVALID_PARAMETERS);
+				} else if ((expressionTwo_IMG.getOutput().getDim1() != expressionTwo_IMG.getOutput().getDim2()) && expressionOne_IMG.getOutput().getDim1() != 2) {
+					raiseValidateError("The second argument to " + _opcode + " must be a square 2x2 matrix.", false, LanguageErrorCodes.INVALID_PARAMETERS);
+				}
+
+				DataIdentifier img_transfrom_matrix_Out1 = (DataIdentifier) getOutputs()[0];
+				DataIdentifier img_transfrom_matrix_Out2 = (DataIdentifier) getOutputs()[1];
+
+				//describe the matrix characteristics type and value
+				img_transfrom_matrix_Out1.setDataType(DataType.MATRIX);
+				img_transfrom_matrix_Out1.setValueType(ValueType.FP64);
+				img_transfrom_matrix_Out2.setDataType(DataType.MATRIX);
+				img_transfrom_matrix_Out2.setValueType(ValueType.FP64);
+				//the output dimensions are not known beforehand and vary based on input values
+				break;
 		case REMOVE: {
 			checkNumParameters(2);
 			checkListParam(getFirstExpr());
@@ -1325,7 +1351,6 @@ else if( getOpCode() == Builtins.RBIND ) {
 			output.setBlocksize(0);
 			output.setValueType(ValueType.BOOLEAN);
 			break;
-
 		// Contingency tables
 		case TABLE:
 

src/main/java/org/apache/sysds/parser/DMLTranslator.java

@@ -2240,6 +2240,7 @@ private Hop processMultipleReturnBuiltinFunctionExpression(BuiltinFunctionExpres
 			case IFFT:
 			case FFT_LINEARIZED:
 			case IFFT_LINEARIZED:
+			case IMG_TRANSFORM_MATRIX:
 			case STFT:
 			case LSTM:
 			case LSTM_BACKWARD:
@@ -2490,7 +2491,6 @@ else if ( sop.equalsIgnoreCase("!=") )
 				new NaryOp(target.getName(), target.getDataType(), target.getValueType(), appendOpN,
 					processAllExpressions(source.getAllExpr(), hops));
 			break;
-
 		case TABLE:
 
 			// Always a TertiaryOp is created for table().

src/main/java/org/apache/sysds/runtime/instructions/CPInstructionParser.java

@@ -338,6 +338,7 @@ public class CPInstructionParser extends InstructionParser {
 		String2CPInstructionType.put( "ifft",  CPType.MultiReturnComplexMatrixBuiltin);
 		String2CPInstructionType.put( "fft_linearized", CPType.MultiReturnBuiltin);
 		String2CPInstructionType.put( "ifft_linearized", CPType.MultiReturnComplexMatrixBuiltin);
+		String2CPInstructionType.put( "img_transform_matrix", CPType.MultiReturnComplexMatrixBuiltin);
 		String2CPInstructionType.put( "stft", CPType.MultiReturnComplexMatrixBuiltin);
 		String2CPInstructionType.put( "svd",   CPType.MultiReturnBuiltin);
 		String2CPInstructionType.put( "rcm",   CPType.MultiReturnComplexMatrixBuiltin);

src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnBuiltinCPInstruction.java

@@ -138,8 +138,8 @@ else if(parts.length == 3 && opcode.equalsIgnoreCase("fft_linearized")) {
 
 			return new MultiReturnBuiltinCPInstruction(null, null, outputs, opcode, str, threads);
 
-		}
-		else if ( opcode.equalsIgnoreCase("stft") ) {
+
+		} else if ( opcode.equalsIgnoreCase("stft") ) {
 			// one input and two outputs
 			CPOperand in1 = new CPOperand(parts[1]);
 			outputs.add ( new CPOperand(parts[2], ValueType.FP64, DataType.MATRIX) );

src/main/java/org/apache/sysds/runtime/instructions/cp/MultiReturnComplexMatrixBuiltinCPInstruction.java

@@ -138,6 +138,16 @@ else if(parts.length == 8 && opcode.equalsIgnoreCase("stft")) {
 
 			return new MultiReturnComplexMatrixBuiltinCPInstruction(null, in1, in2, windowSize, overlap, outputs, opcode,
 				str, threads);
+		} else if (opcode.equalsIgnoreCase("img_transform_matrix")) {
+			// 2 inputs and two outputs
+			CPOperand in1 = new CPOperand(parts[1]); //transformation matrix
+			CPOperand in2 = new CPOperand(parts[2]); //dimension matrix [[orig_w, orig_h],[out_w, out_h]]
+
+			outputs.add(new CPOperand(parts[3], ValueType.FP64, DataType.MATRIX));
+			outputs.add(new CPOperand(parts[4], ValueType.FP64, DataType.MATRIX));
+			int threads = Integer.parseInt(parts[5]);
+			//throw new NotImplementedException("Has yet to be done. Check number of inputs" + Arrays.toString(parts) + "; Number of parts: " + parts.length);
+			return new MultiReturnComplexMatrixBuiltinCPInstruction(null, in1, in2, outputs, opcode, str, threads);
 		}
 		else if ( opcode.equalsIgnoreCase("rcm") ) {
 			CPOperand in1 = new CPOperand(parts[1]);

src/main/java/org/apache/sysds/runtime/matrix/data/LibCommonsMath.java

@@ -23,11 +23,13 @@
 import static org.apache.sysds.runtime.matrix.data.LibMatrixFourier.fft_linearized;
 import static org.apache.sysds.runtime.matrix.data.LibMatrixFourier.ifft;
 import static org.apache.sysds.runtime.matrix.data.LibMatrixFourier.ifft_linearized;
+import static org.apache.sysds.runtime.matrix.data.LibMatrixIMGTransform.transformationMatrix;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.math3.exception.MaxCountExceededException;
@@ -91,6 +93,7 @@ public static boolean isSupportedMultiReturnOperation( String opcode ) {
 			case "fft_linearized":
 			case "ifft":
 			case "ifft_linearized":
+			case "img_transform_matrix":
 			case "lu":
 			case "qr":
 			case "rcm":
@@ -169,6 +172,9 @@ public static MatrixBlock[] multiReturnOperations(MatrixBlock in1, MatrixBlock i
 				return computeIFFT_LINEARIZED(in1, in2, threads);
 			case "rcm":
 				return computeRCM(in1, in2);
+			case "img_transform_matrix":
+				//throw new NotImplementedException("Hope we get here");
+				return transformationMatrix(in1, in2, threads);
 			default:
 				return null;
 		}