Skip to content

Commit

Permalink
Fix overflows in two functions in src/reformat.c
Browse files Browse the repository at this point in the history
Fix overflows when multiplying with rowBytes in avifImageRGBToYUV() and
avifImageYUVAnyToRGBAnySlow(), by storing the various uint32_t rowBytes
fields in local variables of the size_t type. Then multiplications with
the size_t rowBytes local variables will be done in size_t.

Part of the fix to AOMediaCodec#2271.
  • Loading branch information
wantehchang committed Aug 1, 2024
1 parent 1f86eb6 commit 40e49c5
Showing 1 changed file with 6 additions and 6 deletions.
12 changes: 6 additions & 6 deletions src/reformat.c
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
// Convert an entire 2x2 block to YUV, and populate any fully sampled channels as we go
for (int bJ = 0; bJ < blockH; ++bJ) {
for (int bI = 0; bI < blockW; ++bI) {
int i = outerI + bI;
int j = outerJ + bJ;
size_t i = outerI + bI;
size_t j = outerJ + bJ;

// Unpack RGB into normalized float
if (state.rgb.channelBytes > 1) {
Expand Down Expand Up @@ -420,8 +420,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)

const int chromaShiftX = 1;
const int chromaShiftY = 1;
int uvI = outerI >> chromaShiftX;
int uvJ = outerJ >> chromaShiftY;
size_t uvI = outerI >> chromaShiftX;
size_t uvJ = outerJ >> chromaShiftY;
if (state.yuv.channelBytes > 1) {
uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])];
*pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
Expand All @@ -448,8 +448,8 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb)
float avgV = sumV / totalSamples;

const int chromaShiftX = 1;
int uvI = outerI >> chromaShiftX;
int uvJ = outerJ + bJ;
size_t uvI = outerI >> chromaShiftX;
size_t uvJ = outerJ + bJ;
if (state.yuv.channelBytes > 1) {
uint16_t * pU = (uint16_t *)&yuvPlanes[AVIF_CHAN_U][(uvI * 2) + (uvJ * yuvRowBytes[AVIF_CHAN_U])];
*pU = (uint16_t)avifYUVColorSpaceInfoUVToUNorm(&state.yuv, avgU);
Expand Down

0 comments on commit 40e49c5

Please sign in to comment.