[freetype2] master 5b90440: * src/base/ftcalc.c (FT_Matrix_Check): Fix integer overflow.

[Werner LEMBERG]

branch: master
commit 5b904409fc3ee6de45b60df722f95c6499951c2f
Author: Werner Lemberg <address@hidden>
Commit: Werner Lemberg <address@hidden>

    * src/base/ftcalc.c (FT_Matrix_Check): Fix integer overflow.
    
    Reported as
    
      https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9811
---
 ChangeLog                   | 8 ++++++++
 include/freetype/freetype.h | 5 +++++
 src/base/ftcalc.c           | 8 ++++++--
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 11f2de4..6f2ca3f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2018-08-11  Werner Lemberg  <address@hidden>
+
+       * src/base/ftcalc.c (FT_Matrix_Check): Fix integer overflow.
+
+       Reported as
+
+         https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9811
+
 2018-08-10  Alexei Podtelezhnikov  <address@hidden>
 
        * src/sfnt/ttsbit.c (tt_sbit_decoder_load_compound): Follow specs.
diff --git a/include/freetype/freetype.h b/include/freetype/freetype.h
index 620df4c..fa937cd 100644
--- a/include/freetype/freetype.h
+++ b/include/freetype/freetype.h
@@ -4669,6 +4669,11 @@ FT_BEGIN_HEADER
    *   This section contains various functions used to perform
    *   computations on 16.16 fixed-float numbers or 2d vectors.
    *
+   *   *Attention*: Most arithmetic functions take `FT_Long' as arguments.
+   *   For historical reasons, FreeType was designed under the assumption
+   *   that `FT_Long' is a 32-bit integer; results can thus be undefined
+   *   if the arguments don't fit into 32 bits.
+   *
    * @order:
    *   FT_MulDiv
    *   FT_MulFix
diff --git a/src/base/ftcalc.c b/src/base/ftcalc.c
index c96d5d2..2cea03c 100644
--- a/src/base/ftcalc.c
+++ b/src/base/ftcalc.c
@@ -701,8 +701,8 @@
     if ( !delta )
       return FT_THROW( Invalid_Argument );  /* matrix can't be inverted */
 
-    matrix->xy = - FT_DivFix( matrix->xy, delta );
-    matrix->yx = - FT_DivFix( matrix->yx, delta );
+    matrix->xy = -FT_DivFix( matrix->xy, delta );
+    matrix->yx = -FT_DivFix( matrix->yx, delta );
 
     xx = matrix->xx;
     yy = matrix->yy;
@@ -784,6 +784,10 @@
         nonzero_minval = val[i];
     }
 
+    /* we only handle 32bit values */
+    if ( maxval > 0x7FFFFFFFL )
+      return 0;
+
     if ( maxval > 23170 )
     {
       FT_Fixed  scale = FT_DivFix( maxval, 23170 );