File libass-CVE-2020-26682.patch of Package libass.20681

Index: libass-0.14.0/libass/ass_outline.c
===================================================================
--- libass-0.14.0.orig/libass/ass_outline.c	2017-10-31 13:14:11.000000000 +0100
+++ libass-0.14.0/libass/ass_outline.c	2021-03-16 09:14:17.114485654 +0100
@@ -47,6 +47,11 @@ static void outline_clear(ASS_Outline *o
     outline->n_segments = outline->max_segments = 0;
 }
 
+static bool valid_point(const FT_Vector *pt)
+{
+    return abs(pt->x) <= OUTLINE_MAX && abs(pt->y) <= OUTLINE_MAX;
+}
+
 bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
 {
     if (!source || !source->n_points) {
@@ -77,12 +82,16 @@ bool outline_convert(ASS_Outline *outlin
             continue;
         }
 
+        if (!valid_point(source->points + j))
+            goto fail;
         switch (FT_CURVE_TAG(source->tags[j])) {
         case FT_CURVE_TAG_ON:
             st = S_ON;
             break;
 
         case FT_CURVE_TAG_CONIC:
+            if (!valid_point(source->points + last))
+                goto fail;
             pt.x =  source->points[last].x;
             pt.y = -source->points[last].y;
             switch (FT_CURVE_TAG(source->tags[last])) {
@@ -110,6 +119,8 @@ bool outline_convert(ASS_Outline *outlin
         outline->points[outline->n_points++] = pt;
 
         for (j++; j <= last; j++) {
+            if (!valid_point(source->points + j))
+                goto fail;
             switch (FT_CURVE_TAG(source->tags[j])) {
             case FT_CURVE_TAG_ON:
                 switch (st) {
@@ -236,6 +247,9 @@ void outline_free(ASS_Outline *outline)
  */
 bool outline_add_point(ASS_Outline *outline, ASS_Vector pt, char segment)
 {
+    if(abs(pt.x) > OUTLINE_MAX || abs(pt.y) > OUTLINE_MAX)
+        return false;
+
     if (outline->n_points >= outline->max_points) {
         size_t new_size = 2 * outline->max_points;
         if (!ASS_REALLOC_ARRAY(outline->points, new_size))
@@ -1303,7 +1317,7 @@ bool outline_stroke(ASS_Outline *result,
 {
     const int dir = 3;
     int rad = FFMAX(xbord, ybord);
-    assert(rad >= eps);
+    assert(rad >= eps && rad <= OUTLINE_MAX);
 
     result->n_points = result->n_segments = 0;
     result1->n_points = result1->n_segments = 0;
@@ -1329,12 +1343,10 @@ bool outline_stroke(ASS_Outline *result,
     str.err_c = 390 * rel_err * rel_err;
     str.err_a = e;
 
-    for (size_t i = 0; i < path->n_points; i++) {
-        if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX)
-            return false;
-        if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX)
-            return false;
-    }
+#ifndef NDEBUG
+    for (size_t i = 0; i < path->n_points; i++)
+        assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX);
+#endif
 
     ASS_Vector *start = path->points, *cur = start;
     for (size_t i = 0; i < path->n_segments; i++) {
Index: libass-0.14.0/libass/ass_outline.h
===================================================================
--- libass-0.14.0.orig/libass/ass_outline.h	2017-10-31 13:14:11.000000000 +0100
+++ libass-0.14.0/libass/ass_outline.h	2021-03-16 09:14:17.114485654 +0100
@@ -83,8 +83,9 @@ typedef struct {
     char *segments;
 } ASS_Outline;
 
-#define OUTLINE_MIN  (-((int32_t) 1 << 28))
+// ouline point coordinates should always be in [-OUTLINE_MAX, +OUTLINE_MAX] range
 #define OUTLINE_MAX  (((int32_t) 1 << 28) - 1)
+// cubic spline splitting requires 8 * OUTLINE_MAX + 4 <= INT32_MAX
 
 bool outline_alloc(ASS_Outline *outline, size_t n_points, size_t n_segments);
 bool outline_convert(ASS_Outline *outline, const FT_Outline *source);
Index: libass-0.14.0/libass/ass_rasterizer.c
===================================================================
--- libass-0.14.0.orig/libass/ass_rasterizer.c	2017-10-31 13:14:11.000000000 +0100
+++ libass-0.14.0/libass/ass_rasterizer.c	2021-03-16 09:14:17.114485654 +0100
@@ -269,12 +269,10 @@ bool rasterizer_set_outline(RasterizerDa
     }
     rst->size[0] = rst->n_first;
 
-    for (size_t i = 0; i < path->n_points; i++) {
-        if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX)
-            return false;
-        if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX)
-            return false;
-    }
+#ifndef NDEBUG
+    for (size_t i = 0; i < path->n_points; i++)
+        assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX);
+#endif
 
     ASS_Vector *start = path->points, *cur = start;
     for (size_t i = 0; i < path->n_segments; i++) {