GluTess и многоугольник с совпадающими вершинами на разных рёбрах.

void __stdcall combineCallback(GLdouble coords[3], 
           GLdouble *vertex_data[4],
           GLfloat weight[4], GLdouble **dataOut )
{
  GLdouble *vertex;
  int i;

  vertex = (GLdouble *) malloc(6 * sizeof(GLdouble));
  vertex[0] = coords[0];
  vertex[1] = coords[1];
  vertex[2] = coords[2];
  for (i = 3; i < 7; i++)
    vertex[i] = weight[0] * vertex_data[0][i] 
  + weight[1] * vertex_data[1][i]
  + weight[2] * vertex_data[2][i] 
  + weight[3] * vertex_data[3][i];
  *dataOut = vertex;
}

  gluTessCallback(tess,GLU_TESS_BEGIN,(void (__stdcall *)())glTessBegin);
  gluTessCallback(tess,GLU_TESS_END,glTessEnd);
  gluTessCallback(tess, GLU_TESS_VERTEX,(void (__stdcall *) ()) glVertex2dv);
  gluTessCallback(tess, GLU_TESS_ERROR, (void (__stdcall *) ()) errorCallback);
  gluTessCallback(tess, GLU_TESS_COMBINE, (void (__stdcall *) ()) combineCallback);
  //gluTessProperty(tess, GLU_TESS_TOLERANCE, 0.0);

  displayList1 = glGenLists(1);
  glNewList(displayList1,GL_COMPILE);
    gluTessBeginPolygon(tess,vert);
      gluTessBeginContour(tess);
        /*gluTessVertex(tess,vert[0],vert[0]);
        gluTessVertex(tess,vert[1],vert[1]);
        gluTessVertex(tess,vert[2],vert[2]);
        gluTessVertex(tess,vert[3],vert[3]);
        gluTessVertex(tess,vert[4],vert[4]);*/
        for (int i = 0; i < points; i++)
        {
          gluTessVertex(tess,vert[i],vert[i]);
        }
      gluTessEndContour(tess);
    gluTessEndPolygon(tess);
  glEndList();
  gluDeleteTess(tess);