calculate perpendicular distance to find furthest point from line

Display.cpp

@@ -137,6 +137,7 @@ void Display::update ()
                 sf::Vector2f pos = pt.getPosition();
                 float x = pos.x;
                 float y = pos.y;
+                // TODO: only check explicit x and y seperate!
                 if (x < topLeft.x && y < topLeft.y) topLeft = pos;
                 if (x > topRight.x && y < topLeft.y) topRight = pos;
                 if (x < botLeft.x && y > botLeft.y) botLeft = pos;
@@ -186,17 +187,48 @@ void Display::update ()
 
         for (auto& pt : m_points)
         {
-            sf::Vector2f pos = pt.getPosition();
+            pt.setFillColor(sign(
-            float x = pos.x;
+                pt.getPosition().x, pt.getPosition().y,
-            float y = pos.y;
+                m_hull[0].position.x, m_hull[0].position.y,
-
+                m_hull[1].position.x, m_hull[1].position.y) > 0 ? sf::Color::Red : sf::Color::Green);
-            pt.setFillColor(sign(x, y, m_hull[0].position.x, m_hull[0].position.y, m_hull[1].position.x, m_hull[1].position.y) > 0 ? sf::Color::Red : sf::Color::Green);
         }
     }
     else if (curStep == 3)
     {
         // third step: draw triangle, remove inner points
         m_textStatus.setString(text + "find furthest point and draw triangle...");
+
+        sf::Vector2f pos = m_points[0].getPosition();
+        float maxDistance = 0;
+        for (auto& pt : m_points)
+        {
+            float distance = pDistance(
+                pt.getPosition().x, pt.getPosition().y,
+                m_hull[0].position.x, m_hull[0].position.y,
+                m_hull[1].position.x, m_hull[1].position.y);
+
+            if (pt.getFillColor() == sf::Color::Green)
+            {
+                std::cout << "distance to green point " << pt.getPosition().x << ", " << pt.getPosition().y << ": " << distance << std::endl;
+                    
+            }
+            else if (pt.getFillColor() == sf::Color::Red)
+            {
+                std::cout << "distance to red point " << pt.getPosition().x << ", " << pt.getPosition().y << ": " << distance << std::endl;
+            }
+
+            if (distance > maxDistance)
+            {
+                pos = pt.getPosition();
+                maxDistance = distance;
+            }
+        }
+
+        if (maxDistance > 0)
+        {
+            // TODO: not append but insert between last line points
+            m_hull.append(sf::Vertex(pos, sf::Color::Blue));
+        }
     }
     else if (curStep == 4)
     {

Utility.h

@@ -1,9 +1,9 @@
 #pragma once
 #ifndef UTILITY_H
 
-static float sign(float x1, float y1, float x2, float y2, float x3, float y3)
+static float sign(float x, float y, float x1, float y1, float x2, float y2)
 {
-    return (x1 - x3) * (y2 - y3) - (x2 - x3) * (y1 - y3);
+    return (x - x2) * (y1 - y2) - (x1 - x2) * (y - y2);
 }
 
 static float sign (Point &p1, Point &p2, Point &p3)
@@ -23,6 +23,39 @@ static bool IsPointInTriangle(Point &pt, Point &p1, Point &p2, Point &p3)
     return !(has_neg && has_pos);
 }
 
+static float pDistance(float x, float y, float x1, float y1, float x2, float y2) {
+
+    float A = x - x1;
+    float B = y - y1;
+    float C = x2 - x1;
+    float D = y2 - y1;
+
+    float dot = A * C + B * D;
+    float len_sq = C * C + D * D;
+    float param = -1;
+    if (len_sq != 0) // in case of 0 length line
+        param = dot / len_sq;
+
+    float xx, yy;
+    if (param < 0) {
+        xx = x1;
+        yy = y1;
+    }
+    else if (param > 1) {
+        xx = x2;
+        yy = y2;
+    }
+    else {
+        xx = x1 + param * C;
+        yy = y1 + param * D;
+    }
+
+    float dx = x - xx;
+    float dy = y - yy;
+    return dx * dx + dy * dy;
+    //return sqrt(dx * dx + dy * dy);
+}
+
 static bool SortForMinXMaxY (const Point& a, const Point& b)
 {
     if (a.x() != b.x())