how remove values which are out of ROI?

I'm trying remove these red values on the vertical lines of attached image, and those on black region? I would really appreciate any idea. I tried the following code, but I'm getting an error which saying Warning: Polynomial is not unique; degree >= number of data points.

now it removed some of the values, but I think the way I'm doing isn't efficient or I'm doing something wrong. the image before remove those values is attached as well.

 function display_skew_final2(ST, Im,Frame)
close all;
load Im
Contrast_maxi = max(Frame(1:1).Contrast(:));
colors = {'r'};
figure; imshow (Im(:,:,1));
for idxcol = 1:size(ST.KernelX,1)
    for idxrow = 1:size(ST.KernelY,1)  
        Contrast= Frame(1:1).Contrast(idxrow,idxcol);
        Skew= Frame(1:1).Skew(idxrow,idxcol);
        Normalize_Contrast = Contrast/Contrast_maxi ;
        radi_Contrast(:) = Normalize_Contrast;
        Skew_radi(:) = Skew;   
        Skew_values= Frame(1:1).Skew;

        for    ROWCircleCenter = ST.KernelY(idxrow);
            for   COLCircleCenter = ST.KernelX(idxcol);
                centers = [COLCircleCenter ROWCircleCenter];

                coefs = polyfit( ROWCircleCenter, COLCircleCenter,1);
                slope = coefs(1);
                T = table(ROWCircleCenter,COLCircleCenter,slope,'VariableNames',{'X','Y','slope'})             
                if (Skew_radi < 0)
                    Skew_radi2 = abs(Skew_radi); else Skew_radi2 = Skew_radi; end
                if (radi_Contrast > 0.18)

                    if slope >2;  Skew_radi2 = 0; end
                    axis ij
                    viscircles (centers,Skew_radi2,'Color',colors{1});
                    %                     rectangle('Position',[(COLCircleCenter) (ROWCircleCenter) Skew_radi2 Skew_radi2],'Curvature',[1 1],'FaceColor',[1 0 0],'EdgeColor',[1 0 0],'LineWidth',5)
                end           
            end
        end
    end
end
end