Cropping an image after detecting the corners using opencv python

Question

I'm new one to openCV and computer-vision. Just now i'm trying to crop the Tiff scan after detecting the corners and then extract information from it based on exact coordinates x:y using python, openCV, numpy and OCR with Tesseract.

What i achieved right now is that i upload an image (scan), binarize it, fix rotation and remove empty spaces. Result is already good, but not good enough. My image is still always rotated little bit. Here is image example Example Example(w/o Arrows)

The question is: How to detect these corners and crop everything outside them?

Here is my current code:

for filenumber in range(2,7):
    img = cv2.imread('img' + str(filenumber) + '.tif')

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img2 = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    gray = cv2.bitwise_not(gray)

    img = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 25, 11)
    kernel = np.ones((2, 2), np.uint8)
    img = cv2.erode(img, kernel, iterations=3)
    thresh = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
    coords = np.column_stack(np.where(thresh > 0))
    angle = cv2.minAreaRect(coords)[-1]

    if angle < -45:
        angle = -(90 + angle)
    else:
        angle = -angle
    # rotate the image to deskew it
    (h, w) = img.shape[:500]
    center = (w // 400, h // 400)
    M = cv2.getRotationMatrix2D(center, angle, 1)
    rotated = cv2.warpAffine(img, M, (w, h),
        flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
    # draw the correction angle on the image so we can validate it
    cv2.putText(rotated, "Angle: {:.2f} degrees".format(angle),
        (100, 400), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 0, 255), 2)

    img = rotated

    th, threshed = cv2.threshold(img, 240, 255, cv2.THRESH_BINARY_INV)
    ## (2) Morph-op to remove noise
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11,11))
    morphed = cv2.morphologyEx(threshed, cv2.MORPH_CLOSE, kernel)

    ## (3) Find the max-area contour
    cnts = cv2.findContours(morphed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
    cnt = sorted(cnts, key=cv2.contourArea)[-1]
    ## (4) Crop and save it
    x,y,w,h = cv2.boundingRect(cnt)
    dst = img2[y:y+h, x:x+w]
    img = dst
    size_multiplier = szm = 1
    cv2.imwrite('img_' + str(filenumber) + '_Cropped' + '.jpg', img)

#Configs for OCR segments
for nnumb in range(2, 7):
    print('[INFO2]:   File=' + str(filenumber) + ';  nnumb=' + str(nnumb))
    if nnumb == 1:
        sub_image = img[130:130 + 90, 1220:1220 + 600]
        config = ('-l rus --oem 0 --psm 3 -c tessedit_char_whitelist="0123456789"')
    if nnumb == 2:
        sub_image = img[150:150 + 60, 1980:1980 + 460]
        config = ('-l rus --oem 1 --psm 3 -c tessedit_char_whitelist="0123456789"')
    if nnumb == 3:
        sub_image = img[230:230 + 70, 620:620 + 3000]
        config = ('-l rus --oem 0 --psm 3')
    if nnumb == 4:
        sub_image = img[410:410 + 70, 835:835 + 470]
        config = ('-l rus --oem 0 --psm 1 -c tessedit_char_whitelist="0123456789"')
    if nnumb == 5:
        sub_image = img[480:480 + 220, 610:610 + 1300]
        config = ('-l rus --oem 0 --psm 3')
    if nnumb == 6:
        sub_image = img[720:720 + 70, 110:110 + 500]
        config = ('-l rus --oem 0 --psm 3 -c tessedit_char_whitelist="0123456789"')

[

UPDATE: Final Code

def cornersandcrop(img):
    main_image = img
    main_imageF = main_image.copy()
    gray_image = main_image.copy()
    #Remove parts of image except corners
    gray_image[70:70 + 500, 70:70 + 500] = [255, 255, 255]
    gray_image[44:44 + 100, 1900:1900 + 550] = [255, 255, 255]
    gray_image[2270:2270 + 700, 45:45 + 200] = [255, 255, 255]
    gray_image[140:2880, 0:2500] = [255, 255, 255]
    gray_image[0:3000, 150:2350] = [255, 255, 255]

    gray_image = cv2.cvtColor(gray_image, cv2.COLOR_BGR2GRAY)
    gray_image = cv2.medianBlur(gray_image, 5)
    gray_image = cv2.adaptiveThreshold(gray_image, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY,11,20)
    kernel = np.ones((2, 2), np.uint8)
    gray_image = cv2.erode(gray_image, kernel, iterations=5)
    gray_image = cv2.dilate(gray_image, kernel, iterations=2)
    gray_image = cv2.morphologyEx(gray_image, cv2.MORPH_OPEN, np.ones((1, 1), np.uint8))

    template = cv2.imread('Templates\\Template_Corner_Top_Left.png', 0)
    template2 = cv2.imread('Templates\\Template_Corner_Top_Right.png', 0)
    template3 = cv2.imread('Templates\\Template_Corner_Bot_Right.png', 0)
    template4 = cv2.imread('Templates\\Template_Corner_Bot_Left.png', 0)

    width, height = template.shape[::-1] #get the width and height
    width2, height2 = template2.shape[::-1]
    width3, height3 = template3.shape[::-1]
    width4, height4 = template4.shape[::-1]

    match = cv2.matchTemplate(gray_image, template, cv2.TM_CCOEFF_NORMED)
    match2 = cv2.matchTemplate(gray_image, template2, cv2.TM_CCOEFF_NORMED)
    match3 = cv2.matchTemplate(gray_image, template3, cv2.TM_CCOEFF_NORMED)
    match4 = cv2.matchTemplate(gray_image, template4, cv2.TM_CCOEFF_NORMED)

    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match)
    top_Pos1 = max_loc
    Pos1 = (top_Pos1[0] + width-115, top_Pos1[1] + height-115)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match2)
    top_Pos2 = max_loc
    Pos2 = (top_Pos2[0] + width2-5, top_Pos2[1] + height2-115)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match3)
    top_Pos3 = max_loc
    Pos3 = (top_Pos3[0] + width3-5, top_Pos3[1] + height3-5)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(match4)
    top_Pos4 = max_loc
    Pos4 = (top_Pos4[0] + width4-115, top_Pos4[1] + height4-5)

    src_pts = np.array([Pos1, Pos2, Pos3, Pos4], dtype=np.float32)
    dst_pts = np.array([[0, 0],   [3000, 0],  [3000, 2500], [0, 2500]], dtype=np.float32)
    M = cv2.getPerspectiveTransform(src_pts, dst_pts)
    warp = cv2.warpPerspective(main_imageF, M, (3000, 2500))
    warp = cv2.resize(warp, (int(2500), int(3000)),fx=1, fy=1, interpolation = cv2.INTER_CUBIC)
return (warp)

Answer 1

This works for me in Python/OpenCV for locating one corner using template matching. Just make the template image larger than your corner so that there is some white around it.

Input:

Template:

import cv2
import numpy as np

# read image
img = cv2.imread('drawing.jpg')

# convert img to grayscale
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# read template as grayscale
tmplt = cv2.imread('corner_ul.png', cv2.IMREAD_GRAYSCALE)
hh, ww = tmplt.shape

# define corner intersection in template
offset_x = 23
offset_y = 28

# do template matching
corrimg = cv2.matchTemplate(img_gray,tmplt,cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(corrimg)
max_val_ncc = '{:.3f}'.format(max_val)
print("normalize_cross_correlation: " + max_val_ncc)
xx = max_loc[0]
yy = max_loc[1]
corner_x = xx + offset_x
corner_y = yy + offset_y
print('xmatchloc =',xx,'ymatch =',yy)
print('cornerlocx =',corner_x,'cornerlocy =',corner_y)

# draw template bounds and corner intersection in red onto img
result = img.copy()
cv2.rectangle(result, (xx, yy), (xx+ww, yy+hh), (0, 0, 255), 2)
cv2.circle(result, (corner_x,corner_y), 1, (0, 0, 255), 2)

cv2.imshow('image', img)
cv2.imshow('template', tmplt)
cv2.imshow('result', result)
cv2.waitKey(0)
cv2.destroyAllWindows()

# save results
cv2.imwrite('drawing_template_match_location.jpg', result)

Information:

normalize_cross_correlation: 1.000
xmatchloc = 0 ymatch = 318
cornerlocx = 23 cornerlocy = 346

Result:

You can also refine the result to get sub-pixel accuracy. See https://www.bbsmax.com/A/lk5aBbGod1/

Rotate the template 3 times each by 90 degrees to form the other 3 templates, measure or compute the offsets from the top left corner to the intersection of the corner lines and do the template matching. Then once you have all 4 matches, you can do your cropping using numpy slicing.

Answer 2

I suggest using template matching by "adjusted filter":

• Convert image to binary image (as you did), and use closing instead of erode.
• Convert image to "-1"s and "1"s: replace 0 with -1 and 255 with 1.
• Build L shaped kernel h (for finding the bottom left corner):
  • Place -1 where value in im needs to be -1, and 1 when value needs to be 1.
  • Make sure the corner of the L shape in h is at the center (it's a bit of a waste - you may avoid it, and fix the position later).
  • Example for kernel (small scale):
    0 -1 1 0 0
0 -1 1 0 0
0 -1 1 1 1
0 -1 -1 -1 -1
0 0 0 0 0
• Filter im with kernel h - the maximum value of the output is the position that best matches h.
• Find x, y coordinate of maximum value of filtered image.

Here is a code sample that finds the bottom left corner:

import numpy as np
import cv2

img = cv2.imread('img1.tif')
orig_img = img.copy()

gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = cv2.bitwise_not(gray)

img = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 25, 11)
img = cv2.morphologyEx(img, cv2.MORPH_OPEN, np.ones((3, 3), np.uint8))  # Opening - remove white dots around corners.

# cv2.imwrite('img.tif', img)

# Convert from 0 to -1 and 255 to 1 (im is "binary" image with "-1"s and "1"s).
im = img.astype(float) / 127.5 - 1

# Build L shape kernel h that matches the L shape we want to search.
# Place "-1" where value in im needs to be "-1", and "1" when value needs to be "1".
# Make sure the corner of the L shape in h is at the center (it's a bit of a waste).
# Example for kernel (small scale):
#   0 -1  1  0  0
#   0 -1  1  0  0
#   0 -1  1  1  1
#   0 -1 -1 -1 -1
#   0  0  0  0  0

h = np.zeros((75, 75))  # Kernel size is 75x75
h[0:37, 37:39] = 1  # Two columns of "1"s from top to center
h[36:38, 37:] = 1   # Two rows of "1"s from center to right side
h[0:39, 36] = -1    # One column of "-1"s
h[38, 36:] = -1     # One row of "-1"s

# Save h kernel as an image for testing
h2 = h.copy()
h2 = ((h2+1)*127.5).astype(np.uint8)
cv2.imwrite('h2.png', h2)

# Filter im with kernel h - the maximum value of the output is the position that best matches h
imf = cv2.filter2D(im, -1, h)

# Find index of maximum value from 2D numpy array
pos_y, pos_x = np.where(imf == np.amax(imf))

# Draw red circle around coordinate (pos_x, pos_y) for testing.
cv2.circle(orig_img, (int(pos_x), int(pos_y)), 8, (0, 0, 255), thickness=2)
cv2.imwrite('circled_im.png', orig_img)  # Save image for testing

---

Result (bottom left corner):

Filter kernel (as an image):

---

Update:

In case there are other "L shaped" objects, you may need to use more "aggressive" kernel.

Example:

# More "aggressive" kernel
h = np.zeros((75, 75))  # Kernel size is 75x75
h[0:37, 37:41] = 1  # 4 columns of "1"s from top to center
h[34:38, 37:] = 1   # 4 rows of "1"s from center to right side
h[0:39, 36] = -1    # One column of "-1"s
h[38, 36:] = -1     # One row of "-1"s
h[0:34, 41] = -1  # 1 columns of "-1"s from top to center
h[33, 41:] = -1   # 1 rows of "-1"s from center to right side