# -----------------------------------
# Imports des librairies
# -----------------------------------
try:
from PIL import Image
from uuid import uuid4
import json
except ImportError:
import Image
import pytesseract
from pytesseract import Output
import cv2
import sys
import re
from pdf2image import convert_from_path
from TSLibOCR import *
import time
import os
import numpy as np
from TS_OCR import *
import MySQLdb
from datetime import datetime
from collections import Counter
debug = False
""" Retourne le temps passé depuis le lancement du programme """
def getTime():
return ""
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKCYAN = '\033[96m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
class SBA(TS_OCR):
sqlUser = 'toosmart'
sqlHost = ''
sqlPasswd = 'i5Hf7MSHKkPN1hy'
sqlDb = ''
idAbattoir = 23
linesPoids = []
linesLots = []
lineMerged = {}
dateLivraison = ""
anneeLivraison = ""
typeLivraison = ""
enAnomalie = 0
poidsTotal = 0.000
cleCoche = "2 [CÇ]O[CÇ]HE FILIE[RP.]+E LBC VPF"
ocrDpi = 500
ocrFolder = '/home/serge/www/toosmart_digicoche/data/tmp/'
outputFileName = ''
motsASurligner = []
dateTuerie = ""
poidsTotalCalcule = 0.000
poidsTotalDetecte = 0
qttCocheCalcule = 0
qttCocheDetecte = 0
nonIntegrable = False
mergedImage = []
def __init__(self, pdf, outputFileName,ocrDirectory,dbName,sqlHost):
try:
self.sqlDb = dbName
self.sqlHost = sqlHost
""" Connexion à la base de données MySQL de Digicoche (Eteko)"""
self.conn = MySQLdb.connect(host=self.sqlHost, user=self.sqlUser, passwd=self.sqlPasswd, db=self.sqlDb)
self.cursor = self.conn.cursor()
self.pdf = pdf
self.outputFileName = outputFileName
self.ocrFolder = ocrDirectory
self.printDebug("Connexion à la base de données OK")
self.printDebug("pdfs = " + str(pdf))
self.printDebug("outputFileName = " + str(outputFileName))
except:
self.printDebug("Erreur de connexion à la base de données")
def printDebug(self,pTxt):
self._debugOCR = self._debugOCR + '
' + pTxt
if debug:
print(pTxt)
# Analyse du BL de l'abattoir SBA
def read(self):
# conversion en fichier jpeg
try:
imageAmelioreeSurlignee = []
# lecture
pagesPdf = convert_from_path(self.pdf, self.ocrDpi)
iPage = 0
ocrReturnArray = []
except Exception as err:
self.printDebug(f"Unexpected A{err=}, {type(err)=}")
self.printDebug("ERROR")
""" On parcours les pages du pdf """
for page in pagesPdf:
try:
# conversion de l'image en fichier jpeg
image_name = "Page_" + str(iPage) + ".jpg"
page.save(self.ocrFolder + 'Page_' + str(iPage) + '.jpg', "JPEG")
# ---------------------------------
# Optimisation de l'image
img = cv2.imread(self.ocrFolder + 'Page_' + str(iPage) + '.jpg')
result = img.copy()
# On supprime les lignes horizontales
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray, 6, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (40, 1))
remove_horizontal = cv2.morphologyEx(thresh,
cv2.MORPH_OPEN,
horizontal_kernel,
iterations=2)
cnts = cv2.findContours(remove_horizontal, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
cv2.drawContours(result, [c], -1, (255, 255, 255), 1)
# ---------------------------------
"""
" On supprime les lignes verticales
"""
vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, 40))
remove_vertical = cv2.morphologyEx(thresh,
cv2.MORPH_OPEN,
vertical_kernel,
iterations=2)
cnts = cv2.findContours(remove_vertical, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
cv2.drawContours(result, [c], -1, (255, 255, 255), 1)
"""
" ---------------------------------------
"""
# On lance la reconaissance OCR sur l'image
df = pytesseract.image_to_string(np.array(result), lang='fra',
config=' --oem 0 --psm 4')
cv2.imwrite(self.ocrFolder + 'Page_' + str(iPage) + '_result.jpg', result)
ocrReturnArray.append(df)
# ---------------------------------
imageAmelioreeSurlignee.append(Image.open(self.ocrFolder + 'Page_' + str(iPage) + '_result.jpg'))
iPage = iPage + 1
except Exception as err:
self.printDebug(f"Unexpected B{err=}, {type(err)=}")
self.printDebug("ERROR")
try:
# Recherche de la date de livraison
m = re.search(r"\b(0[1-9]|[12]\d|3[01])[-/](0[1-9]|1[0-2])[-/](19\d\d|20\d\d)\b", ocrReturnArray[0])
if m is not None:
self.dateLivraison = datetime.strptime(m.group(), "%d/%m/%Y").strftime("%Y%m%d")
self.anneeLivraison = datetime.strptime(m.group(), "%d/%m/%Y").strftime("%Y")
# Recherche du type de livraison
m = re.search(self.cleCoche, ocrReturnArray[0])
if m is not None:
self.typeLivraison = m.group()
else:
# BL Non intégrable car la clef n'est pas correcte
# On retourne dans le bash l'information puis on quitte l e script
# (NE PAS SUPPRIMER CETTE LIGNE !!!)
print('{"blNonIntegrable":1,"cleCocheDetectee":"'+ocrReturnArray[0]+'"}')
self.nonIntegrable = True
return False
except Exception as err:
self.printDebug(f"Unexpected A{err=}, {type(err)=}")
self.printDebug("ERROR")
# ---------------------------------
# Recherche du numéro du BL
try:
m = re.findall(r"Bon de livraison\n+([\w\W])+APE[\D]+[\d ]+Z? ([0-9]{8})", ocrReturnArray[0], re.MULTILINE)
for element in m:
self._numBl = element[1]
except Exception as err:
self._numBl = ""
self.printDebug(f"Unexpected1 {err=}, {type(err)=}")
self.printDebug("ERROR1")
if self._numBl != "":
if self.blExist():
# Le BL Existe déjà en base de donneés
print('{"blEnDoublon":1}')
self.nonIntegrable = True
return False
# ---------------------------------
j = 0
""" On parcours les pages du pdf """
for Sfile in ocrReturnArray:
self.printDebug('Page '+str(j+1)+' ')
self.printDebug(Sfile)
# Recherche du nombre de coches, du poids et du lot
if self.typeLivraison != "":
m = re.findall(r"[^.0-9]([0-9]{1,3}[.,][0-9]*) Kg P[B38]([S0-9?]{10})", Sfile)
if len(m) > 0:
self.linesPoids.extend(m)
self.qttCocheCalcule = len(m) + self.qttCocheCalcule
for poid in m:
# on ajoute le poids détecté au poids total
self.poidsTotalCalcule = round(self.poidsTotalCalcule + round(float(poid[0].replace(",",".")),3),3)
# ---------------------------------
# Recherche du poids total et du nombre de pièces
try:
m = re.findall(r"TOTAL MARCHANDISES\D+(([0-9])+)\D+(([0-9.,])+)\D+K", Sfile,
re.MULTILINE)
if len(m) > 0:
for element in m:
self.poidsTotalDetecte = element[2].replace(".", "")
self.qttCocheDetecte = round(int(element[0]) / 2)
except Exception as err:
self.printDebug(f"Unexpected2 {err=}, {type(err)=}")
self.printDebug("ERROR1")
# ---------------------------------
# Gestion de l'image pour surlignage des zones
try:
tesseract_output = pytesseract.image_to_data(imageAmelioreeSurlignee[j],
output_type=pytesseract.Output.DICT)
self.surlignerZones(tesseract_output, j)
except:
self.printDebug("ERROR")
j = j + 1
# ---------------------------------
# Merge des fichiers PDF
try:
im = Image.open(self.ocrFolder + self.outputFileName + '-color-0.jpg')
im.save(self.ocrFolder+self.outputFileName+"-BL.pdf", save_all=True, append_images=self.mergedImage)
except:
self.printDebug("ERROR")
# ---------------------------------
""" Surligne les zones détectées par l'OCR dans le fichier """
def surlignerZones(self, tesseract_output,iPage):
try:
imageASurligner = cv2.imread(self.ocrFolder + 'Page_'+str(iPage)+'_result.jpg')
blockASurligner = []
except Exception as err:
print(f"UnexpectedC {err=}, {type(err)=}")
print("ERRORC")
for i, level_idx in enumerate(tesseract_output['level']):
if level_idx == 5:
# if tesseract_output['text'][i] in self.motsASurligner:
bbox = {
'x': 1 * tesseract_output['left'][i],
'y': 1 * tesseract_output['top'][i],
'width': 1 * tesseract_output['width'][i],
'height': 1 * tesseract_output['height'][i],
'rotation': 0,
'confidence': tesseract_output['conf'][i],
'text': tesseract_output['text'][i]
}
if int(tesseract_output['conf'][i]) >= 85:
color = (42, 219, 151)
elif int(tesseract_output['conf'][i]) >= 50:
color = (0, 165, 255)
else:
color = (0, 0, 255)
try:
overlay = imageASurligner.copy()
cv2.rectangle(overlay, (bbox['x'], bbox['y']),
(bbox['x'] + bbox['width'], bbox['y'] + bbox['height']), color, cv2.FILLED)
alpha = 0.5
imageASurligner = cv2.addWeighted(overlay, alpha, imageASurligner, 1 - alpha, 0)
blockASurligner.append(bbox)
except Exception as err:
print(f"UnexpectedD {err=}, {type(err)=}")
print("ERRORD")
# On sauvegarde le fichier
try:
imageASurligner = cv2.cvtColor(imageASurligner, cv2.COLOR_BGR2RGB)
img = Image.fromarray(imageASurligner, "RGB")
width, height = img.size
new_size = (width // 4, height // 4)
resized_image = img.resize(new_size)
if iPage > 0:
self.mergedImage.append(resized_image)
resized_image.save(self.ocrFolder + self.outputFileName + '-color-'+str(iPage)+'.jpg')
except Exception as err:
print("erreur")
print(f"UnexpectedE {err=}, {type(err)=}")
return True
def mergeData(self):
try:
if self.nonIntegrable:
return False
# Parcours des lignes de poids
for linePoids in self.linesPoids:
self.lineMerged[linePoids[1]] = {"poids" : linePoids[0],
"date" : str(self.quantiemeVersDate(int(linePoids[1][1:4]),self.anneeLivraison)),
"lot":linePoids[1].replace("?","7")}
# Date de création de la ligne
currentDate = datetime.now().strftime('%Y%m%d%H%M')
sql = (
"INSERT INTO tier_att(id_base, id_type, id_tier,attdatcre, id_work, actif, newobj, att1, att2, att3,att6,att7,att8,att9,attf100,att17,att90) "
"VALUES (1, 10, NULL,'" + str(
currentDate) + "', 16, 1, 0," + str(self.idAbattoir) + ", '") + self.typeLivraison + "', '" + str(
self.poidsTotalCalcule) + "','" + str(self.poidsTotalDetecte) + "','" + str(
self.poidsTotalCalcule) + "','" + str(self.qttCocheDetecte) + "','" + str(
self.qttCocheCalcule) + "', '" + self.dateLivraison + "','"+ self._numBl+"','"+ self._debugOCR.replace("'", "\\'") +"');"
try:
self.cursor.execute(sql)
except Exception as err:
print(f"UnexpectedF {err=}, {type(err)=}")
print("ERROR A2")
# On récupère l'id du BL créé dans Eteko
blId = self.cursor.lastrowid
# On retourne dans le bash l'id du BL créé
# (NE PAS SUPPRIMER CETTE LIGNE !!!)
print('{"blId":' + str(blId) + "}")
# ------------------------------------------
for key in self.lineMerged:
# Initialisation des anomalies de la ligne à 0
self.enAnomalie = 0
try:
# On stocke la date de livraison dans le format YYYYMMDD
self.dateLivraisonLigne = self.lineMerged[key]["date"]
except:
self.dateLivraisonLigne = ""
self.enAnomalie = 1
# Récupération du numéro de tuerie
numeroTuerie = str(self.lineMerged[key]["lot"])[-4:]
# On créé la ligne du BL dans eteko
sql = ("INSERT INTO prod_att(id_base, id_type, id_prod, id_work, actif, newobj, fid_prod, type, name, revision, `desc`, att1, att2, att3, att4, attf100, attf101,attf104) VALUES (1, 11, NULL, 11, 1, 0, null, null, null, null, null, '" + str(blId) + "','" + str(self.lineMerged[key]["lot"]) +"', '" + str(numeroTuerie) +"', '" + str(self.lineMerged[key]["poids"])+ "', '" + str(self.dateLivraisonLigne) + "', '" + str(self.enAnomalie) + "', '" + self.dateLivraison + "');")
self.cursor.execute(sql)
# Fermeture de la connexion à MySQL
self.conn.close()
except Exception as e:
exception_type, exception_object, exception_traceback = sys.exc_info()
filename = exception_traceback.tb_frame.f_code.co_filename
line_number = exception_traceback.tb_lineno
print("Exception type: ", exception_type)
print("File name: ", filename)
print("Line number: ", line_number)
self.printDebug(f"Unexpected {e=}, {type(e)=}")
self.printDebug("ERROR")
def quantiemeVersDate(self, n, anneeLivraison):
try:
anneeLivraison = int(anneeLivraison)
"""Donne la date d=[j,m,a] qui est le nième jour de l'année a"""
if ((anneeLivraison % 4 == 0 and anneeLivraison % 100 != 0) or anneeLivraison % 400 == 0):
# bissextile?
jm = (0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366)
else:
jm = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
for m in range(1, 13):
if jm[m] >= n:
return str(anneeLivraison) + str("{:02d}".format(m)) + str("{:02d}".format(n - jm[m - 1]))
anneeLivraison = anneeLivraison - 1
"""Donne la date d=[j,m,a] qui est le nième jour de l'année a"""
if ((anneeLivraison % 4 == 0 and anneeLivraison % 100 != 0) or anneeLivraison % 400 == 0):
# bissextile?
jm = (0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366)
else:
jm = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
for m in range(1, 13):
if jm[m] >= n:
return str(anneeLivraison) + str("{:02d}".format(m)) + str("{:02d}".format(n - jm[m - 1]))
except Exception as e:
exception_type, exception_object, exception_traceback = sys.exc_info()
filename = exception_traceback.tb_frame.f_code.co_filename
line_number = exception_traceback.tb_lineno
print("Exception type: ", exception_type)
print("File name: ", filename)
print("Line number: ", line_number)
self.printDebug(f"Unexpected {e=}, {type(e)=}")
self.printDebug("ERROR")
print(anneeLivraison)
print("
")