Skip to content

Update to deal with market demand and supply level information #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deflektor wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Update to deal with market demand and supply level information #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cca22e6
Working Version for new ED Marketlevel data
Jan 17, 2016
a9d9dd8
Merge branch 'refs/heads/Changes_201512'
Jan 17, 2016
9ef0df5
Minor bugfix for supply numbers ending with 4
Jan 17, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,3 +231,4 @@ EliteOCR.spec

# file generated during build
EliteOCRcmd.py
errorlog.txt
1 change: 1 addition & 0 deletions EliteOCR.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,7 @@ def drawSnippet(self, graphicsview, item):
"""Draw single result item to graphicsview"""
res = self.current_result
snippet = self.cutImage(res.contrast_commodities_img, item)
#print "eliteocr.py 1070: my Snippet Itemname ", item.value, "file ", res.contrast_commodities_img.shape, "Shape y1 ", item.y1, " y2 ", item.y2, " x1 ", item.x1, " x2 ", item.x2
if self.dark_theme:
snippet = 255 - snippet
#cv2.imwrite('snippets/'+unicode(self.currentsnippet)+'.png',snippet)
Expand Down
60 changes: 48 additions & 12 deletions engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,10 @@ def getAreas(self, x1, x2):
areas = [[0.0, 0.357],
[0.361, 0.447],
[0.448, 0.533],
[0.625, 0.731],
[0.732, 0.802],
[0.805, 0.909],
[0.911, 0.999]]
[0.625, 0.767],
[0.773, 0.794], #### Demand Level (should result in pixel for level, add 80 pixel here for cut off)
[0.800, 0.9637],
[0.970, 0.999]] #### Supply Level

new_areas = []
x = x2 - x1
Expand All @@ -168,6 +168,7 @@ def __init__(self, image, settings, areas, isstation, calibration = False):
else:
self.numbers = TrainedDataNumbers(settings)
self.letters = TrainedDataLetters(settings)
self.level = TrainedDataLevel(settings)
"""
layers = np.array([400,36,12])
self.nnetwork = cv2.ANN_MLP(layers, 1,0.6,1)
Expand Down Expand Up @@ -308,7 +309,11 @@ def findBoxes(self, input, lines):
startchar = line[3]+2
endchar = 0

if firstflag and not blackflag and whitecount > (line[3] - line[2])/2.6:
if (
(firstflag and not blackflag and whitecount > (line[3] - line[2])/2.6)
Copy link
Copy Markdown
Contributor

@Marginal Marginal Jan 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or just change the detection factor from 2.6 to 4 because of the taller commodity lines?

Copy link
Copy Markdown
Author

@deflektor deflektor Jan 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe. It is difficult for me to understand, how this would change something.

The main problem I had, was that the columns are defined with areas, but the OCR deals with boxes/words, and does not use the area definition. In the new screenshot the demand/supply number is only some pixels away from the level information. So I was capturing the number and the level in one word.

or (last+7 >= self.areas[4][0] and last+7 < self.areas[4][0]+7)
or (last+7 >= self.areas[6][0] and last+7 < self.areas[6][0]+7)
):
#h = line[3] - line[2]
last = i - whitecount + 1
whitecount = 0
Expand Down Expand Up @@ -411,6 +416,10 @@ def mlpocr(self, input, box, type):
resultclasses = -1 * np.ones((len(data),self.station.keys), dtype='float32')
self.station.nnetwork.predict(data, resultclasses)
classdict = self.station.classdict
elif type == "level":
resultclasses = -1 * np.ones((len(data),self.level.keys), dtype='float32')
self.level.nnetwork.predict(data, resultclasses)
classdict = self.level.classdict


for k in range(len(resultclasses)):
Expand Down Expand Up @@ -447,9 +456,9 @@ def ocrImage(self, input, isstation, calibration):
if word["box"][1] < self.areas[0][1]:
result = self.mlpocr(input, word, "letters")
elif word["box"][0] > self.areas[4][0] and word["box"][1] < self.areas[4][1]:
result = self.mlpocr(input, word, "letters")
result = self.mlpocr(input, word, "level")
elif word["box"][0] > self.areas[6][0] and word["box"][1] < self.areas[6][1]:
result = self.mlpocr(input, word, "letters")
result = self.mlpocr(input, word, "level")
else:
result = self.mlpocr(input, word, "numbers")

Expand All @@ -468,7 +477,8 @@ def __init__(self, ocr_data, path, language = "big", levels = True):

self.levels = {u"eng": [u'LOW', u'MED', u'HIGH'],
u"deu": [u'NIEDRIG', u'MITTEL', u'HOCH'],
u"fra": [u'FAIBLE', u'MOYEN', u'ÉLEVÉ']}
u"fra": [u'FAIBLE', u'MOYEN', u'ÉLEVÉ'],
u"lev": [u'#', u'*', u'+']}
file = codecs.open(path + ""+ os.sep +"commodities.json", 'r', "utf-8")
self.comm_list = json.loads(file.read())
file.close()
Expand Down Expand Up @@ -525,26 +535,38 @@ def cleanCommodities(self, data, levels):
if not data[i][4] is None and levels:
try:
topratio = 0.0
toplev = ""
for lev in self.levels[self.lang]:
toplev = "MED"
for lev in self.levels[u"lev"]:
if data[i][4].value is None:
print "None!"
rat = ratio(unicode(data[i][4].value), unicode(lev))
if rat > topratio:
topratio = rat
toplev = lev
if toplev == u'*':
toplev = "MED"
elif toplev == u'+':
toplev = "HIGH"
elif toplev == u'#':
toplev = "LOW"
data[i][4].value = toplev
except:
pass
if not data[i][6] is None and levels:
try:
topratio = 0.0
toplev = ""
for lev in self.levels[self.lang]:
toplev = "MED"
for lev in self.levels[u"lev"]:
rat = ratio(data[i][6].value, unicode(lev))
if rat > topratio:
topratio = rat
toplev = lev
if toplev == u'*':
toplev = "MED"
elif toplev == u'+':
toplev = "HIGH"
elif toplev == u'#':
toplev = "LOW"
data[i][6].value = toplev
except:
pass
Expand Down Expand Up @@ -730,6 +752,20 @@ def __init__(self, settings):
datapath = (settings.app_path + os.sep + "letters.xml").encode(sys.getfilesystemencoding())
self.nnetwork.load(datapath, "OCRMLP")
self.classdict = dict((v,k.decode("utf-8")) for k,v in self.revclassdict.iteritems())

class TrainedDataLevel():
def __init__(self, settings):
self.revclassdict = {"*":0,"+":1,"#":2}
self.keys = len(self.revclassdict)
layers = np.array([400,71,self.keys])
self.nnetwork = cv2.ANN_MLP(layers, 1,0.65,1)
datapath = (settings.storage_path + os.sep + "user_level.xml").encode(sys.getfilesystemencoding())
if isfile(datapath):
self.nnetwork.load(datapath, "OCRMLP")
else:
datapath = (settings.app_path + os.sep + "level.xml").encode(sys.getfilesystemencoding())
self.nnetwork.load(datapath, "OCRMLP")
self.classdict = dict((v,k.decode("utf-8")) for k,v in self.revclassdict.iteritems())

class TrainedDataStation():
def __init__(self, settings):
Expand Down
11 changes: 7 additions & 4 deletions export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,10 +164,13 @@ def bpcExport(self):
# QMessageBox.warning(self.parent,"System and station names identical", "There are rows where system and station names are identical. Export aborted.")
# return
timescreenshot = datetime.strptime(row[9],"%Y-%m-%dT%H:%M:%S+00:00")
if allowedtime > timescreenshot:
QMessageBox.warning(self.parent,"Too old for BPC", "You have been using at least one screenshot which is too old. BPC format only allows screenshots younger than 2 hours. Export aborted.")
return


Copy link
Copy Markdown
Contributor

@Marginal Marginal Jan 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why remove the age limit restriction?

Copy link
Copy Markdown
Author

@deflektor deflektor Jan 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am using BPC offline for the route optimization. And I am not instantly OCR all screenshots. So in case I have data from last day, I still can export and import to BPC, without messing with the time-stamps of the files.


#if allowedtime > timescreenshot:
# QMessageBox.warning(self.parent,"Too old for BPC", "You have been using at least one screenshot which is too old. BPC format only allows screenshots younger than 2 hours. Export aborted.")
# return


bpc_format.append([unicode(id)]+row)

self.exportToCsv(bpc_format, file)
Expand Down
16 changes: 10 additions & 6 deletions learningwizard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def deleteUserImages(self):
def showSummary(self):
summary = ""
userdata = {}
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '*', '+', '#']
for word in self.words:
for letter in word:
if letter[1] in characters:
Expand All @@ -107,19 +107,23 @@ def trainOCR(self):
testnumbers = self.getRandomData(alldata,[',', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
testletters = self.getRandomData(alldata,["'", ',', '-', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'])
teststation = self.getRandomData(alldata,["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'])
testlevel = self.getRandomData(alldata,['*', '+', '#'])
self.movie = QMovie(":/ico/loader.gif")
self.loader.setMovie(self.movie)
self.movie.start()
self.numberstrainerthread = Trainer(self, "numbers", self.base, self.user, testnumbers, testletters, teststation)
self.letterstrainerthread = Trainer(self, "letters", self.base, self.user, testnumbers, testletters, teststation)
self.stationtrainerthread = Trainer(self, "station", self.base, self.user, testnumbers, testletters, teststation)
self.numberstrainerthread = Trainer(self, "numbers", self.base, self.user, testnumbers, testletters, teststation, testlevel)
self.letterstrainerthread = Trainer(self, "letters", self.base, self.user, testnumbers, testletters, teststation, testlevel)
self.stationtrainerthread = Trainer(self, "station", self.base, self.user, testnumbers, testletters, teststation, testlevel)
self.leveltrainerthread = Trainer(self, "level" , self.base, self.user, testnumbers, testletters, teststation, testlevel)
QObject.connect(self.numberstrainerthread, SIGNAL('finished(QString, int)'), self.stepFinished)
QObject.connect(self.letterstrainerthread, SIGNAL('finished(QString, int)'), self.stepFinished)
QObject.connect(self.stationtrainerthread, SIGNAL('finished(QString, int)'), self.stepFinished)
QObject.connect(self.leveltrainerthread , SIGNAL('finished(QString, int)'), self.stepFinished)
#QObject.connect(self.trainerthread, SIGNAL('finishedall(int)'), self.trainingFinished)
self.numberstrainerthread.execute()
self.letterstrainerthread.execute()
self.stationtrainerthread.execute()
self.leveltrainerthread.execute()
self.training_summary.setText("Training in progress")

def trainingFinished(self):
Expand All @@ -143,7 +147,7 @@ def stepFinished(self, value, error):

def connectData(self):
connected = {}
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '*', '+', '#']

if self.user is None:
self.user = {}
Expand Down Expand Up @@ -180,7 +184,7 @@ def getRandomData(self, data, characters):
return self.testsamples

def saveImgData(self):
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
characters = ["'", ',', '-', '&', '[', ']', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '*', '+', '#']

if self.user is None:
self.user = {}
Expand Down
Loading