From feaa82402a712fbbc643907624ea45e63e7a128f Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:17:32 +0800 Subject: [PATCH 01/10] Change py2 command to py3 --- Makefile | 2 +- bin/pdf2md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f91c688..c1dfeb4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: @echo 'please run "make publish" to publish' publish: - python2 setup.py sdist bdist_wheel + python3 setup.py sdist bdist_wheel twine upload dist/* clean: diff --git a/bin/pdf2md b/bin/pdf2md index 110a1de..3966ae3 100755 --- a/bin/pdf2md +++ b/bin/pdf2md @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 import sys import os From 2ae56f97bbd053b6e6d3afd6fd3dfe600b9b51e5 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:20:42 +0800 Subject: [PATCH 02/10] Fix py3 print parenthesis --- bin/pdf2md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/pdf2md b/bin/pdf2md index 3966ae3..4925d88 100755 --- a/bin/pdf2md +++ b/bin/pdf2md @@ -8,10 +8,10 @@ def main(argv): if len(argv) == 2: filename = argv[1] title = os.path.splitext(os.path.basename(filename))[0] - print 'Parsing', filename + print('Parsing', filename) else: - print 'usage:' - print ' python main.py ' + print('usage:') + print(' python main.py ') return parser = pdf2md.Parser(filename) @@ -26,7 +26,7 @@ def main(argv): writer.set_title(title) writer.write(piles) - print 'Your markdown is at', writer.get_location() + print('Your markdown is at', writer.get_location()) if __name__ == '__main__': main(sys.argv) From 674405a01540f36b229a29863fdd5a098a6dcdae Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:22:02 +0800 Subject: [PATCH 03/10] Fix py3 relative import --- bin/pdf2md | 5 +++++ pdf2md/__init__.py | 8 ++++---- pdf2md/parser.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/pdf2md b/bin/pdf2md index 4925d88..e7bf946 100755 --- a/bin/pdf2md +++ b/bin/pdf2md @@ -2,6 +2,11 @@ import sys import os + +PACKAGE_PARENT = '..' +SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))) +sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT))) + import pdf2md def main(argv): diff --git a/pdf2md/__init__.py b/pdf2md/__init__.py index bc259ab..6d99071 100644 --- a/pdf2md/__init__.py +++ b/pdf2md/__init__.py @@ -1,4 +1,4 @@ -from parser import Parser -from writer import Writer -from syntax import BaseSyntax -from syntax import UrbanSyntax +from .parser import Parser +from .writer import Writer +from .syntax import BaseSyntax +from .syntax import UrbanSyntax diff --git a/pdf2md/parser.py b/pdf2md/parser.py index 7ca8ce5..e3c6e31 100644 --- a/pdf2md/parser.py +++ b/pdf2md/parser.py @@ -6,7 +6,7 @@ from pdfminer.pdfdevice import PDFDevice from pdfminer.layout import LAParams from pdfminer.converter import PDFPageAggregator -from pile import Pile +from .pile import Pile class Parser(object): From a5e628bb5167ac15b2ea846305d228d6c731bc7f Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:24:29 +0800 Subject: [PATCH 04/10] Update help description --- bin/pdf2md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/pdf2md b/bin/pdf2md index e7bf946..d544ebd 100755 --- a/bin/pdf2md +++ b/bin/pdf2md @@ -10,15 +10,15 @@ sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT))) import pdf2md def main(argv): - if len(argv) == 2: - filename = argv[1] - title = os.path.splitext(os.path.basename(filename))[0] - print('Parsing', filename) - else: + if len(argv) < 2 or argv[1] in ['-h', '--help']: print('usage:') - print(' python main.py ') + print(' {} '.format(argv[0])) return + filename = argv[1] + title = os.path.splitext(os.path.basename(filename))[0] + print('Parsing', filename) + parser = pdf2md.Parser(filename) parser.extract() piles = parser.parse() From 52c86b91a552ead983c55f6a5bcaa009f848b8d7 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:54:09 +0800 Subject: [PATCH 05/10] Remove line end space --- pdf2md/pile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index e28b36a..e7e7c97 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -53,7 +53,7 @@ def parse_layout(self, layout): elif type(obj) == LTChar: pass elif type(obj) == LTLine: - pass + pass else: assert False, "Unrecognized type: %s" % type(obj) From c186a33d2fa70cb34eb040b8ddc145d1b5edbf74 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:54:39 +0800 Subject: [PATCH 06/10] Convert py3 filter to list --- pdf2md/pile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index e7e7c97..62e5823 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -202,7 +202,7 @@ def _find_paragraphs(self, tables): paragraphs[idx].texts.append(text) break - paragraphs = filter(None, paragraphs) + paragraphs = list(filter(lambda x: x._empty(), paragraphs)) return paragraphs From 24a35b958472caba337415758a96479a0fb32f0b Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:55:01 +0800 Subject: [PATCH 07/10] Fix is_overlap top equal bottom bug --- pdf2md/pile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index 62e5823..b4c44a9 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -225,7 +225,7 @@ def _get_anything(self): def _is_overlap(self, top, bottom, obj): - assert top > bottom + assert top >= bottom return (bottom - self._SEARCH_DISTANCE) <= obj.y0 <= (top + self._SEARCH_DISTANCE) or \ (bottom - self._SEARCH_DISTANCE) <= obj.y1 <= (top + self._SEARCH_DISTANCE) From 6791abf93da7c2aa79ab3e7cd4ae87957bcae271 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:55:43 +0800 Subject: [PATCH 08/10] Remove encode to utf8 --- pdf2md/pile.py | 4 ++-- pdf2md/syntax.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index b4c44a9..f118772 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -99,7 +99,7 @@ def gen_html(self): 'height': text.y1 - text.y0, 'x': text.x0, 'y': text.y0, - 'text': text.get_text().encode('utf8'), + 'text': text.get_text(), 'fill': 'green', } html += rect.format(**info) @@ -411,7 +411,7 @@ def _create_tag(self, tag_name, start, level): def _create_td_tag(self, cell): indent = '\t' * 2 - texts = [text.get_text().encode('utf8').strip() for text in cell['texts']] + texts = [text.get_text().strip() for text in cell['texts']] texts = ' '.join(texts) colspan = ' colspan={}'.format(cell['colspan']) if 'colspan' in cell else '' rowspan = ' rowspan={}'.format(cell['rowspan']) if 'rowspan' in cell else '' diff --git a/pdf2md/syntax.py b/pdf2md/syntax.py index a430b6c..9784cbd 100644 --- a/pdf2md/syntax.py +++ b/pdf2md/syntax.py @@ -18,7 +18,7 @@ def newline(self, text): def purify(self, text): - return text.get_text().encode('utf8').strip() + return text.get_text().strip() class UrbanSyntax(BaseSyntax): @@ -27,7 +27,7 @@ def __init__(self): def pattern(self, text): - content = text.get_text().encode('utf8').strip() + content = text.get_text().strip() if not content: return 'none' @@ -62,7 +62,7 @@ def pattern(self, text): return 'plain-text' def newline(self, text): - content = text.get_text().encode('utf8').strip() + content = text.get_text().strip() if text.x0 < 90.1: # special case for neihu page 2 return True @@ -78,7 +78,7 @@ def newline(self, text): def purify(self, text): - content = text.get_text().encode('utf8').strip() + content = text.get_text().strip() mo = re.match('(一|二|三|四|五|六|七|八|九|十)、(.*)', content) if mo: From 701bdb070176a5243169e68f26d7b06e2d9b7ae7 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:56:18 +0800 Subject: [PATCH 09/10] Fix split_piles empty error --- pdf2md/pile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index f118772..1d4e738 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -63,7 +63,9 @@ def split_piles(self): paragraphs = self._find_paragraphs(tables) images = self._find_images() - piles = sorted(tables + paragraphs + images, reverse=True, key=lambda x: x._get_anything().y0) + piles = tables + paragraphs + images + piles = list(filter(lambda x: x._empty(), piles)) + piles = sorted(piles, reverse=True, key=lambda x: x._get_anything().y0) return piles @@ -426,4 +428,6 @@ def _calc_coordinates(self, axes, attr, reverse): coor_list.sort(reverse=reverse) return coor_list + def _empty(self): + return bool(self.images) or bool(self.texts) From b1020c70864ea57062443bdd84ba34e266eea8c6 Mon Sep 17 00:00:00 2001 From: Allen Guan Date: Fri, 5 Jun 2020 15:56:27 +0800 Subject: [PATCH 10/10] Fix index error --- pdf2md/pile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdf2md/pile.py b/pdf2md/pile.py index 1d4e738..7901776 100644 --- a/pdf2md/pile.py +++ b/pdf2md/pile.py @@ -352,9 +352,9 @@ def _is_ignore_cell(self, left, top, right, bottom): def _find_exist_coor(self, minimum, maximum, start_idx, line_coor, direction): - span = 0 + span = -1 line_exist = False - while not line_exist: + while not line_exist and start_idx+span+1 < len(line_coor): span += 1 coor = line_coor[start_idx + span] line_exist = self._line_exists(coor, minimum, maximum, direction)