Merge pull request #74 from nipunsadvilkar/npn-newline-fix

nipunsadvilkar · web-flow · commit 90699972c8b5 · 2020-08-11T18:14:08.000+05:30
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+# v0.3.1
+- 🚑 ✅ Handle Newline character & update tests
+
+# v0.3.0
+-   ✨ 💫  Support Multiple languages - \#2
+-   🏎⚡️💯 Benchmark across Segmentation Tools, Libraries and Algorithms
+-   🎨 ♻️ Update sentence char_span logic
+-   ⚡️  Performance improvements - \#41
+-   ♻️🐛 Refactor AbbreviationReplacer
+
 # v0.3.0rc
 -   ✨ 💫 sent `char_span` through with spaCy & regex approach - \#63
 -   ♻️ Refactoring to support multiple languages
diff --git a/pysbd/about.py b/pysbd/about.py
@@ -2,7 +2,7 @@
 # https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
 
 __title__ = "pysbd"
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 __summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
 __uri__ = "http://nipunsadvilkar.github.io/"
 __author__ = "Nipun Sadvilkar"
diff --git a/pysbd/processor.py b/pysbd/processor.py
@@ -28,6 +28,7 @@ def __init__(self, text, lang, char_span=False):
     def process(self):
         if not self.text:
             return self.text
+        self.text = self.text.replace('\n', '\r')
         li = ListItemReplacer(self.text)
         self.text = li.add_line_break()
         self.replace_abbreviations()
diff --git a/tests/regression/test_issues.py b/tests/regression/test_issues.py
@@ -58,7 +58,13 @@
     ('#55', "She turned to him, \"This is great.\" She held the book out to show him.",
         [
             ('She turned to him, "This is great." ', 0, 36), ('She held the book out to show him.', 36, 70)
-        ])
+        ]),
+('#56',
+"""This eBook is for the use of anyone anywhere at no cost
+you may copy it, give it away or re-use it under the terms of the this license
+""",
+[('This eBook is for the use of anyone anywhere at no cost\n', 0, 56),
+ ('you may copy it, give it away or re-use it under the terms of the this license\n', 56, 135)])
 
 ]
 

Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,13 @@`
`58`	`58`	`('#55', "She turned to him, \"This is great.\" She held the book out to show him.",`
`59`	`59`	`[`
`60`	`60`	`('She turned to him, "This is great." ', 0, 36), ('She held the book out to show him.', 36, 70)`
`61`		`- ])`
	`61`	`+ ]),`
	`62`	`+('#56',`
	`63`	`+"""This eBook is for the use of anyone anywhere at no cost`
	`64`	`+you may copy it, give it away or re-use it under the terms of the this license`
	`65`	`+""",`
	`66`	`+[('This eBook is for the use of anyone anywhere at no cost\n', 0, 56),`
	`67`	`+ ('you may copy it, give it away or re-use it under the terms of the this license\n', 56, 135)])`
`62`	`68`
`63`	`69`	`]`
`64`	`70`