Merge pull request GeneralNewsExtractor#88 from kingname/develop

kingname · web-flow · commit f80ab69686c6 · 2020-09-15T12:36:08.000+08:00
useless_attr 现在必须完全匹配才会移除节点
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # General News Extractor Changelog
 
+## 0.2.3 (2020-09-15)
+
+### Bug fix
+
+1. `USELESS_ATTR`对应的节点，只有 class 完全匹配才需要删除。之前包含就删除的匹配方式会导致 ifeng 的正文被删除。
+
 ## 0.2.2 (2020-08-02)
 
 ### Features
diff --git a/gne/defaults.py b/gne/defaults.py
@@ -57,7 +57,8 @@
 # if one tag in the follow list does not contain any child node nor content, it could be removed
 TAGS_CAN_BE_REMOVE_IF_EMPTY = ['section', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'span']
 
-USELESS_ATTR = ['share',
+USELESS_ATTR = {
+                'share',
                 'contribution',
                 'copyright',
                 'copy-right',
@@ -69,7 +70,7 @@
                 'social',
                 'submeta',
                 'report-infor'
-                ]
+                }
 
 
 HIGH_WEIGHT_ARRT_KEYWORD = ['content',
diff --git a/gne/utils.py b/gne/utils.py
@@ -33,10 +33,9 @@ def normalize_node(element: HtmlElement):
 
         class_name = node.get('class')
         if class_name:
-            for attribute in USELESS_ATTR:
-                if attribute in class_name:
-                    remove_node(node)
-                    break
+            if class_name in USELESS_ATTR:
+                remove_node(node)
+                break
 
 
 def html2element(html):
@@ -47,6 +46,7 @@ def html2element(html):
 
 def pre_parse(element):
     normalize_node(element)
+
     return element
 
 
diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
     name='gne',
     packages=find_packages(exclude=[]),
     install_requires=['lxml', 'numpy', 'pyyaml'],
-    version='0.2.2',
+    version='0.2.3',
     description='General extractor of news pages.',
     long_description=readme,
     long_description_content_type='text/markdown',
diff --git a/tests/ifeng/3.html b/tests/ifeng/3.html
