Add userCommand for pre-defined functions

minjk-bl · minjk-bl · commit 29a3a1ccbbb9 · 2022-02-24T10:55:25.000+09:00
diff --git a/js/com/com_Config.js b/js/com/com_Config.js
@@ -160,7 +160,8 @@ define([
                 'printCommand.py',
                 'fileNaviCommand.py',
                 'pandasCommand.py',
-                'variableCommand.py'
+                'variableCommand.py',
+                'userCommand.py'
             ];
             let promiseList = [];
             libraryList.forEach(libName => {
diff --git a/js/m_apps/PDF.js b/js/m_apps/PDF.js
@@ -30,37 +30,6 @@ import fitz
 import nltk
 nltk.download('punkt')`;
 
-    const PDF_FUNC = `def vp_pdf_get_sentence(fname_lst):
-    '''
-    Get sentence from pdf file by PyMuPDF
-    '''
-    df = pd.DataFrame()
-    for fname in fname_lst:
-        if fname.split('.')[-1] != 'pdf': continue
-        try:
-            doc = fitz.open(fname)
-            sentence_lst = []
-            for page in doc:
-                block_lst = page.get_text('blocks')
-        
-                text_lst = [block[4] for block in block_lst if block[6] == 0]
-                text = '\\n'.join(text_lst)
-        
-                sentence_lst.extend([sentence for sentence in nltk.sent_tokenize(text)])
-                
-            doc.close()
-        except Exception as e:
-            print(e)
-            continue
-            
-        df_doc = pd.DataFrame({
-            'fname': fname.split('/')[-1],
-            'sentence': sentence_lst
-        })
-        df = pd.concat([df,df_doc])
-        
-    return df.reset_index().drop('index', axis=1)`;
-
     const PDF_CMD = 'df = vp_pdf_get_sentence(pdf_lst)\ndf'
     /**
      * PDF
@@ -98,7 +67,6 @@ nltk.download('punkt')`;
             // click import
             $(this.wrapSelector('.vp-pdf-import-btn')).on('click', function () {
                 com_interface.insertCell('code', PDF_IMPORT);
-                com_interface.insertCell('code', PDF_FUNC);
             });
 
             // click file navigation button
diff --git a/python/userCommand.py b/python/userCommand.py
@@ -0,0 +1,54 @@
+import pandas as pd
+import numpy as np
+import fitz
+import nltk
+nltk.download('punkt')
+
+def vp_pdf_get_sentence(fname_lst):
+    '''
+    Get sentence from pdf file by PyMuPDF
+    '''
+    df = pd.DataFrame()
+    for fname in fname_lst:
+        if fname.split('.')[-1] != 'pdf': continue
+        try:
+            doc = fitz.open(fname)
+            sentence_lst = []
+            for page in doc:
+                block_lst = page.get_text('blocks')
+        
+                text_lst = [block[4] for block in block_lst if block[6] == 0]
+                text = '\\n'.join(text_lst)
+        
+                sentence_lst.extend([sentence for sentence in nltk.sent_tokenize(text)])
+                
+            doc.close()
+        except Exception as e:
+            print(e)
+            continue
+            
+        df_doc = pd.DataFrame({
+            'fname': fname.split('/')[-1],
+            'sentence': sentence_lst
+        })
+        df = pd.concat([df,df_doc])
+        
+    return df.reset_index().drop('index', axis=1)
+
+def vp_drop_outlier(df, col, weight=1.5):
+    sr = df[col]
+    
+    q25 = np.percentile(sr.values, 25)
+    q75 = np.percentile(sr.values, 75)
+    
+    iqr   = q75 - q25
+    iqr_w = iqr * weight
+    
+    val_l = q25 - iqr_w
+    val_h = q75 + iqr_w
+    
+    outlier_index = sr[(sr < val_l) | (sr > val_h)].index
+    
+    df_res = df.drop(outlier_index).copy()
+    
+    return df_res
