ner-bert/modules/utils.py at master · flypythoncom/ner-bert · GitHub

70 lines (54 loc) · 1.55 KB

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

import os

import json

import numpy

import bson

import sys

def ipython_info():

ip = False

if 'ipykernel' in sys.modules:

ip = 'notebook'

elif 'IPython' in sys.modules:

ip = 'terminal'

return ip

def get_tqdm():

ip = ipython_info()

if ip == "terminal" or not ip:

from tqdm import tqdm

return tqdm

else:

try:

from tqdm import tqdm_notebook

return tqdm_notebook

except:

from tqdm import tqdm

return tqdm

class JsonEncoder(json.JSONEncoder):

def default(self, obj):

if isinstance(obj, numpy.integer):

return int(obj)

elif isinstance(obj, numpy.floating):

return float(obj)

elif isinstance(obj, numpy.ndarray):

return obj.tolist()

elif isinstance(obj, bson.ObjectId):

return str(obj)

else:

return super(JsonEncoder, self).default(obj)

def jsonify(data):

return json.dumps(data, cls=JsonEncoder)

def read_config(config):

if isinstance(config, str):

with open(config, "r", encoding="utf-8") as f:

config = json.load(f)

return config

def save_config(config, path):

with open(path, "w") as file:

json.dump(config, file, cls=JsonEncoder)

def if_none(origin, other):

return other if origin is None else origin

def get_files_path_from_dir(path):

f = []

for dir_path, dir_names, filenames in os.walk(path):

for f_name in filenames:

f.append(dir_path + "/" + f_name)

return f