"""Load lookup-table files with word corrections and tags into one dictionary

# Check input argument type - - - - - - - - - - - - - - - - - - - - - - - - -

#

if (type(file_names) == types.StringType):

file_names = [file_names] # Make a list out of a single file name

if (type(file_names) != types.ListType):

msg = ['Input argument is of wrong type: '+file_names, \

'Must be either of type "string" or "list"']

log_message(msg,'err')

raise Exception()

dict = {} # New empty dictionary

max_len = 1 # Maximal length of the longest key sequence

# Loop over file names - - - - - - - - - - - - - - - - - - - - - - - - - - -

#

for fn in file_names:

# Open file and read all lines into a list

#

try:

f = open(fn,'r')

except:

log_message('Cannot open file: '+fn,'err')

raise IOError()

file_data = f.readlines()

f.close()

tag = '' # Start with no tag

# Now process all lines - - - - - - - - - - - - - - - - - - - - - - - - - -

#

for line in file_data:

l = line.strip() # Remove line separators

if (len(l) > 0) and (l[0] != '#'): # Not empty line and no comment line

if (l[:5] == 'tag=<'): # It's a line with a new tag

tag = l[5:7]

if (tag not in config.name_hmm_obser) and \

(tag not in config.geoloc_hmm_obser): # Make sure tag is valid

log_message('Illegal tag: '+tag,'err')

raise Exception()

else: # A line with an entry

if (tag == ''): # make sure a tag is set

log_message('No tag set in file: '+fn,'err')

raise Exception()

ll = l.split(':') # Separate key from values

if (len(ll) == 2): # Line contains a key - - - - - - - - - - - - - -

k = ll[0].strip().lower() # Get key, make lower and strip spaces

k_list = k.split(' ') # Make a list of key words

if (len(k_list) > max_len): # Update maximal key sequence length

max_len = len(k_list)

val = string.join(k_list,'_')

key = tuple(k_list)

this_tag = tag

if (k != ''): # If key is non-empty insert it into dictionary

if (dict.has_key(key)):

test_val = dict[key][0] # Value without tag

test_tag = dict[key][1]

if (val == test_val): # Same values

if (test_tag.find(this_tag) < 0): # This tag is not in tags

this_tag = test_tag+'/'+this_tag

else:

msg = ['Key already in dictionary with different value', \

'Key: "'+str(key)+'", old value: "'+ \

str(dict[key][0])+'", new value: "'+str(val)+'"']

log_message(msg,'err')

raise Exception()

this_val = (val, this_tag)

dict.update({key:this_val}) # Insert key itself into dicionary

v = ll[1].lower() # Get values in a string

elif (len(ll) == 1): # Line contains only values - - - - - - - - - -

v = ll[0].lower() # Get values in a string

else:

log_message('Illegal file format in file: '+fn+', line: '+l,'err')

raise Exception()

vv = v.split(',') # Split values into a list

for v in vv: # Loop over all values - - - - - - - - - - - - - - - -

vs = v.strip()

if (vs != ''): # Only append non-empty values

k_list = vs.split(' ') # Make a list of key words

if (len(k_list) > max_len): # Update maximal key sequence length

max_len = len(k_list)

key = tuple(k_list)

this_tag = tag

if (dict.has_key(key)):

test_val = dict[key][0] # Value without tag

test_tag = dict[key][1]

if (val == test_val): # Same values

if (test_tag.find(this_tag) < 0): # This tag is not in tags

this_tag = test_tag+'/'+this_tag

else:

msg = ['Key already in dictionary with different value', \

'Key: "'+str(key)+'", old value: "'+ \

str(dict[key][0])+'", new value: "'+str(val)+'"']

log_message(msg,'err')

raise Exception()

this_val = (val, this_tag)

dict.update({key:this_val}) # Insert key itself into dicionary

"""Load one correction list file into a sorted list

# Check input argument type and open file - - - - - - - - - - - - - - - - - -

#

if (type(file_name) != types.StringType):

log_message('File name is not of type "string": '+file_name,'err')

raise Exception()

try:

f = open(file_name,'r')

except:

log_message('Cannot open file: '+file_name,'err')

raise IOError()

file_data = f.readlines()

f.close()

org_list = [] # List of original strings (the ones to be replaced)

repl_list = [] # List of replacement strings

len_list = [] # List of original string lengths

repl = '' # Set inital replacement to nothing

# Now process all lines - - - - - - - - - - - - - - - - - - - - - - - - - - -

#

for line in file_data:

l = line.strip() # Remove line separators at the end

if (len(l) > 0) and (l[0] != '#'): # Not an empty line and no comment line

ll = l.split(':=') # Separate replacement from values

if (len(ll) == 2): # Line contains a replacement - - - - - - - - - - - -

repl = ll[0].strip().lower() # Make replacement lower and strip spaces

if (not ((repl[0] == '"') and (repl[-1] == '"') or \

(repl[0] == "'") and (repl[-1] == "'"))):

log_message(['Replacement string is not properly quoted: '+repl, \

'In file: '+file_name], 'err')

raise Exception()

repl = repl[1:-1] # Remove quotes from replacement string

v = ll[1].lower() # Get values in a string and make lowercase

elif (len(ll) == 1): # Line contains only values - - - - - - - - - - - -

v = ll[0].lower() # Get values in a string and make lowercase

else: # More than one ':=' separator in the line - - - - - - - - - - - -

log_message('Too many ":=" separators in this line: '+l, 'err')

raise Exception()

# Now process the values and append them to the list - - - - - - - - - -

vv = v.split(',') # Split values into a list

for v in vv: # Loop over all values - - - - - - - - - - - - - - - - -

org = v.strip() # Get the original string

if (org != ''): # Only process non-empty values

if (not ((org[0] == '"') and (org[-1] == '"') or \

(org[0] == "'") and (org[-1] == "'"))):

log_message(['Original string is not properly quoted: '+org, \

'In file: '+file_name], 'err')

raise Exception()

org = org[1:-1] # Remove quotes from original string

if (org != ''): # Only append non-empty values

org_list.append(org)

repl_list.append(repl)

len_list.append(len(org))

tmp_list = map(None,len_list,org_list,repl_list)

tmp_list.sort()

tmp_list.reverse()

corr_list = []

for (i,org,repl) in tmp_list:

corr_list.append((org,repl))

msg = ['Loaded "'+file_name+'" into correction list"'] # Prepare log message

for i in corr_list:

msg.append(' '+str(i))

log_message(msg,'v2')

"""Load one or more comma separated text files into a lookup-dictionary.

# Check input argument - - - - - - - - - - - - - - - - - - - - - - - - - - -

#

if (type(file_names) == types.StringType):

file_names = [file_names] # Make a list out of a single file name

if (type(file_names) != types.ListType):

msg = ['Input argument is of wrong type: '+file_names, \

'Must be either of type string or list']

log_message(msg,'err')

raise TypeError(load_dict)

dict = {} # New empty dictionary

# Loop over file names - - - - - - - - - - - - - - - - - - - - - - - - - - -

#

for fn in file_names:

# Open file and read all lines into a list

#

try:

f = open(fn,'r')

except:

log_message('Cannot open file: '+fn,'err')

raise IOError(load_dict)

file_data = f.readlines()

f.close()

# Now process all lines - - - - - - - - - - - - - - - - - - - - - - - - - -

#

for line in file_data:

l = lline.strip()

if (len(l) > 0) and (l[0] != '#'): # Not empty line and no comment line

ll = l.split(',') # Get fields from a line

if (len(ll) != 2): # Not two columns

msg = 'Illegal file format (not 2 columns) in file: '+fn+', line: '+l

log_message(msg,'err')

raise Exception(load_dict)

k = ll[0].strip().lower() # Make sure it's lower case

v = ll[1].strip().lower()

if (dict.has_key(k)):

if (dict[k] != v):

msg = ['Key '+v+' already in dictionary with different value.', \

'(old value will be over written)', \

'Old value: '+dict[v]+', new value: '+k]

log_message(msg, 'warn')

dict.update({k:v})

else: # New key

dict.update({k:v})

"""Process a text line (one record) according to settings from config.py

name_comp_list = []

givenname_comp_list = []

surname_comp_list = []

geocode_comp_list = []

locality_comp_list = []

date1_comp_list = []

date2_comp_list = []

# Split the line into the basic fields - - - - - - - - - - - - - - - - - - -

#

if (config.in_file_type in ['CSV','CSVQ','TAB','TABQ']):

# Comma or tabulator separated

try:

line_list = config.line_parser.parse(line)

except:

log_message('CSV line parsing failed with inout: '+line,'err')

if (len(line_list) < config.input_len):

log_message('Input line does not contain enough fields,' +\

'fill up with empty fields','warn')

while (len(line_list) < config.input_len):

line_list.append('')

config.curr_line_list = line_list # Save current line list

# Extract fields into different component lists - - - - - - - - - - - - - -

#

if (config.input_component['name'] != []): # Extract name fields

for i in config.input_component['name']:

name_comp_list.append(line_list[i])

else: # Extract givenname and surname into separate components - - - - - -

if (config.input_component['givenname'] != []): # Extract g-name fields

for i in config.input_component['givenname']:

givenname_comp_list.append(line_list[i])

if (config.input_component['surname'] != []): # Extract surname fields

for i in config.input_component['surname']:

surname_comp_list.append(line_list[i])

if (config.input_component['geocode'] != []): # Extract geocode fields

for i in config.input_component['geocode']:

geocode_comp_list.append(line_list[i])

if (config.input_component['locality'] != []): # Extract locality fields

for i in config.input_component['locality']:

locality_comp_list.append(line_list[i])

if (config.input_component['date1'] != []): # Extract date1 fields

for i in config.input_component['date1']:

date1_comp_list.append(line_list[i])

if (config.input_component['date2'] != []): # Extract date2 fields

for i in config.input_component['date2']:

date2_comp_list.append(line_list[i])

elif (config.in_file_type == 'COL'): # Column based input file - - - - - - -

if (len(line) < config.input_len):

log_message('Input line is not long enough, fill up with spaces','warn')

line += ' '*(config.input_len-len(line))

if (config.input_component['name'] != []): # Extract name fields

for (col_start,length) in config.input_component['name']:

name_comp_list.append(line[col_start,col_start+length])

else: # Extract givenname and surname into separate components - - - - - -

if (config.input_component['givenname'] != []): # Extract g-name fields

for (col_start,length) in config.input_component['givenname']:

givenname_comp_list.append(line[col_start,col_start+length])

if (config.input_component['surname'] != []): # Extract surname fields

for (col_start,length) in config.input_component['surname']:

surname_comp_list.append(line[col_start,col_start+length])

if (config.input_component['geocode'] != []): # Extract geocode fields

for (col_start,length) in config.input_component['geocode']:

geocode_comp_list.append(line[col_start,col_start+length])

if (config.input_component['locality'] != []): # Extract locality fields

for (col_start,length) in config.input_component['locality']:

locality_comp_list.append(line[col_start,col_start+length])

if (config.input_component['date1'] != []): # Extract date1 fields

for (col_start,length) in config.input_component['date1']:

date1_comp_list.append(line[col_start,col_start+length])

if (config.input_component['date2'] != []): # Extract date2 fields

for (col_start,length) in config.input_component['date2']:

date2_comp_list.append(line[col_start,col_start+length])

# elif (config.in_file_type == 'SQL'): # - - - - - - - - - - - - - - - - - -

################################

# Add later: SQL database access

################################

msg = [' Component basic field lists:', \

' Name: '+str(name_comp_list), \

' Given name: '+str(givenname_comp_list), \

' Surname: '+str(surname_comp_list), \

' Geocode: '+str(geocode_comp_list), \

' Locality: '+str(locality_comp_list), \

' Date1: '+str(date1_comp_list), \

' Date2: '+str(date2_comp_list)]

log_message(msg,'v2')

name_comp = ''

givenname_comp = ''

surname_comp = ''

geocode_comp = ''

locality_comp = ''

date1_comp = ''

date2_comp = ''

# Now clean and then concatenate component lists into strings - - - - - - - -

#

if (name_comp_list != []): # Name component

name_comp = name_comp_list[0] # Start with first field in list

for f in name_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['name'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['name'] == 1):

sep = check_field_spill(name_comp, f)

name_comp = name_comp+sep+f # Append separator and field

if (givenname_comp_list != []): # Givenname component - - - - - - - - - - -

givenname_comp = givenname_comp_list[0] # Start with first field in list

for f in givenname_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['givenname'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['givenname'] == 1):

sep = check_field_spill(givenname_comp, f)

givenname_comp = givenname_comp+sep+f # Append separator and field

if (surname_comp_list != []): # Surname component - - - - - - - - - - - - -

surname_comp = surname_comp_list[0] # Start with first field in list

for f in surname_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['surname'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['surname'] == 1):

sep = check_field_spill(surname_comp, f)

surname_comp = surname_comp+sep+f # Append separator and field

if (geocode_comp_list != []): # Geocode component - - - - - - - - - - - - -

geocode_comp = geocode_comp_list[0] # Start with first field in list

for f in geocode_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['geocode'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['geocode'] == 1):

sep = check_field_spill(geocode_comp, f)

geocode_comp = geocode_comp+sep+f # Append separator and field

if (locality_comp_list != []): # Locality component - - - - - - - - - - - -

locality_comp = locality_comp_list[0] # Start with first field in list

for f in locality_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['locality'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['locality'] == 1):

sep = check_field_spill(locality_comp, f)

locality_comp = locality_comp+sep+f # Append separator and field

if (date1_comp_list != []): # Date1 component - - - - - - - - - - - - - - -

date1_comp = date1_comp_list[0] # Start with first field in list

for f in date1_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['date1'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['date1'] == 1):

if (date1_comp[-1] != ' ') and (f[0] != ' '):

tmp_list0 = date1_comp.split()

tmp_list1 = f.split()

check_word = tmp_list0[-1]+tmp_list1[0]

if (check_word in ['jan','feb','mar','apr','may','jun','jul','aug', \

'sep','oct','nov','dec','january','february','march','april', \

'may','june','july','august','september','october','november', \

'december']):

sep = '' # Set separator to no space

msg = ' Correct date1 word spilling: "'+date1_comp+'","'+f+'"'

log_message(msg,'v1')

date1_comp = date1_comp+sep+f # Append separator and field

if (date2_comp_list != []): # Date2 component - - - - - - - - - - - - - - -

date2_comp = date2_comp_list[0] # Start with first field in list

for f in date2_comp_list[1:]: # Loop over following fields (if any)

if (f != ''):

if (config.input_space_sep['date2'] == 1):

sep = ' ' # Set separator to space between fields

else:

sep = '' # No space between fields

# Check field spilling only if space separator is set to ' '

#

if (sep == ' ') and (config.input_check_spilling['date2'] == 1):

if (date2_comp[-1] != ' ') and (f[0] != ' '):

tmp_list0 = date1_comp.split()

tmp_list1 = f.split()

check_word = tmp_list0[-1]+tmp_list1[0]

if (check_word in ['jan','feb','mar','apr','may','jun','jul','aug', \

'sep','oct','nov','dec','january','february','march','april', \

'may','june','july','august','september','october','november', \

'december']):

sep = '' # Set separator to no space

msg = ' Correct date1 word spilling: "'+date1_comp+'","'+f+'"'

log_message(msg,'v1')

date2_comp = date2_comp+sep+f # Append separator and field

# Check if name component is given or givenname and surname separately - - -

#

if (config.input_component['givenname'] != []) or \

(config.input_component['surname'] != []):

name_comp = [givenname_comp, surname_comp]

msg = [' Components:', \

' Name: "'+str(name_comp)+'"', \

' Geocode: "'+geocode_comp+'"', \

' Locality: "'+locality_comp+'"', \

' Date1: "'+date1_comp+'"', \

' Date2: "'+date2_comp+'"']

log_message(msg,'v1')

"""Compose a text line (one record) for the output file.

line = ''

# Loop over all activated output fields - - - - - - - - - - - - - - - - - - -

#

for (field_pos,field_name) in config.output_field_list:

if (header == 1): # This is a header line, field values are field names

field = output_dict[field_name]

elif (field_name == 'record_id'): # Add current record (line) number

field = str(config.curr_line_no)

elif (field_name[:14] == 'original_input'): # Output original input

if (len(field_name) == 14):

field = config.curr_line # The complete input line (without line sep.)

else: # There is an input field or input column range given

v = field_name[14:].strip() # Get the value

v = v[1:-1] # Remove brackets '[' and ']' from beginning and end

if (v[0] == '(') and (v[-1] == ')'): # It's a tuple

v = v[1:-1] # Remove tuple brackets

v = v.split(',') # Make a list

for i in range(len(v)):

v[i] = int(v[i]) # Make integers

if (len(v) == 1): # A input field number

field = config.curr_line_list[v[0]]

else: # Length must be two, for column start and end

field = config.curr_line(v[0],v[1]+1)

elif (output_dict.has_key(field_name)):

field = output_dict[field_name] # Get current output field

if (field == []):

field = '' # Make all empty fields to empty strings

elif (type(field) == types.ListType):

field = string.join(field,' ') # Convert into string (space separator)

if (type(field) != types.StringType): # Illegal field type

log_message('Output field is of illegal type: '+str(field),'err')

raise Exception()

else: # No current output field

field = '' # Create an empty output field

# Now append current field to output line - - - - - - - - - - - - - - - - -

#

if (config.out_file_type in ['CSV','TAB']): # Comma or tabulator separated

if (',' in field): # The field contains a comma, quote it

field = config.output_quote_character + field + \

config.output_quote_character

line = line + field + config.out_field_sep

if (config.out_file_type in ['CSVQ','TABQ']): # Quoted comma or tabulator

line = line + config.output_quote_character + field + \

config.output_quote_character + config.out_field_sep

elif (config.out_file_type == 'COL'):

[(start_col, field_len)] = field_pos

if (len(field) > field_len): # Field content is too long

log_message('Field content too long: '+str(field)+' (max '+ \

str(field_len)+' characters)', 'warn')

field = field.ljust(field_len) # Expand field with spaces

line = line + field[:field_len] # Append field to line, cut trail off if

# too long

# elif (config.out_file_type == 'SQL'):

################################

# Add later: SQL database access

################################

if (config.out_file_type in ['CSV','CSVQ','TAB','TABQ']):

line = line[:-len(config.out_field_sep)] # Remove last field separator

log_message(' Output line: |'+line+'|','v2')

"""Routine to check if a known word is spilling from one string into another

org_str1 = str1 # Keep copies of the original input string

org_str2 = str2

alphanum = string.letters+string.digits # String with all letters and digits

sep = ' ' # Default return value for separator

if (str1 == '') or (str1 == ''): # One input string is empty

pass # Make sure the strings are not empty

# Check if the characters at the 'boundary' are either letters or digits

#

elif (str1[-1] in alphanum) and (str2[0] in alphanum):

test_str1 = ''

while (str1 != '') and (str1[-1] in alphanum): # Get last char. from str1

test_str1 = str1[-1] + test_str1

str1 = str1[:-1]

test_str2 = ''

while (str2 != '') and (str2[0] in alphanum): # Get first char. from str2

test_str2 = test_str2 + str2[0]

str2 = str2[1:]

check_word = test_str1 + test_str2

if (config.name_lookup_dict.has_key((check_word,)) or \

config.geoloc_lookup_dict.has_key((check_word,))):

sep = '' # Set separator to no space

msg = ' Correct name word spilling: "'+org_str1.strip()+ \

'","'+org_str2.strip()+'" -> "'+check_word+'"'

log_message(msg,'v2')

config.num_word_spills += 1 # Increment number of corrected word spills

"""Print and/or append and a message to a log file.

if (config.verbose == 0) and (config.logging == 0) and \

(msg_type not in ['err','warn']):

return None # A normal information message and no verbose output or logging

if (msg_type == 'v1'):

msg_level = 1

msg_type = 'info'

elif (msg_type == 'v2'):

msg_level = 2

msg_type = 'info'

elif (msg_type not in ['err','warn']):

print

print '***** Error **********'

print '***** Illegal message type:', msg_type

raise Exception()

# Create message header for warning and error messages - - - - - - - - - - -

#

if (msg_type == 'err'):

if (config.curr_line_no >= 0):

m = ['', '***** Error! Line number: '+str(config.curr_line_no)]

else:

m = ['', '***** Error!']

if (config.curr_line != ''):

m.append('***** Line: "'+config.curr_line+'"')

elif (msg_type == 'warn'):

config.num_warning += 1 # Increment number of warnings

if (config.curr_line_no >= 0):

m = ['', '***** Warning! Line number: '+str(config.curr_line_no)]

else:

m = ['', '***** Warning!']

if (config.curr_line != ''):

m.append('***** Line: "'+config.curr_line+'"')

else:

m = []

# Make sure message is of type 'list' - - - - - - - - - - - - - - - - - - - -

#

if (type(message) == types.StringType):

message = [message]

elif (type(message) != types.ListType): # Illegal type of the message

print

print '***** Error **********'

print '***** Illegal type of message data:', type(message)

print '***** Must be either of type string or list'

raise TypeError()

for msg in message:

if (msg_type == 'err') or (msg_type == 'warn'):

m.append('***** '+msg)

else:

m.append(msg)

if (msg_type in ['warn','err']) or (config.verbose >= msg_level):

if (config.logging > 0): # Write message to log file

try:

f_log = open(config.log_file, 'a') # Open file for appending

except:

print

print '***** Error **********'

print '***** Cannot open log file:', log_file

raise IOError()

for line in m:

f_log.write(line+os.linesep) # Append message to log file

f_log.close()

if (msg_type != 'warn') or ((msg_type == 'warn') and (config.nowarn == 0)):

for line in m: # Print message

print line