# Configurations for csv2sql

# You can replace any required configuration below

# You can edit the other configurations and change their default values to improve performance

# (However this may lead to crashes or unexpected behaviour)

# The location were the generated schema file will be stored, default to the csv2sql_source_csv_directory

export csv2sql_schema_file_path=

# The source directory where the csvs are located, default to the current directory

export csv2sql_source_csv_directory=

# The directory were the csvs will be moved after importing to database, defaults to the <csv2sql_source_csv_directory>/imported

export csv2sql_imported_csv_directory=

# The directory were the csvs will be moved after they are validated, defaults to the <csv2sql_source_csv_directory>/validated

export csv2sql_validated_csv_directory=

# Set whether you want to make schema file

export csv2sql_set_make_schema="true"

# Set whether you want to insert the schema file that is generated (this option will only work if set_make_schema="true")

export csv2sql_set_insert_schema="true"

# Set whether to insert the CSVs into the database

export csv2sql_set_insert_data="true"

# Set whether to validate if the csvs have been inserted correctly

export csv2sql_set_validate="true"

# mysql username, required field if database access is required

export csv2sql_username=

# mysql password, required field if database access is required

export csv2sql_password=

# mysql host

export csv2sql_host="localhost"

# This is the name of the database which will be created (if not present already), required field if database access is required

export csv2sql_database_name=

# The socket file location for mysql

export csv2sql_socket="/var/run/mysqld/mysqld.sock"

# The character limit after which a db field becomes a text from a varchar

export csv2sql_varchar_limit=100

# The chunk size to use when the schema for a csv will be infered parallely .

# For example: A chunk size 100 means the csv will be read 100 rows at a time

# and separate processes will be used to infer the schema of each 100 row chunk

export csv2sql_schema_infer_chunk_size=100

# The number of workers

# Increasing worker count may result in errors

export csv2sql_worker_count=10

# The number of database workers

export csv2sql_db_worker_count=15

# Number of records to insert into the database at once

# Increasing this may result in mysql error for too many placeholders

export csv2sql_insertion_chunk_size=100

# Number of chunks to keep in memory (Memory required=insertion_chunk_size * job_count_limit)

export csv2sql_job_count_limit=10

# When false, does not log the query, possible values are false, debug, info, warn

export csv2sql_log=false

# The time in milliseconds to wait for the query call to finish

# (default: 15000)

export csv2sql_timeout=60000

# The number of seconds that the mysqld server waits for a connect packet before responding with Bad handshake

# The timeout for establishing new connections

export csv2sql_connect_timeout=60000

# The pool_size controls how many connections you want to the database.

# The advice of starting one per core is good for CPU work.

# Connections however do a lot of IO work so its recommende a minimum of at least 10.

# To find the ideal number you need to monitor and measure how your queues are behaving.

# A higher number is not necessarily better, as it implies in less re-use between connections and database overload.

export csv2sql_pool_size=20

# Handling requests is done through a queue. When DBConnection is started,

# there are two relevant options to control the queue:

# Our goal is to wait at most :queue_target for a connection.

# If all connections checked out during a :queue_interval takes more than :queue_target, then we double the :queue_target.

# If checking out connections take longer than the new target, then we start dropping messages.

export csv2sql_queue_target=5000

export csv2sql_queue_interval=1000