Skip to content

Feature/csv skip rows #4238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deepsidhu1313 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Feature/csv skip rows #4238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0531f14
CSVParser: add option to skip leading rows
claude Oct 26, 2026
0e12e39
CSV import: add option to skip the first rows of the file
claude Oct 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/ImportCsvDialog.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ ImportCsvDialog::ImportCsvDialog(const std::vector<QString>& filenames, DBBrowse

// Load last used settings and apply them
ui->checkboxHeader->blockSignals(true);
ui->spinBoxSkipRows->blockSignals(true);
ui->checkBoxTrimFields->blockSignals(true);
ui->checkBoxSeparateTables->blockSignals(true);
ui->checkLocalConventions->blockSignals(true);
Expand All @@ -75,6 +76,7 @@ ImportCsvDialog::ImportCsvDialog(const std::vector<QString>& filenames, DBBrowse
ui->checkReplaceNonAlnum->blockSignals(true);

ui->checkboxHeader->setChecked(Settings::getValue("importcsv", "firstrowheader").toBool());
ui->spinBoxSkipRows->setValue(Settings::getValue("importcsv", "skiprows").toInt());
ui->checkBoxTrimFields->setChecked(Settings::getValue("importcsv", "trimfields").toBool());
ui->checkBoxSeparateTables->setChecked(Settings::getValue("importcsv", "separatetables").toBool());
ui->checkLocalConventions->setChecked(Settings::getValue("importcsv", "localconventions").toBool());
Expand All @@ -84,6 +86,7 @@ ImportCsvDialog::ImportCsvDialog(const std::vector<QString>& filenames, DBBrowse
setEncoding(Settings::getValue("importcsv", "encoding").toString());

ui->checkboxHeader->blockSignals(false);
ui->spinBoxSkipRows->blockSignals(false);
ui->checkBoxTrimFields->blockSignals(false);
ui->checkBoxSeparateTables->blockSignals(false);
ui->checkLocalConventions->blockSignals(false);
Expand Down Expand Up @@ -191,6 +194,7 @@ void ImportCsvDialog::accept()
{
// Save settings
Settings::setValue("importcsv", "firstrowheader", ui->checkboxHeader->isChecked());
Settings::setValue("importcsv", "skiprows", ui->spinBoxSkipRows->value());
Settings::setValue("importcsv", "separator", currentSeparatorChar());
Settings::setValue("importcsv", "quotecharacter", currentQuoteChar());
Settings::setValue("importcsv", "trimfields", ui->checkBoxTrimFields->isChecked());
Expand Down Expand Up @@ -411,7 +415,9 @@ CSVParser::ParserResult ImportCsvDialog::parseCSV(const QString &fileName, std::
tstream.setCodec(currentEncoding().toUtf8());
#endif

return csv.parse(rowFunction, tstream, count);
// Rows which are to be skipped are not passed to the row function. The row numbers it receives start at 0 after the skipped rows, so
// the header row handling is always relative to the first row after the skipped ones.
return csv.parse(rowFunction, tstream, count, static_cast<size_t>(ui->spinBoxSkipRows->value()));
}

sqlb::FieldVector ImportCsvDialog::generateFieldList(const QString& filename) const
Expand Down Expand Up @@ -669,7 +675,7 @@ bool ImportCsvDialog::importCsv(const QString& fileName, const QString& name)
#endif

// Save row num for later use. This is used in the case of an error to tell the user in which row the error occurred
lastRowNum = rowNum;
lastRowNum = rowNum + static_cast<size_t>(ui->spinBoxSkipRows->value());

// If this is the first row and we want to use the first row as table header, skip it now because this is the data import, not the header parsing
if(rowNum == 0 && ui->checkboxHeader->isChecked())
Expand Down
81 changes: 59 additions & 22 deletions src/ImportCsvDialog.ui
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,26 @@
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="labelSkipRows">
<property name="text">
<string>S&amp;kip first rows</string>
</property>
<property name="buddy">
<cstring>spinBoxSkipRows</cstring>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QSpinBox" name="spinBoxSkipRows">
<property name="toolTip">
<string>Number of rows at the beginning of the file which are ignored, for example a title or other information preceding the actual data. If the first line contains the column names, this is the first line after the skipped rows.</string>
</property>
<property name="maximum">
<number>1000000</number>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QLabel" name="labelSeparator">
<property name="text">
<string>Field &amp;separator</string>
Expand All @@ -59,7 +79,7 @@
</property>
</widget>
</item>
<item row="2" column="1">
<item row="3" column="1">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QComboBox" name="comboSeparator">
Expand Down Expand Up @@ -124,7 +144,7 @@
</item>
</layout>
</item>
<item row="3" column="0">
<item row="4" column="0">
<widget class="QLabel" name="labelQuote">
<property name="text">
<string>&amp;Quote character</string>
Expand All @@ -134,7 +154,7 @@
</property>
</widget>
</item>
<item row="3" column="1">
<item row="4" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QComboBox" name="comboQuote">
Expand Down Expand Up @@ -194,7 +214,7 @@
</item>
</layout>
</item>
<item row="4" column="0">
<item row="5" column="0">
<widget class="QLabel" name="labelEncoding">
<property name="text">
<string>&amp;Encoding</string>
Expand All @@ -204,7 +224,7 @@
</property>
</widget>
</item>
<item row="4" column="1">
<item row="5" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QComboBox" name="comboEncoding">
Expand Down Expand Up @@ -248,7 +268,7 @@
</item>
</layout>
</item>
<item row="5" column="0">
<item row="6" column="0">
<widget class="QLabel" name="labelTrim">
<property name="text">
<string>Trim fields?</string>
Expand All @@ -258,7 +278,7 @@
</property>
</widget>
</item>
<item row="5" column="1">
<item row="6" column="1">
<widget class="QCheckBox" name="checkBoxTrimFields">
<property name="text">
<string/>
Expand All @@ -268,7 +288,7 @@
</property>
</widget>
</item>
<item row="6" column="0">
<item row="7" column="0">
<widget class="QLabel" name="separateTables">
<property name="text">
<string>Separate tables</string>
Expand All @@ -278,14 +298,14 @@
</property>
</widget>
</item>
<item row="6" column="1">
<item row="7" column="1">
<widget class="QCheckBox" name="checkBoxSeparateTables">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="7" column="1">
<item row="8" column="1">
<widget class="QPushButton" name="buttonAdvanced">
<property name="text">
<string>Advanced</string>
Expand All @@ -299,14 +319,14 @@
</property>
</widget>
</item>
<item row="10" column="1">
<item row="11" column="1">
<widget class="QCheckBox" name="checkIgnoreDefaults">
<property name="toolTip">
<string>When importing an empty value from the CSV file into an existing table with a default value for this column, that default value is inserted. Activate this option to insert an empty value instead.</string>
</property>
</widget>
</item>
<item row="10" column="0">
<item row="11" column="0">
<widget class="QLabel" name="labelIgnoreDefaults">
<property name="text">
<string>Ignore default &amp;values</string>
Expand All @@ -316,14 +336,14 @@
</property>
</widget>
</item>
<item row="11" column="1">
<item row="12" column="1">
<widget class="QCheckBox" name="checkFailOnMissing">
<property name="toolTip">
<string>Activate this option to stop the import when trying to import an empty value into a NOT NULL column without a default value.</string>
</property>
</widget>
</item>
<item row="11" column="0">
<item row="12" column="0">
<widget class="QLabel" name="labelFailOnMissing">
<property name="text">
<string>Fail on missing values </string>
Expand All @@ -333,7 +353,7 @@
</property>
</widget>
</item>
<item row="8" column="0">
<item row="9" column="0">
<widget class="QLabel" name="labelNoTypeDetection">
<property name="text">
<string>Disable data type detection</string>
Expand All @@ -343,14 +363,14 @@
</property>
</widget>
</item>
<item row="8" column="1">
<item row="9" column="1">
<widget class="QCheckBox" name="checkNoTypeDetection">
<property name="toolTip">
<string>Disable the automatic data type detection when creating a new table.</string>
</property>
</widget>
</item>
<item row="9" column="0">
<item row="10" column="0">
<widget class="QLabel" name="labelLocalConventions">
<property name="text">
<string>Use local number conventions</string>
Expand All @@ -360,14 +380,14 @@
</property>
</widget>
</item>
<item row="9" column="1">
<item row="10" column="1">
<widget class="QCheckBox" name="checkLocalConventions">
<property name="toolTip">
<string>Use decimal and thousands separators according to the system locale.</string>
</property>
</widget>
</item>
<item row="12" column="1">
<item row="13" column="1">
<widget class="QComboBox" name="comboOnConflictStrategy">
<property name="toolTip">
<string>When importing into an existing table with a primary key, unique constraints or a unique index there is a chance for a conflict. This option allows you to select a strategy for that case: By default the import is aborted and rolled back but you can also choose to ignore and not import conflicting rows or to replace the existing row in the table.</string>
Expand All @@ -389,7 +409,7 @@
</item>
</widget>
</item>
<item row="12" column="0">
<item row="13" column="0">
<widget class="QLabel" name="labelOnConflictStrategy">
<property name="text">
<string>Conflict strategy</string>
Expand All @@ -399,14 +419,14 @@
</property>
</widget>
</item>
<item row="13" column="1">
<item row="14" column="1">
<widget class="QCheckBox" name="checkReplaceNonAlnum">
<property name="toolTip">
<string>Replace non-alphanumeric characters in column name with an underscore.</string>
</property>
</widget>
</item>
<item row="13" column="0">
<item row="14" column="0">
<widget class="QLabel" name="labelReplaceNonAlnum">
<property name="text">
<string>Replace non-alphanumeric in column name</string>
Expand Down Expand Up @@ -515,6 +535,7 @@
<tabstops>
<tabstop>editName</tabstop>
<tabstop>checkboxHeader</tabstop>
<tabstop>spinBoxSkipRows</tabstop>
<tabstop>comboSeparator</tabstop>
<tabstop>editCustomSeparator</tabstop>
<tabstop>comboQuote</tabstop>
Expand Down Expand Up @@ -666,6 +687,22 @@
</hint>
</hints>
</connection>
<connection>
<sender>spinBoxSkipRows</sender>
<signal>valueChanged(int)</signal>
<receiver>ImportCsvDialog</receiver>
<slot>updatePreview()</slot>
<hints>
<hint type="sourcelabel">
<x>192</x>
<y>82</y>
</hint>
<hint type="destinationlabel">
<x>354</x>
<y>45</y>
</hint>
</hints>
</connection>
<connection>
<sender>toggleSelected</sender>
<signal>toggled(bool)</signal>
Expand Down
2 changes: 2 additions & 0 deletions src/Settings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,8 @@ QVariant Settings::getDefaultValue(const std::string& group, const std::string&
{
if(name == "firstrowheader")
return false;
if(name == "skiprows")
return 0;
if(name == "trimfields")
return true;
if(name == "separatetables")
Expand Down
12 changes: 11 additions & 1 deletion src/csvparser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,18 @@ inline CSVField* addRow(CSVParser::csvRowFunction f, CSVRow& r, size_t& rowCount
}
}

CSVParser::ParserResult CSVParser::parse(csvRowFunction insertFunction, QTextStream& stream, size_t nMaxRecords)
CSVParser::ParserResult CSVParser::parse(csvRowFunction insertFunction, QTextStream& stream, size_t nMaxRecords, size_t nSkipRows)
{
// If rows are to be skipped, wrap the row function so the caller only sees the rows after the skipped ones, numbered from 0.
// The limit of records to read refers to the rows seen by the caller, so it has to be increased by the number of skipped rows here.
if(nSkipRows > 0)
{
auto skippingFunction = [&insertFunction, nSkipRows](size_t rowNum, CSVRow row) -> bool {
return rowNum < nSkipRows || insertFunction(rowNum - nSkipRows, row);
};
return parse(skippingFunction, stream, nMaxRecords > 0 ? nMaxRecords + nSkipRows : 0);
}

ParseStates state = StateNormal; // State of the parser
QByteArray sBuffer; // Buffer for reading the file
CSVRow record; // Buffer for parsing the current row
Expand Down
7 changes: 5 additions & 2 deletions src/csvparser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,13 @@ class CSVParser
* @param insertFunction A function pointer that is called for each parsed row. It is passed two parameters, the row number and a list of all parsed columns
* in the row. The called function may return false if an error occurred to stop the import process. Otherwise it should return true.
* \param stream Stream with the CSV parser
* \param nMaxRecords Max records too read, 0 if unlimited
* \param nMaxRecords Max records too read, 0 if unlimited. Skipped rows are not counted.
* \param nSkipRows Number of rows at the start of the stream to skip. These rows are parsed (so a quoted field spanning several lines
* counts as part of one row) but are not passed to insertFunction. The row numbers passed to insertFunction start at 0
* after the skipped rows.
* \return ParserResult value that indicated whether action finished normally, was cancelled or errored.
*/
ParserResult parse(csvRowFunction insertFunction, QTextStream& stream, size_t nMaxRecords = 0);
ParserResult parse(csvRowFunction insertFunction, QTextStream& stream, size_t nMaxRecords = 0, size_t nSkipRows = 0);

void setCSVProgress(CSVProgress* csvp) { m_pCSVProgress = csvp; }

Expand Down
Loading
Loading