added CSVParser - a csv parser

pull/4/head
Tomasz Sowa 2 years ago
parent ae1d25d5f3
commit 0ea5497094

@ -20,6 +20,8 @@ tests: FORCE
tests-gcc10: FORCE
env CXX=g++10 CXXFLAGS="-Wl,-rpath=/usr/local/lib/gcc10/ -Wall -pedantic -O0 -g3 -std=c++20 -I../src -I/usr/local/include" $(MAKE) -C src
env CXX=g++10 CXXFLAGS="-Wl,-rpath=/usr/local/lib/gcc10/ -Wall -pedantic -O0 -g3 -std=c++20 -I../src -I/usr/local/include" $(MAKE) -C tests
#env CXX=clang++ CXXFLAGS="-fsanitize=address -Wall -pedantic -O0 -g3 -std=c++20 -I../src -I/usr/local/include" $(MAKE) -C src
#env CXX=clang++ CXXFLAGS="-fsanitize=address -Wall -pedantic -O0 -g3 -std=c++20 -I../src -I/usr/local/include" $(MAKE) -C tests
clean: FORCE

@ -0,0 +1,416 @@
/*
* This file is a part of PikoTools
* and is distributed under the (new) BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name Tomasz Sowa nor the names of contributors to this
* project may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "csvparser.h"
#include "utf8/utf8.h"
namespace PT
{
CSVParser::Status CSVParser::parse_file(const char * file_name, Space & out_space)
{
reading_from_file = true;
space = &out_space;
file.clear();
file.open(file_name, std::ios_base::binary | std::ios_base::in);
if( file )
{
parse();
file.close();
}
else
{
status = cant_open_file;
}
return status;
}
CSVParser::Status CSVParser::parse_file(const std::string & file_name, Space & out_space)
{
return parse_file(file_name.c_str(), out_space);
}
CSVParser::Status CSVParser::parse_file(const wchar_t * file_name, Space & out_space)
{
std::string file_name_utf8;
PT::WideToUTF8(file_name, file_name_utf8);
return parse_file(file_name_utf8.c_str(), out_space);
}
CSVParser::Status CSVParser::parse_file(const std::wstring & file_name, Space & out_space)
{
return parse_file(file_name.c_str(), out_space);
}
CSVParser::Status CSVParser::parse(const char * str, Space & out_space)
{
reading_from_file = false;
reading_from_wchar_string = false;
pchar_ascii = str;
pchar_unicode = 0;
space = &out_space;
parse();
return status;
}
CSVParser::Status CSVParser::parse(const std::string & str, Space & out_space)
{
return parse(str.c_str(), out_space);
}
CSVParser::Status CSVParser::parse(const wchar_t * str, Space & out_space)
{
reading_from_file = false;
reading_from_wchar_string = true;
pchar_unicode = str;
pchar_ascii = 0;
space = &out_space;
parse();
return status;
}
CSVParser::Status CSVParser::parse(const std::wstring & str, Space & out_space)
{
return parse(str.c_str(), out_space);
}
void CSVParser::parse()
{
line = 1;
status = ok;
space->set_empty_table();
read_char(); // put first character to lastc
if( lastc == -1 )
{
// an empty file/string, in such a case we return such a space struct (if would be serialized to json): [[]]
Space * row_space = new Space();
row_space->set_empty_table();
space->add(row_space);
}
while( lastc != -1 )
{
/*
* even if there is an error when parsing we continue to read the file/string
*
*/
Space * row_space = new Space();
row_space->set_empty_table();
parse_row(row_space);
space->add(row_space);
}
}
void CSVParser::parse_row(Space * row_space)
{
bool continue_reading;
do
{
continue_reading = read_value_to(row_space);
}
while(continue_reading);
}
bool CSVParser::read_value_to(Space * row_space)
{
Space & space_value = row_space->add_empty_space();
space_value.set_empty_wstring();
if( lastc == '"' )
{
return read_quoted_value_to(space_value.value.value_wstring);
}
else
{
return read_non_quoted_value_to(space_value.value.value_wstring);
}
}
bool CSVParser::read_quoted_value_to(std::wstring & value)
{
bool is_comma = false;
bool is_value_character = true;
while( lastc != -1 && is_value_character )
{
read_char();
if( lastc == '"' )
{
read_char();
if( lastc == '"' )
{
value += lastc;
}
else
{
is_value_character = false;
}
}
else
if( lastc != -1 )
{
value += lastc;
}
}
if( lastc == ',' )
{
is_comma = true;
read_char(); // skip the comma character
}
else
if( lastc == 13 )
{
read_char(); // skip CR character
if( lastc == 10 )
read_char();
}
else
if( lastc == 10 )
{
read_char(); // skip new line character
}
return is_comma;
}
bool CSVParser::read_non_quoted_value_to(std::wstring & value)
{
while( lastc != -1 && lastc != ',' && lastc != 10 )
{
value += lastc;
read_char();
}
bool is_comma = (lastc == ',');
if( is_comma )
{
read_char(); // skip the comma character
}
else
{
bool is_new_line = (lastc == 10);
// check CRLF sequence
if( is_new_line && !value.empty() && value.back() == 13 )
{
value.erase(value.size() - 1, 1);
}
if( is_new_line )
{
read_char(); // skip the new line character
}
}
return is_comma;
}
int CSVParser::read_utf8_char()
{
int c;
bool correct;
lastc = -1;
do
{
PT::UTF8ToInt(file, c, correct);
if( !file )
return lastc;
}
while( !correct );
lastc = c;
if( lastc == '\n' )
++line;
return lastc;
}
int CSVParser::read_ascii_char()
{
lastc = file.get();
if( lastc == '\n' )
++line;
return lastc;
}
int CSVParser::read_char_from_wchar_string()
{
if( *pchar_unicode == 0 )
lastc = -1;
else
lastc = *(pchar_unicode++);
if( lastc == '\n' )
++line;
return lastc;
}
int CSVParser::read_char_from_utf8_string()
{
int c;
bool correct;
lastc = -1;
do
{
size_t len = PT::UTF8ToInt(pchar_ascii, c, correct);
pchar_ascii += len;
}
while( *pchar_ascii && !correct );
if( correct )
lastc = c;
if( lastc == '\n' )
++line;
return lastc;
}
int CSVParser::read_char_from_ascii_string()
{
if( *pchar_ascii == 0 )
lastc = -1;
else
lastc = *(pchar_ascii++);
if( lastc == '\n' )
++line;
return lastc;
}
int CSVParser::read_char_no_escape()
{
if( reading_from_file )
{
if( input_as_utf8 )
return read_utf8_char();
else
return read_ascii_char();
}
else
{
if( reading_from_wchar_string )
{
return read_char_from_wchar_string();
}
else
{
if( input_as_utf8 )
return read_char_from_utf8_string();
else
return read_char_from_ascii_string();
}
}
}
int CSVParser::read_char()
{
return read_char_no_escape();
}
}

@ -0,0 +1,162 @@
/*
* This file is a part of PikoTools
* and is distributed under the (new) BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name Tomasz Sowa nor the names of contributors to this
* project may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef headerfile_picotools_csv_csvparser
#define headerfile_picotools_csv_csvparser
#include "space/space.h"
#include <string>
#include <fstream>
namespace PT
{
/*
*
* https://datatracker.ietf.org/doc/html/rfc4180
*
*/
class CSVParser
{
public:
enum Status
{
ok,
cant_open_file,
};
Status parse_file(const char * file_name, Space & out_space);
Status parse_file(const std::string & file_name, Space & out_space);
Status parse_file(const wchar_t * file_name, Space & out_space);
Status parse_file(const std::wstring & file_name, Space & out_space);
Status parse(const char * str, Space & out_space);
Status parse(const std::string & str, Space & out_space);
Status parse(const wchar_t * str, Space & out_space);
Status parse(const std::wstring & str, Space & out_space);
protected:
/*
the last status of parsing, set by Parse() methods
*/
Status status;
Space * space;
/*
true if parse_file() method was called
false if parse() was called
*/
bool reading_from_file;
/*
true if parse(wchar_t *) or parse(std::wstring&) was called
*/
bool reading_from_wchar_string;
/*
pointers to the current character
if parse() is being used
*/
const char * pchar_ascii;
const wchar_t * pchar_unicode;
/*
last read char
or -1 if the end
*/
int lastc;
/*
a number of a line in which there is a syntax_error
*/
int line;
/*
current file
may it would be better to make a pointer?
if we parse only a string then there is no sense to have such an object
*/
std::ifstream file;
/*
input file is in UTF-8
default: true
*/
bool input_as_utf8;
void parse();
void parse_row(Space * row_space);
bool read_value_to(Space * row_space);
bool read_quoted_value_to(std::wstring & value);
bool read_non_quoted_value_to(std::wstring & value);
/*
* copied from SpaceParser
* may it would be better to have a class with those methods and inherit from it?
*/
int read_utf8_char();
int read_ascii_char();
int read_char_from_wchar_string();
int read_char_from_utf8_string();
int read_char_from_ascii_string();
int read_char_no_escape();
int read_char();
};
}
#endif

@ -0,0 +1,329 @@
/*
* This file is a part of PikoTools
* and is distributed under the (new) BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name Tomasz Sowa nor the names of contributors to this
* project may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "csvparser.h"
#include "csv/csvparser.h"
#include "test.h"
namespace pt
{
// remove me in the future (when PT will be changed to pt)
using namespace PT;
namespace pt_csvparser_tests
{
void test_csvparser(const char * input_str, const char * expected_json)
{
CSVParser csv_parser;
Space space;
std::string json;
CSVParser::Status status = csv_parser.parse(input_str, space);
space.serialize_to_json_to(json);
std::cout << "csv parsed as: " << json << std::endl;
test(json.c_str(), expected_json);
}
void test_csvparser1()
{
const char * input_str = "";
const char * expected_json = R"json([[]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser2()
{
const char * input_str = ",";
const char * expected_json = R"json([["",""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser3()
{
const char * input_str = "field1";
const char * expected_json = R"json([["field1"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser4()
{
const char * input_str = R"csvstring(field1,field2,field3)csvstring";
const char * expected_json = R"json([["field1","field2","field3"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser5()
{
const char * input_str = "\n";
const char * expected_json = R"json([[""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser6()
{
const char * input_str = "\r\n";
const char * expected_json = R"json([[""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser7()
{
const char * input_str = "field1\r\n";
const char * expected_json = R"json([["field1"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser8()
{
const char * input_str = ",\r\n";
const char * expected_json = R"json([["",""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser9()
{
const char * input_str = "field1\r\nfield2";
const char * expected_json = R"json([["field1"],["field2"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser10()
{
const char * input_str = "field1,field2\r\nfield3,field4";
const char * expected_json = R"json([["field1","field2"],["field3","field4"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser11()
{
const char * input_str = "field1,field2\r\nfield3,field4\r\n";
const char * expected_json = R"json([["field1","field2"],["field3","field4"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser12()
{
const char * input_str = "field1,field2\nfield3,field4\n";
const char * expected_json = R"json([["field1","field2"],["field3","field4"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser13()
{
const char * input_str = R"csv("")csv";
const char * expected_json = R"json([[""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser14()
{
const char * input_str = "\"\"\n";
const char * expected_json = R"json([[""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser15()
{
const char * input_str = "\"\"\r\n";
const char * expected_json = R"json([[""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser16()
{
const char * input_str = "\"\",\r\n";
const char * expected_json = R"json([["",""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser17()
{
const char * input_str = "\"\",\n";
const char * expected_json = R"json([["",""]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser18()
{
const char * input_str = "\"field1\"";
const char * expected_json = R"json([["field1"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser19()
{
const char * input_str = "\"field1, with comma\"";
const char * expected_json = R"json([["field1, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser20()
{
const char * input_str = "\"field1, with comma\"\r\n";
const char * expected_json = R"json([["field1, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser21()
{
const char * input_str = "\"field1, with comma\"\n";
const char * expected_json = R"json([["field1, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser22()
{
const char * input_str = "\"field1, with comma\",\"field2\"";
const char * expected_json = R"json([["field1, with comma","field2"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser23()
{
const char * input_str = "\"field1, with comma\",\"field2\"\r\n\"field3\",\"field4, with comma\"";
const char * expected_json = R"json([["field1, with comma","field2"],["field3","field4, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser24()
{
const char * input_str = "\"field1, with comma\",\"field2\"\r\n\"field3\",\"field4, with comma\"\r\n";
const char * expected_json = R"json([["field1, with comma","field2"],["field3","field4, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser25()
{
const char * input_str = "\"field1, with comma\",\"field2 with \"\" double quote\"\r\n\"field3\",\"field4, with comma\"";
const char * expected_json = R"json([["field1, with comma","field2 with \" double quote"],["field3","field4, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser26()
{
const char * input_str = "\"field1, with comma\",\"field2 with \"\" double quote\"\n\"field3\",\"field4, with comma\"\n";
const char * expected_json = R"json([["field1, with comma","field2 with \" double quote"],["field3","field4, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void test_csvparser27()
{
const char * input_str = "\"field1, with comma\",\"field2 with \"\" double quote\"syntax error\n\"field3\",\"field4, with comma\"\n";
const char * expected_json = R"json([["field1, with comma","field2 with \" double quote"],["syntax error"],["field3","field4, with comma"]])json";
test_csvparser(input_str, expected_json);
}
void make_tests()
{
reset_test_counter("CSVParser");
test_csvparser1();
test_csvparser2();
test_csvparser3();
test_csvparser4();
test_csvparser5();
test_csvparser6();
test_csvparser7();
test_csvparser8();
test_csvparser9();
test_csvparser10();
test_csvparser11();
test_csvparser12();
test_csvparser13();
test_csvparser14();
test_csvparser15();
test_csvparser16();
test_csvparser17();
test_csvparser18();
test_csvparser19();
test_csvparser20();
test_csvparser21();
test_csvparser22();
test_csvparser23();
test_csvparser24();
test_csvparser25();
test_csvparser26();
test_csvparser27();
}
}
}

@ -0,0 +1,56 @@
/*
* This file is a part of PikoTools
* and is distributed under the (new) BSD licence.
* Author: Tomasz Sowa <t.sowa@ttmath.org>
*/
/*
* Copyright (c) 2021, Tomasz Sowa
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* * Neither the name Tomasz Sowa nor the names of contributors to this
* project may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef headerfile_picotools_tests_csvparser
#define headerfile_picotools_tests_csvparser
namespace pt
{
namespace pt_csvparser_tests
{
void make_tests();
}
}
#endif

@ -37,6 +37,7 @@
#include "convert.h"
#include "mainoptionsparser.h"
#include "csvparser.h"
#include <iostream>
@ -53,6 +54,7 @@ int main(int argc, const char ** argv)
{
pt::pt_convert_tests::make_tests();
pt::pt_mainoptions_tests::make_tests();
pt::pt_csvparser_tests::make_tests();
if( pt::was_error )
{

Loading…
Cancel
Save