/*
 * This file is a part of PikoTools
 * and is distributed under the (new) BSD licence.
 * Author: Tomasz Sowa <t.sowa@ttmath.org>
 */

/* 
 * Copyright (c) 2008-2021, Tomasz Sowa
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 *  * Neither the name Tomasz Sowa nor the names of contributors to this
 *    project may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "htmlfilter.h"
#include "convert/text.h"


namespace pt
{
const int HTMLFilter::WHITE_MODE_ORIGIN;
const int HTMLFilter::WHITE_MODE_SINGLE_LINE;
const int HTMLFilter::WHITE_MODE_TREE;



void HTMLFilter::Item::Clear()
{
	name.clear();
	type            = none;
	is_commentary   = false;
	porphans        = nullptr;
	new_line_before = false;
	new_line        = false;
	new_line_in_the_middle = false;
	has_body_tag    = false;
	tree_index      = 0;
}


HTMLFilter::Item::Item()
{
	Clear();
}



void HTMLFilter::Filter(const wchar_t * in, std::wstring & out)
{
	reading_from_file         = false;
	reading_from_wchar_string = true;
	pchar_unicode             = in;
	pchar_ascii               = 0;

	stack_len     = 0;
	out_string    = &out;
	//last_new_line = false;
	was_ending_commentary = false;
	line_len      = 0;
	out_string->clear();

	Init();
	Read();
	Uninit();
}



void HTMLFilter::Init()
{
}


void HTMLFilter::Uninit()
{
}



void HTMLFilter::Filter(const std::wstring & in, std::wstring & out)
{
	if( &in == &out )
	{
		// out cannot be the same string as in
		return;
	}

	size_t out_projected_len = in.size() * 2 + 1;

	if( out.capacity() < out_projected_len )
		out.reserve(out_projected_len);

	Filter(in.c_str(), out);
}


void HTMLFilter::SetSomeDefaults()
{
	white_mode  = WHITE_MODE_ORIGIN;

	tab_size    = 2;
	wrap_line   = 0;
	orphan_mode = orphan_nbsp;
	safe_mode   = false;
	skip_tags = false;
	skip_commentaries = false;
	skip_entities = false;
	analyze_entities = false;
}


HTMLFilter::HTMLFilter()
{
	pstack = new Item[WINIX_HTMLFILTER_STACK_MAXLEN];
	buffer = new wchar_t[WINIX_HTMLFILTER_BUFFER_MAXLEN];

	SetSomeDefaults();
}


HTMLFilter::HTMLFilter(const HTMLFilter & f)
{
	// don't need to copy the stack
	pstack = new Item[WINIX_HTMLFILTER_STACK_MAXLEN];
	buffer = new wchar_t[WINIX_HTMLFILTER_BUFFER_MAXLEN];

	SetSomeDefaults();
}


HTMLFilter & HTMLFilter::operator=(const HTMLFilter & f)
{
	// don't need to copy the stack
	pstack = new Item[WINIX_HTMLFILTER_STACK_MAXLEN];
	buffer = new wchar_t[WINIX_HTMLFILTER_BUFFER_MAXLEN];

	// we can copy some fields from f

return *this;
}


HTMLFilter::~HTMLFilter()
{
	delete [] pstack;
	delete [] buffer;
}



void HTMLFilter::white_chars_mode(int mode)
{
	if( mode >= WHITE_MODE_ORIGIN && mode <= WHITE_MODE_TREE )
		white_mode = mode;
}




void HTMLFilter::WrapLine(size_t wrap_line_)
{
	wrap_line = wrap_line_;

	if( wrap_line > 10000 )
		wrap_line = 10000;
}



void HTMLFilter::InsertTabs(size_t tabsize)
{
	tab_size = tabsize;

	if( tab_size > 1000 )
		tab_size = 1000;
}


int HTMLFilter::current_white_char_mode()
{
	if( !white_char_mode_tab.empty() )
		return white_char_mode_tab.back();

	return WHITE_MODE_ORIGIN;
}


void HTMLFilter::CalcOrphansMaxLen(Orphans & orphans)
{
size_t i;

	orphans.max_len = 0;

	for(i=0 ; i<orphans.tab.size() ; ++i)
	{
		if( orphans.tab[i].size() > orphans.max_len )
			orphans.max_len = orphans.tab[i].size();
	}
}


void HTMLFilter::AssignOrphans(const wchar_t * lang_code, const std::vector<std::wstring> & otab)
{
	lang_code_lower = lang_code;
	ToLower(lang_code_lower);

	orphans_temp.tab = otab;
	std::sort(orphans_temp.tab.begin(), orphans_temp.tab.end());
	CalcOrphansMaxLen(orphans_temp);

	orphans_tab[lang_code_lower] = orphans_temp;
}



void HTMLFilter::AssignOrphans(const std::wstring & lang_code, const std::vector<std::wstring> & otab)
{
	AssignOrphans(lang_code.c_str(), otab);
}


void HTMLFilter::ClearOrphans()
{
	orphans_tab.clear();
}




void HTMLFilter::OrphansMode(const std::wstring & orphan_mode_str)
{
	if( orphan_mode_str == L"160" )
		orphan_mode = orphan_160space;
	else
		orphan_mode = orphan_nbsp;
}


void HTMLFilter::SafeMode(bool safe_mode_)
{
	safe_mode = safe_mode_;
}


void HTMLFilter::SkipTags(bool skip_tags)
{
	this->skip_tags = skip_tags;
}

void HTMLFilter::SkipCommentaries(bool skip_commentaries)
{
	this->skip_commentaries = skip_commentaries;
}


void HTMLFilter::SkipEntities(bool skip_entities)
{
	this->skip_entities = skip_entities;

	if( this->skip_entities )
	{
		this->analyze_entities = true;
	}
}


void HTMLFilter::AnalyzeEntities(bool analyze_entities)
{
	this->analyze_entities = analyze_entities;
}


void HTMLFilter::SetNoFilterTag(const std::wstring & tag_name)
{
	no_filter_tag = tag_name;
}




HTMLFilter::Item & HTMLFilter::GetItem(size_t i)
{
	if( i >= stack_len )
	{
		empty.Clear();
		return empty;
	}

return pstack[i];
}


HTMLFilter::Item & HTMLFilter::LastItem()
{
	if( stack_len == 0 )
	{
		empty.Clear();
		return empty;
	}

return pstack[stack_len-1];
}


bool HTMLFilter::PushStack()
{
	if( stack_len == WINIX_HTMLFILTER_STACK_MAXLEN )
		// oops, too many items
		return false;

	pstack[stack_len].Clear();

	if( stack_len > 0 )
	{
		// 'porphans', 'has_body_tag' and 'tree_index' attributes are propagated
		pstack[stack_len].porphans     = pstack[stack_len-1].porphans;
		pstack[stack_len].has_body_tag = pstack[stack_len-1].has_body_tag;
		pstack[stack_len].tree_index   = pstack[stack_len-1].tree_index;
	}

	stack_len += 1;

return true;
}

void HTMLFilter::PopStack()
{
	if( stack_len == 0 )
		// oops
		return;

	stack_len -= 1;
	pstack[stack_len].Clear();
}


bool HTMLFilter::IsWhite(int c)
{
	// dont use c==10 here

	if( c==' ' || c=='\t' || c==13 || c==160 )
		return true;

return false;
}


void HTMLFilter::SkipWhite()
{
	while( IsWhite(lastc) )
		read_char();
}


void HTMLFilter::SkipWhiteLines()
{
	while( lastc==10 || IsWhite(lastc) )
		read_char();
}


void HTMLFilter::SkipWhiteWithFirstNewLine()
{
	SkipWhite();

	if( lastc == 10 )
	{
		read_char();
		SkipWhite();
	}
}


//void HTMLFilter::CheckNewLine()
//{
//	if( white_mode == WHITE_MODE_TREE )
//	{
//		SkipWhite();
//	}
//
//	last_new_line = (lastc==10);
//}




void HTMLFilter::SkipAndCheckClosingTag(std::wstring * remember_text)
{
	bool is_quoted = false;
	wchar_t quote_char = 0;

	while( lastc != -1 )
	{
		if( lastc == '"' || lastc == '\'' )
		{
			if( is_quoted )
			{
				if( lastc == quote_char )
				{
					is_quoted = false;
				}
			}
			else
			{
				is_quoted = true;
				quote_char = lastc;
			}
		}
		else
		if( !is_quoted && LastItem().type == Item::opening && IsClosingXmlSimpleTagMark(lastc) ) // closing xml tag: default '/'
		{
			LastItem().type = Item::simple;
		}
		else
		if( !is_quoted && IsClosingTagMark(lastc) )
		{
			read_char();
			break;
		}

		if( remember_text )
			(*remember_text) += lastc;

		read_char();
	}
}



bool HTMLFilter::IsValidCharForName(int c)
{
	if( (c>='a' && c<='z') ||
		(c>='A' && c<='Z') ||
		(c>='0' && c<='9') ||
		c=='-' || c=='!' || c==':' || c=='-') // : is for a namespace character, - is for a commentary
		return true;

return false;
}


bool HTMLFilter::IsValidCharForAttrName(int c)
{
	if( (c>='a' && c<='z') ||
		(c>='A' && c<='Z') ||
		(c>='0' && c<='9') ||
		c=='-' || c==':' )
		return true;

return false;
}


bool HTMLFilter::IsValidCharForEntityName(int c)
{
	if( (c>='a' && c<='z') ||
		(c>='A' && c<='Z') ||
		(c>='0' && c<='9') ||
		c=='#' )
		return true;

return false;
}


void HTMLFilter::ReadItemName(std::wstring & name, bool clear_name)
{
size_t i;

	if( clear_name )
		name.clear();

	for(i=0 ; IsValidCharForName(lastc) ; ++i)
	{
		if( i < WINIX_HTMLFILTER_ITEM_NAME_MAXLEN )
		{
			name += lastc;

			if( LastItem().type == Item::special && name == L"!--" )
			{
				LastItem().is_commentary = true;
				read_char();
				break;
			}
		}

		read_char();
	}
}



void HTMLFilter::ReadItemAttrName()
{
size_t i;

	attr_name.clear();

	for( i=0 ; lastc != -1 && IsValidCharForAttrName(lastc) ; ++i )
	{
		if( i < WINIX_HTMLFILTER_ATTR_NAME_MAXLEN )
			attr_name += lastc;

		read_char();
	}
}



void HTMLFilter::ReadItemAttrValueAdd(const std::wstring & str)
{
	if( analyze_entities )
	{
		attr_value.push_back(std::wstring());
		AnalyzeEntitiesAndPut(str.c_str(), str.c_str() + str.size(), &attr_value.back());
	}
	else
	{
		attr_value.push_back(str);
	}
}


void HTMLFilter::ReadItemAttrValue(bool has_quote, wchar_t quote_char)
{
	attr_value.clear();
	tmp_text.clear();

	while( lastc != -1 )
	{
		if( has_quote )
		{
			if( lastc == quote_char )
				break;
		}
		else
		{
			if( IsClosingTagMark(lastc) || lastc == 10 || IsWhite(lastc) )
				break;
		}

		if( lastc==10 || IsWhite(lastc) )
		{
			if( tmp_text.size() > 0 && tmp_text.size() <= WINIX_HTMLFILTER_ATTR_VALUE_MAXLEN )
				ReadItemAttrValueAdd(tmp_text);

			tmp_text.clear();
		}
		else
		{
			if( tmp_text.size() > WINIX_HTMLFILTER_ATTR_VALUE_MAXLEN )
				tmp_text.clear();

			tmp_text += lastc;
		}

		read_char();
	}

	if( tmp_text.size() > 0 && tmp_text.size() <= WINIX_HTMLFILTER_ATTR_VALUE_MAXLEN )
		ReadItemAttrValueAdd(tmp_text);
}


void HTMLFilter::CheckChar(wchar_t c)
{
	if( c == 10 )
		line_len = 0;
	else
		line_len += 1;
}


void HTMLFilter::Put(wchar_t c)
{
	(*out_string) += c;
	CheckChar(c);
}


void HTMLFilter::Put(const wchar_t * str, const wchar_t * end)
{
	if( str >= end )
		return;

	size_t len = end - str;
	out_string->append(str, len);

	for( ; str < end ; ++str)
		CheckChar(*str);
}



void HTMLFilter::Put(const std::wstring & str)
{
	if( !str.empty() )
	{
		out_string->append(str);

		for(size_t i=0 ; i < str.size() ; ++i)
			CheckChar(str[i]);
	}
}


// out can be null
void HTMLFilter::AnalyzeEntitiesAndPut(const wchar_t * str, const wchar_t * end, std::wstring * out)
{
	size_t epsilon = 8; // !! IMPROVE ME put as a constant
	const wchar_t * old_str = str;

	while( str < end )
	{
		if( IsStartingEntityMark(*str) )
		{
			const wchar_t * entity_start = str;
			str += 1; // skip &

			for(size_t i=0 ; *str && IsValidCharForEntityName(*str) && i < epsilon ; ++i, ++str)
			{
			}

			if( IsEndingEntityMark(*str) && str - entity_start > 1 ) // at least one character in entity name
			{
				if( out )
					out->append(old_str, entity_start);
				else
					Put(old_str, entity_start);

				str += 1; // skip ;

				if( !skip_entities )
				{
					if( out )
						out->append(entity_start, str);
					else
						Put(entity_start, str);
				}

				EntityFound(entity_start + 1, str - 1); // without & and ;
				old_str = str;
			}
		}
		else
		{
			str += 1;
		}
	}

	if( out )
		out->append(old_str, end);
	else
		Put(old_str, end);
}




int HTMLFilter::CheckOrphan(const wchar_t * str, const wchar_t * end, const std::wstring & orphan_str)
{
size_t res;

	const wchar_t * orphan = orphan_str.c_str();

	for( ; str<end && *orphan!=0  ; ++str, ++orphan )
	{
		res = ToLower(*str) - ToLower(*orphan);
		
		if( res != 0 )
			return res;
	}

	if( str < end )
		return ToLower(*str);

return -int(ToLower(*orphan));
}




// binary search in table (table should be sorted)
bool HTMLFilter::CheckOrphan(const wchar_t * str, const wchar_t * end, const std::vector<std::wstring> & table)
{
int res;

	if( table.empty() )
		return false;

	size_t o1 = 0;
	size_t o2 = table.size() - 1;

	res = CheckOrphan(str, end, table[o1]);

	if( res == 0 )
		return true;

	if( res < 0 )
		return false;

	res = CheckOrphan(str, end, table[o2]);

	if( res == 0 )
		return true;

	if( res > 0 )
		return false;


	while( o1 + 1 < o2 )
	{
		size_t o = (o1 + o2) / 2;
		res = CheckOrphan(str, end, table[o]);

		if( res == 0 )
			return true;

		if( res < 0 )
			o2 = o;
		else
			o1 = o;
	}

return false;
}


bool HTMLFilter::CheckOrphan(const wchar_t * str, const wchar_t * end)
{
	if( str==end || !LastItem().has_body_tag || !LastItem().porphans )
		return false;

	size_t len = end - str;

	if( len > LastItem().porphans->max_len )
		return false;

return CheckOrphan(str, end, LastItem().porphans->tab);
}


void HTMLFilter::PutNormalNonWhite(std::wstring & str, bool allow_put_new_line, bool allow_put_space)
{
	while( lastc != -1 && lastc != 10 && !IsWhite(lastc) && !IsOpeningTagMark(lastc) )
	{
		str += lastc;
		read_char();

		if( IsEndingCommentaryTagMarkAtEndOfString(str) )
		{
			str.erase(str.size() - 3); // IMPROVEME define a function or what
			was_ending_commentary = true;
			break;
		}
	}

	if( !str.empty() )
	{
		if( allow_put_new_line )
		{
			Put(10);
			PutTabs(LastItem().tree_index + 1);
		}
		else
		if( allow_put_space )
		{
			Put(' ');
		}
	}

	if( analyze_entities )
		AnalyzeEntitiesAndPut(str.c_str(), str.c_str() + str.size(), nullptr);
	else
		Put(str);
}


void HTMLFilter::PutNormalWhite(bool & was_white_char, bool & was_new_line)
{
	was_white_char = false;
	was_new_line = false;

	while( lastc == 10 || IsWhite(lastc) )
	{
		if( lastc == 10 )
			was_new_line = true;
		else
			was_white_char = true;

		if( current_white_char_mode() == WHITE_MODE_ORIGIN )
		{
			Put(lastc);
		}

		read_char();
	}

	if( current_white_char_mode() == WHITE_MODE_SINGLE_LINE && (was_white_char || was_new_line) )
	{
		Put(' ');
	}

	// in WHITE_MODE_TREE white characters are written at the beginning of a <tag> or text
}



void HTMLFilter::PutOpeningTagMark()
{
	Put('<');
}


void HTMLFilter::PutClosingTagMark()
{
	Put('>');
}




// !! IMPROVE ME change to a better name
// this functions does not return true when the tag is safe
bool HTMLFilter::IsTagSafe(const wchar_t * tag)
{
	if( !safe_mode )
		return true;

	if( IsNameEqual(tag, no_filter_tag.c_str()) )
		return false;

	static const wchar_t * unsafe_tags[] = {
		L"applet", 	L"base",	L"body",
		L"embed",	L"head",	L"html",
		L"frame",	L"frameset",L"iframe",
		L"link",	L"meta",	L"param"
		L"object",	L"script"
	};

	size_t len = sizeof(unsafe_tags) / sizeof(const wchar_t*);
	size_t i;

	for(i=0 ; i<len ; ++i)
	{
		if( IsNameEqual(tag, unsafe_tags[i]) )
			return false;
	}

return true;
}


bool HTMLFilter::IsTagSafe(const std::wstring & tag)
{
	return IsTagSafe(tag.c_str());
}





bool HTMLFilter::PutOpeningTag()
{
	if( !IsTagSafe(LastItem().name) )
	{
		SkipAndCheckClosingTag();
		return false;
	}

	if( current_white_char_mode() == WHITE_MODE_TREE && LastItem().new_line_before )
	{
		Put(10);
		PutTabs(LastItem().tree_index);
	}

	PutOpeningTagMark();
	Put(LastItem().name);

return true;
}



void HTMLFilter::PutClosingTag(const Item & item)
{
	if( skip_tags || !IsTagSafe(item.name) )
		return;

	if( !item.is_commentary )
	{
		PutOpeningTagMark();
		Put('/');
	}

	Put(item.name);
	PutClosingTagMark();
}



void HTMLFilter::PutTabs(size_t len)
{
	if( len > 30 )
		len = 30;

	for(size_t i=0 ; i < (len*tab_size) ; ++i)
		(*out_string) += ' '; // we do not add them to 'line_len'
}


void HTMLFilter::PutNonBreakingSpace()
{
	if( orphan_mode == orphan_nbsp )
	{
		Put(L"&nbsp;");
	}
	else
	{
		Put(160);
	}
}



// we assume the size of the opening mark to be one
bool HTMLFilter::IsOpeningTagMark(wchar_t c)
{
	return (c == '<');
}


// we assume the size of the closing mark to be one
bool HTMLFilter::IsClosingTagMark(wchar_t c)
{
	return (c == '>');
}


// the slash in the closing tag mark e.g. </p>
bool HTMLFilter::IsClosingTagIndicator(wchar_t c)
{
	return (c == '/');
}


// the slash in the closing tag mark e.g. </p>
bool HTMLFilter::IsSpecialTagIndicator(wchar_t c)
{
	return (c == '!');
}


// the '=' operator e.g. class="value"
bool HTMLFilter::IsAttributeAssignmentMark(wchar_t c)
{
	return (c == '=');
}



// the slash at the end <img src=".." /> (without '>' character)
// we assume the size of the mark to be one
bool HTMLFilter::IsClosingXmlSimpleTagMark(wchar_t c)
{
	return (c == '/');
}


bool HTMLFilter::IsEndingCommentaryTagMarkAtEndOfString(const std::wstring & str)
{
	static wchar_t comm_end[] = L"-->";
	size_t comm_end_len = sizeof(comm_end) / sizeof(wchar_t) - 1;

	if( str.size() >= comm_end_len )
	{
		return IsNameEqual(str.c_str() + str.size() - comm_end_len, comm_end);
	}

	return false;
}


bool HTMLFilter::IsStartingEntityMark(wchar_t c)
{
	return (c == '&');
}


bool HTMLFilter::IsEndingEntityMark(wchar_t c)
{
	return (c == ';');
}



// reading text between html tags
void HTMLFilter::ReadText()
{
	bool was_white_char = false;
	bool was_new_line = false;

	bool was_non_white_text = false;

	was_ending_commentary = false;

	bool allow_put_new_line = false;
	bool allow_put_space = false;

	if( current_white_char_mode() == WHITE_MODE_TREE )
	{
		if( LastItem().new_line || (wrap_line != 0 && LastItem().has_body_tag && line_len >= wrap_line) )
		{
			allow_put_new_line = true;
		}
	}

	while( lastc != -1 && !IsOpeningTagMark(lastc) )
	{
		tmp_text.clear();
		PutNormalNonWhite(tmp_text, allow_put_new_line, allow_put_space);

		if( !tmp_text.empty() )
		{
			allow_put_new_line = false;
			allow_put_space = false;
			was_non_white_text = true;
		}

		if( CheckOrphan(tmp_text.c_str(), tmp_text.c_str() + tmp_text.size()) )
		{
			if( lastc == 10 || IsWhite(lastc) )
			{
				SkipWhiteLines();
				PutNonBreakingSpace();
			}
		}
		else
		{
			if( was_ending_commentary )
				break;

			PutNormalWhite(was_white_char, was_new_line);

			if( (was_white_char || was_new_line) && current_white_char_mode() == WHITE_MODE_TREE )
			{
				allow_put_new_line = false;
				allow_put_space = false;

				if( was_new_line )
				{
					allow_put_new_line = true;
					LastItem().new_line_in_the_middle = true;

					if( !was_non_white_text )
						LastItem().new_line = true;
				}
				else
				{
					allow_put_space = true;
				}

				if( wrap_line != 0 && LastItem().has_body_tag && line_len >= wrap_line )
				{
					allow_put_new_line = true;
				}
			}
		}
	}

	new_item_has_new_line_before = was_new_line;
}



bool HTMLFilter::PrintOpeningItem()
{
	if( skip_tags || IsNameEqual(no_filter_tag, LastItem().name) )
		return true;

	return PutOpeningTag();
}





bool HTMLFilter::ReadItemAttr()
{
	attr_has_value = false;
	attr_name.clear();
	attr_value.clear();

	SkipWhiteLines();
	ReadItemAttrName();

	if( attr_name.empty() )
		return false;

	SkipWhiteLines();

	if( !IsAttributeAssignmentMark(lastc) ) // '='
		return true;
	
	attr_has_value = true;
	read_char();				// skipping '='
	SkipWhiteLines();

	bool has_quote = (lastc == '\"' || lastc == '\'');
	wchar_t quote_char = lastc;

	if( has_quote )
		read_char();			// skipping the first quote mark

	ReadItemAttrValue(has_quote, quote_char);

	if( has_quote && lastc == quote_char )
		read_char();			// skipping the last quote mark

return true;
}



void HTMLFilter::CheckItemLangAttr()
{
	if( attr_has_value && IsNameEqual(L"lang", attr_name) )
	{
		LastItem().porphans = nullptr;

		if( !attr_value.empty() )
		{
			// we are taking the first value only
			attr_value_lower = attr_value[0];
			ToLower(attr_value_lower);

			OrphansTab::iterator i = orphans_tab.find(attr_value_lower);

			if( i != orphans_tab.end()  )
				LastItem().porphans = &i->second;
		}
	}
}


void HTMLFilter::PrintItemAttr()
{
size_t i;

	if( skip_tags || IsNameEqual(no_filter_tag, LastItem().name) )
		return;

	Put(' ');
	Put(attr_name);

	if( attr_has_value )
	{
		Put(L"=\"");

		for(i=0 ; i<attr_value.size() ; ++i)
		{
			Put(attr_value[i]);

			if( i + 1 < attr_value.size() )
				Put(' ');
		}

		Put('\"');
	}
}


void HTMLFilter::ReadItemClosing()
{
	read_char(); // skipping '/'
	SkipWhiteLines();
	ReadItemName(LastItem().name);
	LastItem().type = Item::closing;
	SkipAndCheckClosingTag();

	// closing tags are printed later
}


void HTMLFilter::ReadItemSpecial()
{
	LastItem().type = Item::special;

	if( !skip_tags )
	{
		if( current_white_char_mode() == WHITE_MODE_TREE && LastItem().new_line_before )
		{
			Put(10);
			PutTabs(LastItem().tree_index);
		}

		PutOpeningTagMark();
	}

	read_char(); // skipping '!'
	LastItem().name = '!';
	ReadItemName(LastItem().name, false);

	if( skip_tags )
	{
		SkipAndCheckClosingTag();
	}
	else
	{
		if( LastItem().is_commentary )
		{
			Put(LastItem().name);
		}
		else
		{
			tmp_text.clear();
			SkipWhiteLines();
			SkipAndCheckClosingTag(&tmp_text);
			Put(LastItem().name);
			Put(' ');
			Put(tmp_text);
			Put('>');

			if( is_first_item && current_white_char_mode() == WHITE_MODE_TREE && is_equal_nc(LastItem().name.c_str(), L"!doctype") )
			{
				Put(10);
				Put(10);
				SkipWhiteLines();
			}
		}
	}
}


void HTMLFilter::ReadItemOpening()
{
	LastItem().type = Item::opening;
	ReadItemName(LastItem().name);
	
	if( PrintOpeningItem() )
	{
		while( ReadItemAttr() )
		{
			CheckItemLangAttr();
			PrintItemAttr();
		}

		SkipAndCheckClosingTag(); // here LastItem().type can be changed to 'simple'

		if( !skip_tags && !IsNameEqual(no_filter_tag, LastItem().name) )
		{
			if( LastItem().type == Item::simple )
				Put(L" /");

			PutClosingTagMark();
		}
	}
}


void HTMLFilter::ItemFound()
{
}

void HTMLFilter::EntityFound(const wchar_t * str, const wchar_t * end)
{
}


bool HTMLFilter::ReadItem()
{
	if( lastc == -1 )
		return false;

	if( !PushStack() )
		return false;

	LastItem().new_line_before = new_item_has_new_line_before; // new_item_has_new_line_before is set by ReadText() method

	if( stack_len > 1 && pstack[stack_len-2].new_line_in_the_middle )
		LastItem().tree_index += 1;

	if( was_ending_commentary )
	{
		LastItem().type = Item::closing;
		LastItem().is_commentary = true;
		LastItem().name = L"--";
		was_ending_commentary = false;
	}
	else
	{
		read_char();	// skipping the first opening tag mark '<'
		SkipWhiteLines();

		if( IsSpecialTagIndicator(lastc) )
			ReadItemSpecial();
		else
		if( IsClosingTagIndicator(lastc) )
			ReadItemClosing();
		else
			ReadItemOpening();
	}

	// IMPROVE ME later CheckSingleItemExceptions() can change opening to single type
	ItemFound();

return true;
}	



wchar_t HTMLFilter::ToLower(wchar_t c)
{
	if( c>='A' && c<='Z' )
		return c - 'A' + 'a';

return c;
}


void HTMLFilter::ToLower(std::wstring & str)
{
size_t i;

	for(i=0 ; i<str.size() ; ++i)
		str[i] = ToLower(str[i]);
}


bool HTMLFilter::IsNameEqual(const wchar_t * name1, const wchar_t * name2)
{
	for( ; *name1!=0 && *name2!=0 ; ++name1, ++name2 )
		if( ToLower(*name1) != ToLower(*name2) )
			return false;

	if( *name1==0 && *name2==0 )
		return true;

return false;
}


bool HTMLFilter::IsNameEqual(const wchar_t * name1, const std::wstring & name2)
{
	return IsNameEqual(name1, name2.c_str());
}


bool HTMLFilter::IsNameEqual(const std::wstring & name1, const wchar_t * name2)
{
	return IsNameEqual(name1.c_str(), name2);
}


bool HTMLFilter::IsNameEqual(const std::wstring & name1, const std::wstring & name2)
{
	return IsNameEqual(name1.c_str(), name2.c_str());
}



// len characters from both strings must be equal
// IMPROVE ME change name to something like IsBeginningNameEqual
// and move to text.h (pikotools)
bool HTMLFilter::IsNameEqual(const wchar_t * name1, const wchar_t * name2, size_t len)
{
	for( ; *name1!=0 && *name2!=0 && len>0 ; ++name1, ++name2, --len )
		if( ToLower(*name1) != ToLower(*name2) )
			return false;

	if( len == 0 )
		return true;

return false;
}



bool HTMLFilter::IsNameEqual(const wchar_t * name1, const std::wstring & name2, size_t len)
{
	return IsNameEqual(name1, name2.c_str(), len);
}


bool HTMLFilter::IsNameEqual(const std::wstring & name1, const wchar_t * name2, size_t len)
{
	return IsNameEqual(name1.c_str(), name2, len);
}


bool HTMLFilter::IsNameEqual(const std::wstring & name1, const std::wstring & name2, size_t len)
{
	return IsNameEqual(name1.c_str(), name2.c_str(), len);
}





bool HTMLFilter::IsLastTag(const wchar_t * name)
{
	return IsNameEqual(name, LastItem().name);
}


bool HTMLFilter::IsLastTag(const std::wstring & name)
{
	return IsNameEqual(name, LastItem().name);
}


// checking exceptions for opening tags
void HTMLFilter::CheckSingleItemExceptions()
{
	if( IsLastTag(L"meta")	||
		IsLastTag(L"input")	||
		IsLastTag(L"br")	||
		IsLastTag(L"hr")	||
		IsLastTag(L"img")	||
		IsLastTag(L"link")	||
		IsLastTag(L"param")	||
		IsLastTag(L"col")	||
		IsLastTag(L"area")   )
	{
		LastItem().type = Item::simple;
		PopStack();
		return;
	}

	// move me to a better place
	if( IsLastTag(L"body") )
		LastItem().has_body_tag = true;
}


void HTMLFilter::CheckWhiteCharsExceptions(Item & item)
{
	bool change_white_mode = false;

	// in safe_mode the script tag is ignored
	if( !safe_mode && IsNameEqual(item.name, L"script") )
	{
		change_white_mode = true;
	}

	if( IsNameEqual(item.name, L"pre") || IsNameEqual(item.name, L"textarea") )
	{
		change_white_mode = true;
	}

	if( IsNameEqual(item.name, no_filter_tag) )
	{
		change_white_mode = true;
	}

	if( change_white_mode )
	{
		if( item.type == Item::opening )
		{
			white_char_mode_tab.push_back(WHITE_MODE_ORIGIN);
		}
		else
		{
			if( !white_char_mode_tab.empty() )
				white_char_mode_tab.pop_back();
		}
	}
}




void HTMLFilter::AddForgottenTags()
{
int i;

	if( stack_len < 3 )
		return;

	// we have forgotten to close some tags

	// looking whether there is a matching opening tag
	for(i=int(stack_len)-3 ; i>=0 ; --i)
		if( IsNameEqual(pstack[i].name, pstack[stack_len-1].name) )
			break;

	if( i < 0 )
	{
		// oops, there is no such a tag
		// we don't print the closing and the missing opening tag
		PopStack();
		return;
	}

	for(int z=(int)stack_len-2 ; z>=i ; --z)
	{
		CheckWhiteCharsExceptions(pstack[z]);

		if( !skip_tags && pstack[z].new_line )
		{
			if( current_white_char_mode() == WHITE_MODE_TREE )
			{
				Put(10);
				PutTabs(pstack[z].tree_index);
			}
		}

		PutClosingTag(pstack[z]);
		pstack[z].Clear();
	}

	//last_new_line = pstack[stack_len-1].new_line;

	// invalidate tags
	stack_len = i;
}


void HTMLFilter::CheckStackPrintRest()
{
	while( stack_len-- > 0 )
	{
		if( stack_len==0 || pstack[stack_len-1].new_line )
		{
			if( current_white_char_mode() == WHITE_MODE_TREE )
			{
				Put(10);
				PutTabs(pstack[stack_len-1].tree_index);
			}
			else
			{
				Put(' ');
			}
		}

		PutClosingTag(pstack[stack_len]);
	}
}


void HTMLFilter::CheckClosingTags()
{
	if( stack_len == 0 )
		return;

	// on the stack we have only opening tags
	// but only the last tag is a closing tag

	if( stack_len == 1 )
	{
		// there is only last closing tag
		// we dont print it
		PopStack();
		return;
	}

	// there are more than one tag 
	if( (pstack[stack_len-1].is_commentary && pstack[stack_len-2].is_commentary) || IsNameEqual(pstack[stack_len-1].name, pstack[stack_len-2].name) )
	{
		CheckWhiteCharsExceptions(pstack[stack_len-1]);

		// last closing tag is from the previous one
		if( !skip_tags && pstack[stack_len-2].new_line )
		{
			if( current_white_char_mode() == WHITE_MODE_TREE )
			{
				Put(10);
				PutTabs(pstack[stack_len-2].tree_index);
			}
		}

		PutClosingTag(pstack[stack_len-1]);
		//last_new_line = pstack[stack_len-1].new_line;
		PopStack();
		PopStack();
	}
	else
	{
		AddForgottenTags();
	}
}


bool HTMLFilter::PrintRest()
{
//const wchar_t * start = pchar;

	// in safe mode we do not print the rest html code
	if( safe_mode || skip_tags )
		return false;

	bool was_chars = false;

	while( lastc != -1 )
	{
		Put(lastc);
		read_char();
		was_chars = true;
	}

	return was_chars;

//	if( pchar > start )
//	{
//		Put(start, pchar);
//		return true;
//	}

//return false;
}



void HTMLFilter::ReadLoop()
{
	while( ReadItem() )
	{
		if( LastItem().type == Item::opening )
		{
			CheckSingleItemExceptions();
			CheckWhiteCharsExceptions(LastItem());
		}
		else
		if( LastItem().type == Item::special )
		{
			if( !LastItem().is_commentary )
				PopStack();
		}
		else
		if( LastItem().type == Item::simple )
		{
			PopStack();
		}
		else
		if( LastItem().type == Item::closing )
		{
			CheckClosingTags();
		}
		else
		{
			PopStack();
		}

		ReadText();
		is_first_item = false;
	}
}



void HTMLFilter::Read()
{
	read_char(); // put first character to lastc
	is_first_item = true;

	white_char_mode_tab.clear();
	white_char_mode_tab.push_back(white_mode);

	if( current_white_char_mode() != WHITE_MODE_ORIGIN )
		SkipWhiteLines();

	// it can be some text or white lines before the first html tag (we print it)
	ReadText();

	// reading the whole html source
	ReadLoop();

	// sometimes there can remain some html source (when there is no space on the stack)
	// we print the rest html without filtering (only if safe_mode is false)
	if( !PrintRest() )
		CheckStackPrintRest();
}





}