///
/// \file	btarcmp.cc
///		Compare / diff tool to analyze Barry backup tarballs
///

/*
    Copyright (C) 2012, Net Direct Inc. (http://www.netdirect.ca/)

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

    See the GNU General Public License in the COPYING file at the
    root directory of this project for more details.
*/

#include <barry/barry.h>
#include <barry/barrybackup.h>

#include <iostream>
#include <iomanip>
#include <tr1/memory>
#include <string>
#include <vector>
#include <map>
#include <algorithm>
#include <stdexcept>

#include "barrygetopt.h"
#include "util.h"

using namespace std;
using namespace std::tr1;
using namespace Barry;

/*
Still TODO: should have the ability to copy all differing records
	into another tarball, to function as a "patch", so
	user can write it to his device to update only differing
	records, or to store as a record of changes, etc.
*/
void Usage()
{
   int logical, major, minor;
   const char *Version = Barry::Version(logical, major, minor);

   cerr
   << "btarcmp - Compare Barry backup tarballs\n"
   << "      Copyright 2012, Net Direct Inc. (http://www.netdirect.ca/)\n"
   << "      Using: " << Version << "\n"
   << "\n"
   << " Usage:  btarcmp [options...] tarball_0 tarball_1\n"
   << "\n"
   << "   -b        Use brief filename output\n"
   << "   -d db     Specify a specific database to compare.  Can be used\n"
   << "             multiple times.  If not used at all, all databases are\n"
   << "             compared.\n"
   << "   -D db     Specify a database name to skip.  If both -d and -D are\n"
   << "             used for the same database name, it will be skipped.\n"
   << "   -h        This help\n"
   << "   -I cs     International charset for string conversions\n"
   << "             Valid values here are available with 'iconv --list'\n"
   << "   -P        Only compare records that can be parsed\n"
   << "             This is the same as specifying -d for each database\n"
   << "             listed with -S.\n"
   << "   -S        Show list of supported database parsers.  Use twice\n"
   << "             to show field names as well.\n"
   << "   -v        Show verbose diff output (twice to force hex output)\n"
   << "\n"
   << endl;
}


//////////////////////////////////////////////////////////////////////////////
// Utility functions and functors

bool DBDataCmp(const DBData &a, const DBData &b)
{
	return a.GetUniqueId() < b.GetUniqueId();
}

bool UnknownCmp(const UnknownField &a, const UnknownField &b)
{
	return a.type < b.type;
}

class DBDataIdCmp
{
	uint32_t m_id;

public:
	explicit DBDataIdCmp(uint32_t id)
		: m_id(id)
	{
	}

	bool operator()(const DBData &data) const
	{
		return data.GetUniqueId() == m_id;
	}
};

void ChecksumDBData(const DBData &data, bool include_ids, std::string &sum)
{
	Barry::SHA_CTX m_ctx;

	SHA1_Init(&m_ctx);

	if( include_ids ) {
		SHA1_Update(&m_ctx, data.GetDBName().c_str(),
			data.GetDBName().size());

		uint8_t recType = data.GetRecType();
		SHA1_Update(&m_ctx, &recType, sizeof(recType));

		uint32_t uniqueId = data.GetUniqueId();
		SHA1_Update(&m_ctx, &uniqueId, sizeof(uniqueId));
	}

	int len = data.GetData().GetSize() - data.GetOffset();
	SHA1_Update(&m_ctx,
		data.GetData().GetData() + data.GetOffset(), len);

	unsigned char sha1[SHA_DIGEST_LENGTH];
	SHA1_Final(sha1, &m_ctx);

	ostringstream oss;
	for( int i = 0; i < SHA_DIGEST_LENGTH; i++ ) {
		oss << hex << setfill('0') << setw(2)
			<< (unsigned int) sha1[i];
	}
	sum = oss.str();
}


//////////////////////////////////////////////////////////////////////////////
// Parsed Compare class

class ParsedCompare
{
private:
	const DBData &m_one, &m_two;
	const IConverter *m_ic;
	bool m_known_record;
	std::string m_first_description;

public:
	ParsedCompare(const DBData &one, const DBData &two,
		const IConverter *ic = 0);

	bool CanParse() const { return m_known_record; }
	const std::string& GetDescription() const { return m_first_description;}

	/// Returns true if differing fields found and displayed.
	/// False if no differences found.
	bool ShowDifferingFields();
};

ParsedCompare::ParsedCompare(const DBData &one,
				const DBData &two,
				const IConverter *ic)
	: m_one(one)
	, m_two(two)
	, m_ic(ic)
	, m_known_record(false)
{
#undef HANDLE_PARSER
#define HANDLE_PARSER(tname) \
	else if( tname::GetDBName() == one.GetDBName() ) { \
		m_known_record = true; \
		tname a; \
		ParseDBData(m_one, a, m_ic); \
		m_first_description = a.GetDescription(); \
	}

	if( one.GetDBName() != two.GetDBName() ) {
		throw logic_error("Different database types in ParsedCompare ctor!");
	}
	// fall through and use the else's
	ALL_KNOWN_PARSER_TYPES
}

template <class RecordT>
class FieldHandler
{
private:
	const RecordT &m_one, &m_two;
	mutable bool m_found_difference;

public:
	FieldHandler(const RecordT &one, const RecordT &two)
		: m_one(one)
		, m_two(two)
		, m_found_difference(false)
	{
	}

	bool Differing() const { return m_found_difference; }

	void operator()(EnumFieldBase<RecordT> *ep,
		const FieldIdentity &id) const
	{
		if( ep->GetValue(m_one) == ep->GetValue(m_two) )
			return;

		m_found_difference = true;
		cout << "   " << id.Name << ":\n"
			<< "         tar[0] = "
			<< ep->GetName(ep->GetValue(m_one))
			<< " (" << ep->GetValue(m_one) << ")\n"
			<< "         tar[1] = "
			<< ep->GetName(ep->GetValue(m_two))
			<< " (" << ep->GetValue(m_two) << ")"
			<< endl;
	}

	void operator()(typename FieldHandle<RecordT>::PostalPointer pp,
		const FieldIdentity &id) const
	{
		const std::string
			&a = m_one.*(pp.m_PostalAddress).*(pp.m_PostalField),
			&b = m_two.*(pp.m_PostalAddress).*(pp.m_PostalField);

		if( a == b )
			return;

		m_found_difference = true;
		cout << "   " << id.Name << ":\n"
			<< "         tar[0] = '" << a << "'\n"
			<< "         tar[1] = '" << b << "'"
			<< endl;
	}

	void operator()(std::string RecordT::* mp, const FieldIdentity &id) const
	{
		if( m_one.*mp == m_two.*mp )
			return;

		m_found_difference = true;
		cout << "   " << id.Name << ":\n"
			<< "         tar[0] = '"
			<< Cr2LfWrapper(m_one.*mp) << "'\n"
			<< "         tar[1] = '"
			<< Cr2LfWrapper(m_two.*mp) << "'"
			<< endl;
	}

	void operator()(UnknownsType RecordT::* mp, const FieldIdentity &id) const
	{
		UnknownsType a = m_one.*mp, b = m_two.*mp;

		sort(a.begin(), a.end(), UnknownCmp);
		sort(b.begin(), b.end(), UnknownCmp);

		if( a == b )
			return;

		m_found_difference = true;
		cout << "   " << id.Name << ":\n"
			<< "         tar[0] = '" << a << "'\n"
			<< "         tar[1] = '" << b << "'"
			<< endl;
	}

	template <class TypeT>
	void operator()(TypeT RecordT::* mp, const FieldIdentity &id) const
	{
		if( m_one.*mp == m_two.*mp )
			return;

		m_found_difference = true;
		cout << "   " << id.Name << ":\n"
			<< "         tar[0] = '" << m_one.*mp << "'\n"
			<< "         tar[1] = '" << m_two.*mp << "'"
			<< endl;
	}
};

template <class RecordT>
bool DoParsedCompare(const RecordT &a, const RecordT &b)
{
	FieldHandler<RecordT> handler(a, b);
	ForEachField(RecordT::GetFieldHandles(), handler);
	return handler.Differing();
}

/// Returns true if differing fields found and displayed.
/// False if no differences found.
bool ParsedCompare::ShowDifferingFields()
{
#undef HANDLE_PARSER
#define HANDLE_PARSER(tname) \
	else if( tname::GetDBName() == m_one.GetDBName() ) { \
		tname a, b; \
		ParseDBData(m_one, a, m_ic); \
		ParseDBData(m_two, b, m_ic); \
		return DoParsedCompare<tname>(a, b); \
	}

	if( !m_known_record ) {
		return false;
	}

	ALL_KNOWN_PARSER_TYPES

	else {
		return false;
	}
}


//////////////////////////////////////////////////////////////////////////////
// Main application class

class App
{
public:
	typedef Barry::ConfigFile::DBListType		DBListType;
	typedef std::vector<Barry::DBData>		DBDataList;
	typedef std::map<std::string, DBDataList>	DatabaseMap;

private:
	DBListType m_compare_list;
	DBListType m_skip_list;
	DBListType m_valid_list;	// this list is created during the
					// database name compare... it holds
					// all the names that exist in both
					// maps, in sorted order
	DatabaseMap m_tars[2];
	std::string m_tarpaths[2];	// full filename with path
	std::string m_tarfiles[2];	// just filename, no path; or brief mark
	auto_ptr<IConverter> m_ic;

	int m_main_return;		// 0 - success
					// 1 - low level error or logic error
					// 2 - databases lists not the same
					// 3 - a record was added or deleted
	bool m_verbose;
	bool m_always_hex;
	bool m_sort_on_load;		// if true, sort each database by
					// Unique ID after loading from tarball
	bool m_include_ids;		// if true, include DBData IDs in SHA1

	std::string m_last_dbname;

public:
	App();

	void LoadTarballs();
	void CompareDatabaseNames();
	void CompareData();
	void Compare(const std::string &dbname);
	void Compare(const std::string &dbname,
		const DBDataList &one, const DBDataList &two);
	void Compare(const DBData &one, const DBData &two);

	bool Alike(DBDataList::const_iterator b1, DBDataList::const_iterator b2,
		DBDataList::const_iterator e1, DBDataList::const_iterator e2);
	void SearchCheck(DBDataList::const_iterator &b,
		DBDataList::const_iterator &e, const DBDataList &opposite_list,
		const std::string &action);

	void ShowRecordDiff(const DBData &one, const DBData &two,
		ParsedCompare &pc);
	void DumpRecord(const DBData &data);
	void ShowDatabaseHeader(const std::string &dbname);
	void AddParsersToCompare();

	// returns true if any of the items in Outputs needs a probe
	int main(int argc, char *argv[]);
};

//////////////////////////////////////////////////////////////////////////////
// Memory storage parser

class StoreParser : public Barry::Parser
{
	App::DatabaseMap &m_map;

public:
	explicit StoreParser(App::DatabaseMap &map)
		: m_map(map)
	{
	}

	virtual void ParseRecord(const DBData &data, const IConverter *ic)
	{
		m_map[data.GetDBName()].push_back(data);
	}
};


//////////////////////////////////////////////////////////////////////////////
// Misc helpers dependent on App

bool IdExists(const App::DBDataList &list, uint32_t id)
{
	return find_if(list.begin(), list.end(), DBDataIdCmp(id)) != list.end();
}

//////////////////////////////////////////////////////////////////////////////
// Member function definitions

App::App()
	: m_main_return(0)
	, m_verbose(false)
	, m_always_hex(false)
	, m_sort_on_load(true)
	, m_include_ids(true)
{
}

void App::AddParsersToCompare()
{
#undef HANDLE_PARSER
#define HANDLE_PARSER(tname) \
	m_compare_list.push_back(tname::GetDBName());

	ALL_KNOWN_PARSER_TYPES
}

void App::LoadTarballs()
{
	for( int i = 0; i < 2; i++ ) {
		// load data into memory
		Restore builder(m_tarpaths[i]);
		StoreParser parser(m_tars[i]);

		Pipe pipe(builder);
		pipe.PumpFile(parser, m_ic.get());

		// sort each database's record data by UniqueId
		for( DatabaseMap::iterator b = m_tars[i].begin();
			b != m_tars[i].end();
			++b )
		{
			if( m_sort_on_load )
				sort(b->second.begin(), b->second.end(), DBDataCmp);
		}
	}
}

void App::CompareDatabaseNames()
{
	for( int i = 1; i >= 0; i-- ) {
		int other = i == 0 ? 1 : 0;

		DatabaseMap::const_iterator b = m_tars[i].begin(), match;
		for( ; b != m_tars[i].end(); ++b ) {
			match = m_tars[other].find(b->first);
			if( match == m_tars[other].end() ) {
				cout << m_tarfiles[other] << ": has no database '" << b->first << "'" << endl;
				m_main_return = 2;
			}
			else {
				if( !m_valid_list.IsSelected(b->first) ) {
					m_valid_list.push_back(b->first);
				}
			}
		}
	}

	// sort the valid list
	sort(m_valid_list.begin(), m_valid_list.end());
//	cout << m_valid_list.size() << " valid database names found." << endl;
}

void App::CompareData()
{
	DBListType::const_iterator valid = m_valid_list.begin();
	for( ; valid != m_valid_list.end(); ++valid ) {
		// if m_compare_list contains items, then only compare
		// if this database is present in the list
		if( m_compare_list.size() && !m_compare_list.IsSelected(*valid) )
			continue;

		// check if we should skip this database
		if( m_skip_list.IsSelected(*valid) )
			continue;

		// all's well so far... compare!
		Compare(*valid);
	}
}

void App::Compare(const std::string &dbname)
{
	DatabaseMap::const_iterator tar[2];
	tar[0] = m_tars[0].find(dbname);
	tar[1] = m_tars[1].find(dbname);

	if( tar[0] == m_tars[0].end() || tar[1] == m_tars[1].end() )
		throw logic_error("Comparing non-existant database!" + dbname);

	Compare(dbname, tar[0]->second, tar[1]->second);
}

void App::Compare(const std::string &dbname,
			const DBDataList &one,
			const DBDataList &two)
{
	DBDataList::const_iterator
		b1 = one.begin(), e1 = one.end(),	// begin/end for one
		b2 = two.begin(), e2 = two.end(),	// begin/end for two
		s1, s2;					// search markers

	// if IDs are alike, compare
	// if not alike, then for each b1 and b2, do:
	//	search for id in opposite list
	//	if id found in opposite list, we're done, leave for next match
	//	if id not found, then entry has either been deleted or added
	//
	// NOTE: this algorithm assumes that both one and two are sorted!
	while( b1 != e1 || b2 != e2 ) {
		if( Alike(b1, b2, e1, e2 ) ) {
			Compare(*b1, *b2);
			++b1;
			++b2;
			continue;
		}
		else {
			// SearchCheck increments iterators if needed
			SearchCheck(b1, e1, two, "deleted");
			SearchCheck(b2, e2, one, "added");
		}
	}
}

void App::Compare(const DBData &one, const DBData &two)
{
	// make sure one and two are of the same database, or throw
	if( one.GetDBName() != two.GetDBName() )
		throw logic_error("Tried to compare records from different databases: " + one.GetDBName() + ", and " + two.GetDBName());

	// always compare the sums of the data first, and if match, done
	string sum1, sum2;
	ChecksumDBData(one, m_include_ids, sum1);
	ChecksumDBData(two, m_include_ids, sum2);
	if( sum1 == sum2 )
		return; // done

	// records are different, print concise report
	ShowDatabaseHeader(one.GetDBName());

	// if different, check if there's a parser available for this data
	// if not, display that these records differ, dump verbose if
	// needed, and done
	ParsedCompare pc(one, two, m_ic.get());
	ShowRecordDiff(one, two, pc);
}

void App::ShowRecordDiff(const DBData &one,
			const DBData &two,
			ParsedCompare &pc)
{
	if( !pc.CanParse() ) {
		// if can't parse, print:
		//    UniqueID: sizes (one vs. two), X bytes differ
		// then the differing fields
		cout << "  0x" << hex << one.GetUniqueId() << ": differs: "
			<< dec
			<< "sizes (" << one.GetData().GetSize()
			<< " vs. " << two.GetData().GetSize()
			<< "), SHA1 sums differ"
			<< endl;
	}
	else {
		// otherwise, print:
		//    UniqueID: sizes (one vs. two), (custom display name)
		cout << "  0x" << hex << one.GetUniqueId() << ": differs: "
			<< dec
			<< "sizes (" << one.GetData().GetSize()
			<< " vs. " << two.GetData().GetSize()
			<< "), "
			<< pc.GetDescription()
			<< endl;

		if( !pc.ShowDifferingFields() ) {
			// no difference found...
			cout << "No differences found in parsed records, but SHA1 sums differ." << endl;
		}
	}

	// if verbose and parser is null, or if always_hex,
	// then display a (messy?) hex diff of the raw data
	if( (m_verbose && !pc.CanParse()) || m_always_hex ) {
		cout << "   Hex diff of record:" << endl;
		cout << Diff(one.GetData(), two.GetData()) << endl;
	}
}

bool App::Alike(DBDataList::const_iterator b1,
		DBDataList::const_iterator b2,
		DBDataList::const_iterator e1,
		DBDataList::const_iterator e2)
{
	if( b1 == e1 || b2 == e2 )
		return false;
	return b1->GetUniqueId() == b2->GetUniqueId();
}

std::string GetDBDescription(const DBData &data, const IConverter *ic)
{
	string desc;

	// try to parse it
#undef HANDLE_PARSER
#define HANDLE_PARSER(tname) \
	if( data.GetDBName() == tname::GetDBName() ) { \
		tname rec; \
		ParseDBData(data, rec, ic); \
		return rec.GetDescription(); \
	}

	ALL_KNOWN_PARSER_TYPES

	return desc;
}

void App::SearchCheck(DBDataList::const_iterator &b,
			DBDataList::const_iterator &e,
			const DBDataList &opposite_list,
			const std::string &action)
{
	// nothing to do if we're at end of list
	if( b == e )
		return;

	// if id is found in opposite list, we're done!
	// leave the iterator as-is for the next cycle's match
	if( IdExists(opposite_list, b->GetUniqueId()) )
		return;

	// id not found, so set return value
	m_main_return = 3;

	// if id not found, then entry has either been deleted or added
	// (action says which one), and we need to display the diff
	// and advance the iterator
	ShowDatabaseHeader(b->GetDBName());
	cout << "  0x" << hex << b->GetUniqueId() << ": record has been "
		<< action << " in " << "tar[1]";
	string desc = GetDBDescription(*b, m_ic.get());
	if( desc.size() ) {
		cout << ": " << desc << endl;
	}
	else {
		cout << endl;
	}
	if( m_verbose ) {
		DumpRecord(*b);
	}

	// advance!
	++b;
}

void App::DumpRecord(const DBData &data)
{
#undef HANDLE_PARSER
#define HANDLE_PARSER(tname) \
	if( data.GetDBName() == tname::GetDBName() ) { \
		tname rec; \
		ParseDBData(data, rec, m_ic.get()); \
		cout << rec << endl; \
		return; \
	}

	ALL_KNOWN_PARSER_TYPES

	// if we get here, it's not a known record, so just dump the hex
	cout << data.GetData() << endl;
}

void App::ShowDatabaseHeader(const std::string &dbname)
{
	if( dbname != m_last_dbname ) {
		m_last_dbname = dbname;
		cout << "In database: " << dbname << endl;

	}
}

int App::main(int argc, char *argv[])
{
	bool brief = false;
	bool show_parsers = false, show_fields = false;
	string iconvCharset;

	// process command line options
	for(;;) {
		int cmd = getopt(argc, argv, "bd:D:hI:PSv");
		if( cmd == -1 )
			break;

		switch( cmd )
		{
		case 'b':	// use brief output
			brief = true;
			break;

		case 'd':	// database name to compare
			m_compare_list.push_back(optarg);
			break;

		case 'D':	// skip database to compare
			m_skip_list.push_back(optarg);
			break;

		case 'P':	// only compare parseable records
			AddParsersToCompare();
			break;

		case 'S':	// show parsers and builders
			if( show_parsers )
				show_fields = true;
			else
				show_parsers = true;
			break;

		case 'I':	// international charset (iconv)
			iconvCharset = optarg;
			break;

		case 'v':	// verbose
			if( !m_verbose )
				m_verbose = true;
			else
				m_always_hex = true;
			break;

		case 'h':	// help
		default:
			Usage();
			return 0;
		}
	}

	if( show_parsers ) {
		ShowParsers(show_fields, false);
		return 0;
	}

	if( (optind + 2) > argc ) {
		Usage();
		return 0;
	}

	// save the tarball filenames for later processing
	// start out assuming both arguments are simple, no path filenames
	m_tarpaths[0] = m_tarfiles[0] = argv[optind];
	m_tarpaths[1] = m_tarfiles[1] = argv[optind+1];

	if( brief ) {
		// user wants brief markers... filenames must be huge! :-)
		m_tarfiles[0] = "tar[0]";
		m_tarfiles[1] = "tar[1]";
	}
	else {
		// attempt to trim paths to filenames only
		if( m_tarpaths[0].find('/') != string::npos )
			m_tarfiles[0] = m_tarpaths[0].substr(m_tarpaths[0].rfind('/') + 1);
		if( m_tarpaths[1].find('/') != string::npos )
			m_tarfiles[1] = m_tarpaths[1].substr(m_tarpaths[1].rfind('/') + 1);

		// double check... don't want both markers the same:
		if( m_tarfiles[0] == m_tarfiles[1] ) {
			// doh... back to where we started
			m_tarfiles[0] = m_tarpaths[0];
			m_tarfiles[1] = m_tarpaths[1];
		}
	}

	// display key for user
	cout << "tar[0] = " << m_tarpaths[0] << endl;
	cout << "tar[1] = " << m_tarpaths[1] << endl;

	// initialize the Barry library
	Barry::Init(false);

	// create an IConverter object if needed
	if( iconvCharset.size() ) {
		m_ic.reset( new IConverter(iconvCharset.c_str(), true) );
	}

	// load both tarballs into memory for easy comparisons
	LoadTarballs();

	// compare plain list of database names first
	CompareDatabaseNames();

	// compare the actual data
	CompareData();

	return m_main_return;
}

int main(int argc, char *argv[])
{
	try {
		App app;
		return app.main(argc, argv);
	}
	catch( std::exception &e ) {
		cerr << "Exception: " << e.what() << endl;
		return 1;
	}
}

