Initial community commit

Src/nde/win/Binary32Field.cpp

@@ -0,0 +1,77 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ Binary32Field Class
+Field data layout:
+[4 bytes] length
+[length bytes] binary data
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+#include "Binary32Field.h"
+#include "../ndestring.h"
+//---------------------------------------------------------------------------
+Binary32Field::Binary32Field(const uint8_t *_Data, size_t len) : BinaryField(_Data, (int)len)
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+void Binary32Field::InitField(void)
+{
+	Type = FIELD_BINARY32;
+}
+
+//---------------------------------------------------------------------------
+Binary32Field::Binary32Field()
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+void Binary32Field::ReadTypedData(const uint8_t *data, size_t len)
+{
+	size_t pos = 0;
+	CHECK_INT(len); //len-=4;
+	uint32_t c = GET_INT(); pos += 4;
+	if (c && c<=len)
+	{
+		Size = c;
+		ndestring_release((wchar_t *)Data);
+		Data = (uint8_t *)ndestring_malloc(c);
+		GET_BINARY(Data, data, c, pos);
+	}
+}
+
+//---------------------------------------------------------------------------
+void Binary32Field::WriteTypedData(uint8_t *data, size_t len)
+{
+	uint32_t c;
+	size_t pos = 0;
+
+	CHECK_INT(len); //len-=4;
+
+	if (Data && Size<=len)
+	{
+		c = (uint32_t)Size;
+		PUT_INT(c); pos += 4;
+		if (Data)
+			PUT_BINARY(data, (unsigned char*)Data, c, pos);
+	}
+	else
+	{
+		PUT_INT(0);
+	}
+}
+
+//---------------------------------------------------------------------------
+size_t Binary32Field::GetDataSize(void)
+{
+	if (!Data) return 4;
+	return Size + 4;
+}

Src/nde/win/Binary32Field.h

@@ -0,0 +1,32 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ Binary32Field Class Prototypes
+
+--------------------------------------------------------------------------- */
+
+#ifndef __NDE_BINARY32FIELD_H
+#define __NDE_BINARY32FIELD_H
+
+#include "BinaryField.h"
+#include <bfc/platform/types.h>
+
+class Binary32Field : public BinaryField
+{
+protected:
+	virtual void ReadTypedData(const uint8_t *, size_t len);
+	virtual void WriteTypedData(uint8_t *, size_t len);
+	virtual size_t GetDataSize(void);
+	void InitField(void);
+
+public:
+	Binary32Field(const uint8_t *Data, size_t len);
+	Binary32Field();
+};
+
+#endif

Src/nde/win/BinaryField.cpp

@@ -0,0 +1,172 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ BinaryField Class
+Field data layout:
+[2 bytes] length
+[length bytes] binary data
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+#include "BinaryField.h"
+#include "../NDEString.h"
+//---------------------------------------------------------------------------
+BinaryField::BinaryField(const uint8_t *_Data, int len)
+{
+	InitField();
+	Type = FIELD_BINARY;
+	if (_Data && len > 0)
+	{
+		Data = (uint8_t *)ndestring_malloc(len);
+		memcpy(Data, _Data, len);
+		Size = len;
+	}
+}
+
+//---------------------------------------------------------------------------
+void BinaryField::InitField(void)
+{
+	Type = FIELD_BINARY;
+	Data = NULL;
+	Size = 0;
+}
+
+//---------------------------------------------------------------------------
+BinaryField::BinaryField()
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+BinaryField::~BinaryField()
+{
+	ndestring_release((wchar_t *)Data);
+}
+
+//---------------------------------------------------------------------------
+void BinaryField::ReadTypedData(const uint8_t *data, size_t len)
+{
+	unsigned short c;
+	int pos = 0;
+
+	CHECK_SHORT(len);
+
+	c = GET_SHORT(); pos += 2; 
+	if (c && c<=len)
+	{
+		Size = c;
+		ndestring_release((wchar_t *)Data);
+		Data = (uint8_t *)ndestring_malloc(c);
+		GET_BINARY(Data, data, c, pos);
+	}
+}
+
+//---------------------------------------------------------------------------
+void BinaryField::WriteTypedData(uint8_t *data, size_t len)
+{
+	size_t pos = 0;
+
+	CHECK_SHORT(len); 
+
+	if (Data && Size<=len)
+	{
+		unsigned short c = (unsigned short)Size;
+		PUT_SHORT(c); pos += 2;
+		if (Data)
+			PUT_BINARY(data, (unsigned char*)Data, c, pos);
+	}
+	else
+	{
+		PUT_SHORT(0);
+	}
+}
+
+//---------------------------------------------------------------------------
+const uint8_t *BinaryField::GetData(size_t *len)
+{
+	if (len)
+		*len = Size;
+	return Data;
+}
+
+//---------------------------------------------------------------------------
+void BinaryField::SetData(const uint8_t *_Data, size_t len)
+{
+	if (!_Data || !len) return;
+	ndestring_release((wchar_t *)Data);
+	Size = 0;
+	Data = (uint8_t *)ndestring_malloc(len);
+	memcpy(Data, _Data, len);
+	Size = len;
+}
+
+//---------------------------------------------------------------------------
+size_t BinaryField::GetDataSize(void)
+{
+	if (!Data) return 2;
+	return Size + 2;
+}
+
+//---------------------------------------------------------------------------
+int BinaryField::Compare(Field *Entry)
+{
+	if (!Entry) return -1;
+	size_t compare_length;
+	const uint8_t *compare_data = ((BinaryField*)Entry)->GetData(&compare_length);
+	return memcmp(Data, compare_data, min(compare_length, Size));
+}
+
+//---------------------------------------------------------------------------
+bool BinaryField::ApplyFilter(Field *FilterData, int op)
+{
+	size_t l, s;
+	const uint8_t *p = ((BinaryField *)FilterData)->GetData(&l);
+	const uint8_t *d = GetData(&s);
+	if (!p)
+		p = (const uint8_t *)"";
+	if (!d)
+		d = (const uint8_t *)"";
+	bool r;
+	switch (op)
+	{
+		case FILTER_EQUALS:
+			if (l != s)
+				r = false;
+			else
+				r = !memcmp(d, p, min(s, l));
+			break;
+		case FILTER_CONTAINS:
+			if (l > s)
+				r = FALSE;
+			else
+				r = !!memmem(d, p, s, l);
+			break;
+		case FILTER_ABOVE:
+			r = (memcmp(d, p, min(s, l)) > 0);
+			break;
+		case FILTER_BELOW:
+			r = (memcmp(d, p, min(s, l)) < 0);
+			break;
+		case FILTER_BELOWOREQUAL:
+			r = (memcmp(d, p, min(s, l)) <= 0);
+			break;
+		case FILTER_ABOVEOREQUAL:
+			r = (memcmp(d, p, min(s, l)) >= 0);
+			break;
+		case FILTER_ISEMPTY:
+			r = (s == 0);
+			break;
+		case FILTER_ISNOTEMPTY:
+			r = (s != 0);
+			break;
+		default:
+			r = true;
+			break;
+	}
+	return r;
+}

Src/nde/win/BinaryField.h

@@ -0,0 +1,38 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ BinaryField Class Prototypes
+
+--------------------------------------------------------------------------- */
+
+#ifndef __BINARYFIELD_H
+#define __BINARYFIELD_H
+
+#include "Field.h"
+#include <bfc/platform/types.h>
+class BinaryField : public Field
+{
+protected:
+	virtual void ReadTypedData(const uint8_t *, size_t len);
+	virtual void WriteTypedData(uint8_t *, size_t len);
+	virtual size_t GetDataSize(void);
+	virtual int Compare(Field *Entry);
+	virtual bool ApplyFilter(Field *Data, int flags);
+	void InitField(void);
+	uint8_t *Data;
+	size_t Size;
+
+public:
+	~BinaryField();
+	BinaryField(const uint8_t *Data, int len);
+	BinaryField();
+	const uint8_t *GetData(size_t *len);
+	void SetData(const uint8_t *Data, size_t len);
+};
+
+#endif

Src/nde/win/ColumnField.cpp

@@ -0,0 +1,184 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ColumnField Class
+Windows implementation
+
+Field data layout:
+[1 byte] Field Type
+[1 byte] Unused (maybe convert to 'searchable')
+[1 byte] Name Length
+[Name Length bytes] Name (UTF-8)
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+
+//---------------------------------------------------------------------------
+ColumnField::ColumnField(unsigned char FieldID, const wchar_t *FieldName, unsigned char FieldType, Table *parentTable)
+{
+	InitField();
+	Type = FIELD_COLUMN;
+	MyType = FieldType;
+	Name = ndestring_wcsdup(FieldName);
+	ID = FieldID;
+}
+
+//---------------------------------------------------------------------------
+void ColumnField::InitField(void)
+{
+	searchable = false;
+	MyType = FIELD_UNKNOWN;
+	Type = FIELD_COLUMN;
+	Name = NULL;
+	ID = 0;
+}
+
+//---------------------------------------------------------------------------
+ColumnField::ColumnField()
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+ColumnField::~ColumnField()
+{
+	ndestring_release(Name);
+}
+
+void ColumnField::SetSearchable(bool val)
+{
+	searchable=val;
+}
+
+bool ColumnField::IsSearchableField() const
+{
+	return searchable;
+}
+
+//---------------------------------------------------------------------------
+void ColumnField::ReadTypedData(const uint8_t *data, size_t len)
+{
+	unsigned char c;
+	int pos=0;
+
+	// [1 byte] Field Type
+	CHECK_CHAR(len);
+	MyType = GET_CHAR(); 
+	pos++;
+
+	// [1 byte] unused 
+	CHECK_CHAR(len);
+//cut:	indexUnique = (BOOL)GET_CHAR();
+	pos++;
+
+	// [1 byte] string length
+	CHECK_CHAR(len);
+	c = GET_CHAR(); 
+	pos++;
+
+	if (c)
+	{
+		CHECK_BIN(len, c);
+
+		int string_length = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)(data+pos), c, 0, 0);
+		Name = ndestring_malloc((string_length+1)*sizeof(wchar_t));
+		MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)(data+pos), c, Name, string_length);
+		Name[string_length]=0;
+	}
+}
+
+//---------------------------------------------------------------------------
+void ColumnField::WriteTypedData(uint8_t *data, size_t len)
+{
+	int pos = 0;
+
+	// [1 byte] Field Type
+	CHECK_CHAR(len);
+	PUT_CHAR(MyType); 
+	pos++;
+
+	// [1 byte] unused 
+	CHECK_CHAR(len);
+	PUT_CHAR(0/*(char)indexUnique*/); 
+	pos++;
+
+	CHECK_CHAR(len);
+	if (Name) 
+	{
+		size_t string_length = WideCharToMultiByte(CP_UTF8, 0, Name, -1, 0, 0, 0, 0);
+		if (string_length)
+		{
+			PUT_CHAR((uint8_t)string_length-1);
+			pos++;
+
+			CHECK_BIN(len, string_length-1);		
+			WideCharToMultiByte(CP_UTF8, 0, Name, -1, (LPSTR)(data+pos), (int)string_length-1, 0, 0);
+		}
+		else
+		{
+			PUT_CHAR(0);
+		}
+	}
+	else
+	{
+		PUT_CHAR(0);
+	}
+}
+
+//---------------------------------------------------------------------------
+unsigned char ColumnField::GetDataType(void)
+{
+	return MyType;
+}
+
+//---------------------------------------------------------------------------
+void ColumnField::SetDataType(unsigned char type)
+{
+	if ((MyType == FIELD_INTEGER || MyType == FIELD_BOOLEAN || MyType == FIELD_DATETIME || MyType == FIELD_LENGTH || MyType == FIELD_INT64) && 
+		(type == FIELD_INTEGER || type == FIELD_BOOLEAN || type == FIELD_DATETIME || type == FIELD_LENGTH || type == FIELD_INT64)) {
+			MyType = type;
+	}
+	// going from string to filename or filename to string is OK
+	if ((MyType == FIELD_FILENAME && type == FIELD_STRING)
+		|| (MyType == FIELD_STRING && type == FIELD_FILENAME))
+	{
+		MyType = type;
+	}
+}
+
+//---------------------------------------------------------------------------
+wchar_t *ColumnField::GetFieldName(void)
+{
+	return Name;
+}
+
+//---------------------------------------------------------------------------
+void ColumnField::SetFieldName(wchar_t *name)
+{
+	if (Name) ndestring_release(Name);
+	Name = ndestring_wcsdup(name);
+}
+
+//---------------------------------------------------------------------------
+size_t ColumnField::GetDataSize(void)
+{
+	size_t s=3;
+	if (Name)
+	{
+		int string_length=WideCharToMultiByte(CP_UTF8, 0, Name, -1, 0, 0, 0, 0);
+		if (string_length)
+			s+=string_length-1;
+	}
+	return s;
+}
+
+//---------------------------------------------------------------------------
+int ColumnField::Compare(Field * /*Entry*/)
+{
+	return 0;
+}

Src/nde/win/ColumnField.h

@@ -0,0 +1,49 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ColumnField Class Prototypes
+
+--------------------------------------------------------------------------- */
+
+#ifndef __COLUMNFIELD_H
+#define __COLUMNFIELD_H
+
+#include "Field.h"
+#include "LinkedList.h"
+#include "Table.h"
+
+#include "Scanner.h"
+
+class ColumnField : public Field
+{
+public:
+	ColumnField(unsigned char FieldID, const wchar_t *FieldName, unsigned char FieldType, Table *parentTable);
+	ColumnField();
+	~ColumnField();
+	virtual void ReadTypedData(const uint8_t *, size_t len);
+	virtual void WriteTypedData(uint8_t *, size_t len);
+	virtual size_t GetDataSize(void);
+	virtual int Compare(Field *Entry);
+	void InitField(void);
+
+	bool IsSearchableField() const;
+	void SetSearchable(bool val);
+
+	unsigned char GetDataType(void);
+	void SetDataType(unsigned char type);
+
+	wchar_t *GetFieldName(void); // not const because it's an NDE string
+	void SetFieldName(wchar_t *name);
+
+protected:
+	bool searchable;
+	wchar_t *Name;
+	unsigned char MyType;
+};
+
+#endif

Src/nde/win/FilenameField.cpp

@@ -0,0 +1,128 @@
+#include "FilenameField.h"
+#include "../nde.h"
+
+
+//---------------------------------------------------------------------------
+FilenameField::FilenameField(const wchar_t *Str, int strkind) : StringField(Str, strkind)
+{
+	Type = FIELD_FILENAME;
+}
+
+//---------------------------------------------------------------------------
+FilenameField::FilenameField()
+{
+	Type = FIELD_FILENAME;
+}
+
+//---------------------------------------------------------------------------
+int FilenameField::Compare(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (!CompatibleFields(Entry->GetType(), GetType()))
+		return 0;
+	return mywcsicmp_fn(GetStringW(), ((FilenameField*)Entry)->GetStringW());
+}
+
+//---------------------------------------------------------------------------
+int FilenameField::Starts(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (!CompatibleFields(Entry->GetType(), GetType()))
+		return 0;
+	const wchar_t *p = ((StringField*)Entry)->GetStringW();
+	const wchar_t *d = GetStringW();
+		if (!d || !p) return 0;
+	return nde_fnbegins(d, p);
+}
+
+//---------------------------------------------------------------------------
+int FilenameField::Contains(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (!CompatibleFields(Entry->GetType(), GetType()))
+		return 0;
+	const wchar_t *p = ((StringField*)Entry)->GetStringW();
+	const wchar_t *d = GetStringW();
+	if (!d || !p) return 0;
+	return nde_fncontains(GetStringW(), ((StringField*)Entry)->GetStringW());
+}
+
+Field *FilenameField::Clone(Table *pTable)
+{
+	FilenameField *clone = new FilenameField(StringW, STRING_IS_NDESTRING);
+	clone->Pos = FIELD_CLONE;
+	clone->ID = ID;
+	clone->MaxSizeOnDisk = (uint32_t)GetDataSize();
+	return clone;
+}
+
+// TODO: make file system string comparison functions
+//---------------------------------------------------------------------------
+bool FilenameField::ApplyFilter(Field *Data, int op)
+{
+	// TODO: maybe do this?
+	if (op == FILTER_ISEMPTY || op == FILTER_ISNOTEMPTY)
+	{
+		bool r = (op == FILTER_ISEMPTY);
+		if (!StringW)
+			return r;
+
+		if (StringW && StringW[0] == 0)
+			return r;
+
+		return !r;
+	}
+	//
+	bool r;
+	wchar_t *p=0;
+	if (Data->GetType() == FIELD_STRING)
+		p = ((StringField *)Data)->GetStringW();
+	else
+		p = ((FilenameField *)Data)->GetStringW();
+	wchar_t *d = GetStringW();
+	if (!p)
+		p = L"";
+	if (!d)
+		d = L"";
+
+	switch (op)
+	{
+		case FILTER_EQUALS:
+			r = !nde_wcsicmp_fn(d, p);
+			break;
+		case FILTER_NOTEQUALS:
+			r = !!nde_wcsicmp_fn(d, p);
+			break;
+		case FILTER_CONTAINS:
+			r = nde_fncontains(d, p);
+			break;
+		case FILTER_NOTCONTAINS:
+			r = !nde_fncontains(d, p);
+			break;
+		case FILTER_ABOVE:
+			r = (bool)(nde_wcsicmp_fn(d, p) > 0);
+			break;
+		case FILTER_ABOVEOREQUAL:
+			r = (bool)(nde_wcsicmp_fn(d, p) >= 0);
+			break;
+		case FILTER_BELOW:
+			r = (bool)(nde_wcsicmp_fn(d, p) < 0);
+			break;
+		case FILTER_BELOWOREQUAL:
+			r = (bool)(nde_wcsicmp_fn(d, p) <= 0);
+			break;
+		case FILTER_BEGINS:
+			r = nde_fnbegins(d, p);
+			break;
+		case FILTER_ENDS:
+			r = nde_fnends(d, p);
+			break;
+		case FILTER_LIKE:
+			r = (bool)(nde_wcsicmp_fn(d, p) == 0);
+			break;
+		default:
+			r = true;
+			break;
+	}
+	return r;
+}

Src/nde/win/FilenameField.h

@@ -0,0 +1,26 @@
+#ifndef NDE_FILENAMEFIELD_H
+#define NDE_FILENAMEFIELD_H
+
+/*
+  Mostly the same as StringField
+  but this implements OS-dependent string comparisons that make sense for the file system
+*/
+
+#include "../nde.h"
+#include "../NDEString.h"
+
+class FilenameField : public StringField
+{
+protected:
+	virtual int Compare(Field *Entry);
+	virtual int Starts(Field *Entry);
+	virtual int Contains(Field *Entry);
+	virtual bool ApplyFilter(Field *Data, int op);
+	virtual Field *Clone(Table *pTable);
+
+public:
+	FilenameField(const wchar_t *Str, int strkind=STRING_IS_WCHAR);
+	FilenameField();
+};
+
+#endif

Src/nde/win/IndexField.cpp

@@ -0,0 +1,145 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+IndexField Class
+Windows implementation
+
+Field data layout
+[4 bytes] Position
+[4 bytes] Data Type
+[1 byte] Name Length
+[Name Length bytes] Name (UTF-8)
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+#include "../ndestring.h"
+//---------------------------------------------------------------------------
+IndexField::IndexField(unsigned char id, int Pos, int type, const wchar_t *FieldName)
+{
+	InitField();
+	Type = FIELD_INDEX;
+	Name = ndestring_wcsdup(FieldName);
+	ID = id;
+	Position = Pos;
+	DataType = type;
+}
+
+//---------------------------------------------------------------------------
+void IndexField::InitField(void)
+{
+	index = 0;
+	Type = FIELD_INDEX;
+	Name = NULL;
+	ID = 0;
+	Position = -1;
+	DataType = FIELD_UNKNOWN;
+}
+
+//---------------------------------------------------------------------------
+IndexField::IndexField()
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+IndexField::~IndexField()
+{
+	ndestring_release(Name);
+	delete index;
+}
+
+//---------------------------------------------------------------------------
+void IndexField::ReadTypedData(const uint8_t *data, size_t len)
+{
+	unsigned char c;
+	int pos=0;
+	CHECK_INT(len);
+	Position = GET_INT(); pos += 4;
+	CHECK_INT(len);
+	DataType = GET_INT(); pos += 4;
+	CHECK_CHAR(len);
+	c = GET_CHAR(); pos++;
+	if (c)
+	{
+		CHECK_BIN(len, c);
+
+		int string_length = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)(data+pos), c, 0, 0);
+		Name = ndestring_malloc((string_length+1)*sizeof(wchar_t));
+		MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)(data+pos), c, Name, string_length);
+		Name[string_length]=0;
+	}
+}
+
+//---------------------------------------------------------------------------
+void IndexField::WriteTypedData(uint8_t *data, size_t len)
+{
+	int pos=0;
+	CHECK_INT(len);
+	PUT_INT(Position); pos += 4;
+
+	CHECK_INT(len);
+	PUT_INT(DataType); pos += 4;
+	CHECK_CHAR(len);
+	
+	CHECK_CHAR(len);
+
+	if (Name) 
+	{
+		size_t string_length = WideCharToMultiByte(CP_UTF8, 0, Name, -1, 0, 0, 0, 0);
+		if (string_length)
+		{
+			PUT_CHAR((uint8_t)string_length-1);
+			pos++;
+
+			CHECK_BIN(len, string_length-1);		
+			WideCharToMultiByte(CP_UTF8, 0, Name, -1, (LPSTR)(data+pos), (int)string_length-1, 0, 0);
+		}
+		else
+		{
+			PUT_CHAR(0);
+		}
+	}
+	else
+	{
+		PUT_CHAR(0);
+	}
+}
+
+//---------------------------------------------------------------------------
+wchar_t *IndexField::GetIndexName(void)
+{
+	return Name;
+}
+
+//---------------------------------------------------------------------------
+size_t IndexField::GetDataSize(void)
+{
+	size_t s=9;
+	if (Name)
+	{
+		int string_length=WideCharToMultiByte(CP_UTF8, 0, Name, -1, 0, 0, 0, 0);
+		if (string_length)
+			s+=string_length-1;
+	}
+	s++;
+	return s;
+}
+
+//---------------------------------------------------------------------------
+int IndexField::Compare(Field * /*Entry*/)
+{
+	return 0;
+}
+
+//---------------------------------------------------------------------------
+int IndexField::TranslateToIndex(int Id, IndexField *toindex)
+{
+	if (index && toindex && toindex->index)
+		return index->TranslateIndex(Id, toindex->index);
+	return -1;
+}

Src/nde/win/IndexField.h

@@ -0,0 +1,37 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ IndexField Class Prototypes
+
+--------------------------------------------------------------------------- */
+
+#ifndef __INDEXFIELD_H
+#define __INDEXFIELD_H
+
+class IndexField : public Field
+{
+public:
+	IndexField(unsigned char id, int Pos, int type, const wchar_t *FieldName);
+	IndexField();
+	~IndexField();
+	virtual void ReadTypedData(const uint8_t *, size_t len);
+	virtual void WriteTypedData(uint8_t *, size_t len);
+	virtual size_t GetDataSize(void);
+	virtual int Compare(Field *Entry);
+	void InitField(void);
+
+	wchar_t *GetIndexName(void);
+	int TranslateToIndex(int Id, IndexField *index);
+	Index *index; // TODO: make protected	
+protected:
+	int Position;
+	int DataType;
+	wchar_t *Name;
+};
+
+#endif

Src/nde/win/IndexRecord.cpp

@@ -0,0 +1,169 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+IndexRecord Class
+
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+#include <stdio.h>
+
+IndexRecord::IndexRecord(int RecordPos, int insertionPoint, VFILE *TableHandle, Table *ParentTable)
+{
+	InsertionPoint = insertionPoint;
+	if (RecordPos != 0)
+	{
+		int n=0;
+		uint32_t ThisPos = RecordPos;
+		while (ThisPos)
+		{
+			if (n >= 128)
+				break;
+			Vfseek(TableHandle, ThisPos, SEEK_SET);
+			Field Entry (ThisPos);
+			Field *TypedEntry = Entry.ReadField(ParentTable, ThisPos, &ThisPos);
+
+			if (!TypedEntry) break; // some db error?
+
+			AddField(TypedEntry);
+			//			ThisPos = TypedEntry->GetNextFieldPos();
+			n++;
+		}
+	}
+}
+
+void IndexRecord::BuildCollaboration()
+{
+	for (FieldList::iterator itr = Fields.begin(); itr != Fields.end(); itr++)
+	{
+		IndexField *p = (IndexField *)(* itr);
+		if ((itr + 1) != Fields.end())
+			p->index->Colaborate((IndexField *)(*(itr+1)));
+		else
+			p->index->Colaborate((IndexField *)*(Fields.begin()));
+	}
+}
+
+bool IndexRecord::NeedFix()
+{
+	for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+	{
+		IndexField *p = (IndexField *)*itr;
+		if (p->index->NeedFix())
+			return true;
+	}
+	return false;
+}
+
+IndexField *IndexRecord::GetIndexByName(const wchar_t *name)
+{
+	for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+	{
+		IndexField *p = (IndexField *)*itr;
+		if (!_wcsicmp(p->GetIndexName(), name))
+			return p;
+	}
+	return NULL;
+}
+
+bool IndexRecord::CheckIndexing(int v)
+{
+	int i = v;
+	for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+	{
+		IndexField *p = (IndexField *)*itr;
+		v = p->index->GetCooperative(v);
+	}
+	return v == i;
+}
+
+Field *RecordBase::GetField( unsigned char ID )
+{
+	for ( FieldList::iterator itr = Fields.begin(); itr != Fields.end(); itr++ )
+	{
+		IndexField *p = (IndexField *)*itr;
+		if ( p->GetFieldId() == ID )
+			return p;
+	}
+
+	return NULL;
+}
+
+void RecordBase::AddField(Field *field)
+{
+	if (!field)
+		return;
+
+	if (GetField(field->ID))
+		return;
+
+	Fields.push_back(field);
+}
+
+int IndexRecord::WriteFields( Table *ParentTable )
+{
+	Field      *l_previous_field = NULL;
+	IndexField *l_index_field    = NULL;
+
+	for ( FieldList::iterator itr = Fields.begin(); itr != Fields.end(); itr++ )
+	{
+		l_index_field = (IndexField *)(* itr);
+
+		Field* nextField = (itr + 1) != Fields.end() ? *(itr + 1) : nullptr;
+
+		//l_index_field->WriteField(ParentTable, l_previous_field, (Field*)l_index_field->next);
+		l_index_field->WriteField(ParentTable, l_previous_field, nextField);
+
+		l_previous_field = l_index_field;
+	}
+
+	return WriteIndex( ParentTable );
+}
+
+
+int IndexRecord::WriteIndex( Table *ParentTable )
+{
+	int P = 0;
+
+	if ( !Fields.empty() )
+		P = ( *Fields.begin() )->GetFieldPos();
+
+	return ParentTable->index->Update( INDEX_RECORD_NUM, P, this, FALSE );
+}
+
+void RecordBase::RemoveField(Field *field)
+{
+	if (!field)
+		return;
+
+	//Fields.erase(field);
+	//delete field;
+
+	for (auto it = Fields.begin(); it != Fields.end(); it++)
+	{
+		Field* f = *it;
+		if (f == field)
+		{
+			Fields.erase(it);
+			delete f;
+			break;
+		}
+	}
+}
+
+void IndexRecord::WalkFields(FieldsWalker callback, void *context)
+{
+	if (callback)
+	{
+		for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+		{
+			if (!callback(this, *itr, context))
+				break;
+		}
+	}
+}

Src/nde/win/IndexRecord.h

@@ -0,0 +1,18 @@
+#pragma once
+#include <stdio.h>
+#include "Record.h"
+
+class IndexRecord : public RecordBase
+{
+public:
+	IndexRecord(int RecordPos, int insertionPoint, VFILE *FileHandle,Table *p);
+	void BuildCollaboration();
+	bool NeedFix();
+	IndexField *GetIndexByName(const wchar_t *name);
+	int GetColumnCount() { return (int)(Fields.size() - 1); }
+	bool CheckIndexing(int v);
+	int WriteFields(Table *ParentTable);
+	int WriteIndex(Table *ParentTable);
+	typedef bool (*FieldsWalker)(IndexRecord *record, Field *entry, void *context);
+	void WalkFields(FieldsWalker callback, void *context);
+};

Src/nde/win/IntegerField.h

@@ -0,0 +1,95 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ IntegerField Class Prototypes
+
+--------------------------------------------------------------------------- */
+
+#ifndef __INTEGERFIELD_H
+#define __INTEGERFIELD_H
+
+#include <bfc/platform/types.h>
+
+class TimeParse {
+public:
+  int is_relative;          // ago/after/before used
+  int absolute_datetime;    // if not is_relative, this is the date/time, otherwise it's the resulting date/time if the query was ran now
+  int absolute_hasdate;     // if not, use only the time portion of absolute_datetime, the rest is now
+  int absolute_hastime;     // if not, use only the date portion of absolute_datetime, the rest is now
+  int relative_year;        // -1 = this year
+  int relative_month;       // -1 = this month
+  int relative_day;         // -1 = this day, -2 = this week
+  int relative_hour;        // -1 = this hour
+  int relative_min;         // -1 = this minute
+  int relative_sec;         // -1 = this second
+  int relative_kwday;       // not used(-1), 0 to 6 for sunday to saturday, yesterday(7), today(8), tomorrow(9)
+  int offset_value;         // timeago/offsetby numerical edit field
+  int offset_what;          // not used(-1) years(0), months(1), weeks(2), days(3), hours(4), minutes(5), seconds(6)
+  int offset_whence;        // not used(-1), after(0), ago/before(1)
+  int offset_used;          // if 1, 'time ago' / 'offset by' should be checked
+};
+
+class IntegerField : public Field
+	{
+	protected:
+
+		virtual void ReadTypedData(const uint8_t *, size_t len);
+		virtual void WriteTypedData(uint8_t *, size_t len);
+		virtual size_t GetDataSize(void);
+		virtual int Compare(Field *Entry);
+    virtual bool ApplyFilter(Field *Data, int op);
+		void InitField(void);
+		int Value;
+
+    enum {
+      WHAT_YEARS,
+      WHAT_MONTHS,
+      WHAT_WEEKS,
+      WHAT_DAYS,
+      WHAT_HOURS,
+      WHAT_MINUTES,
+      WHAT_SECONDS,
+    };
+
+    enum {
+      FROM_BARE,
+      FROM_AGO,
+      FROM_SINCE,
+    };
+
+	public:
+
+		~IntegerField();
+		IntegerField(int);
+		IntegerField();
+		int GetValue(void);
+		void SetValue(int);
+		int ApplyConversion(const wchar_t *format, TimeParse *tp=NULL);
+#ifdef _WIN32
+		static int LookupToken(const wchar_t *t);
+#else
+		static int LookupToken(CFStringRef t);
+#endif
+	};
+
+
+class DateTimeField : public IntegerField {
+  public:
+    DateTimeField();
+    DateTimeField(int Val);
+    virtual ~DateTimeField();
+};
+
+class LengthField : public IntegerField {
+  public:
+    LengthField();
+    LengthField(int Val);
+    virtual ~LengthField();
+};
+
+#endif

Src/nde/win/Query.cpp

@@ -0,0 +1,961 @@
+#include "../nde.h"
+#include "../NDEString.h"
+#include "Query.h"
+
+//---------------------------------------------------------------------------
+
+BOOL Scanner::Query(const wchar_t *query)
+{
+	if (!query) return FALSE;
+	ndestring_release(last_query);
+	last_query = ndestring_wcsdup(query);
+	RemoveFilters();
+	in_query_parser = 1;
+
+	BOOL r = Query_Parse(query);
+	if (r == FALSE)
+	{
+		if (!disable_date_resolution) RemoveFilters();
+		last_query_failed = TRUE;
+	}
+	in_query_parser = 0;
+	Query_CleanUp();
+	return r & CheckFilters();
+}
+
+const wchar_t *Scanner::GetLastQuery()
+{
+	return last_query;
+}
+
+typedef struct
+{
+	const wchar_t *token;
+	int tid;
+} tokenstruct;
+
+tokenstruct Tokens[] =   // Feel free to add more...
+{
+	{L"AND", TOKEN_AND },
+	{L"OR", TOKEN_OR },
+	{L"HAS", TOKEN_CONTAINS },
+	{L"NOTHAS",TOKEN_NOTCONTAINS},
+	{L"BEGINS", TOKEN_BEGINS },
+	{L"ENDS", TOKEN_ENDS },
+	{L"ISEMPTY", TOKEN_ISEMPTY},
+	{L"ISNOTEMPTY",TOKEN_ISNOTEMPTY},
+	{L"LIKE", TOKEN_LIKE},
+	{L"BEGINSLIKE", TOKEN_BEGINSLIKE},
+};
+
+typedef struct
+{
+	int Op;
+	int Level;
+} OpLevel;
+
+static int Query_ParseLength(const wchar_t *str)
+{
+	int i = (str ? _wtoi(str) : 0);
+
+	const wchar_t *p;
+	if ((p=wcsstr(str,L":")))
+	{
+		i*=60;
+		i+=_wtoi(++p);
+		if ((p=wcsstr(p,L":")))
+		{
+			i*=60;
+			i+=_wtoi(++p);
+		}
+	}
+	return i;
+}
+
+/*
+    our state machine
+
+                               &, |
+            ----------<-------------------------<----------------------<-----------
+           |                                                                       |
+           v    ID (Col)        =, >, <...       ID / Data / [f]          )        |
+     ---->(0) ----->-----> (1) ------>-----> (2) ------>------> (3) ------>-----> (4) <--
+    |     |^                \isempty------------->------------/  |^                |     |
+    | !(  ||                                                     ||----            |  )  |
+     --<--  ---------<---------------------------<-------------<-|     |            -->--
+                               &, |                              v [f] |
+                                                                  -->--
+*/
+
+//---------------------------------------------------------------------------
+BOOL Scanner::Query_Parse(const wchar_t *query)
+{
+	const wchar_t *p = query; // pointer on next token to read
+	int state = 0;
+	int pcount = 0;
+	VListEntry<OpLevel> *entry = 0;
+
+	if (pstack.GetNElements() > 0)
+		Query_CleanUp();
+
+	while (1)
+	{
+		p = Query_EatSpace(p);
+		int size = 0, t = Query_GetNextToken(p, &size, &token);
+		if (t == TOKEN_UNKNOWN)
+		{
+			Query_SyntaxError((int)(p-query));
+			return FALSE;
+		}
+		if (t == TOKEN_EOQ)
+			break;
+		switch (state)
+		{
+			case 0:
+				switch (t)
+				{
+					case TOKEN_PAROPEN:
+						state = 0;
+						// check too many parenthesis open
+						if (pcount == 255)
+						{
+							Query_SyntaxError((int)(p-query)); // should not be _syntax_ error
+							return FALSE;
+						}
+						// up one level
+						pcount++;
+						break;
+					case TOKEN_NOT:
+						// push not in this level
+						OpLevel o;
+						o.Op = FILTER_NOT;
+						o.Level = pcount;
+						entry = new VListEntry<OpLevel>;
+						entry->SetVal(o);
+						pstack.AddEntry(entry, TRUE);
+						state = 0;
+						break;
+					case TOKEN_IDENTIFIER:
+						state = 1;
+						// create filter column
+
+						if (AddFilterByName(token, NULL, FILTER_NONE) == ADDFILTER_FAILED)
+						{
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+
+						break;
+					default:
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+				}
+				break;
+			case 1:
+				switch (t)
+				{
+					case TOKEN_EQUAL:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_EQUALS);
+						break;
+					}
+					case TOKEN_ABOVE:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_ABOVE);
+						break;
+					}
+					case TOKEN_BELOW:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_BELOW);
+						break;
+					}
+					case TOKEN_CONTAINS:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_CONTAINS);
+						break;
+					}
+					case TOKEN_NOTCONTAINS:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_NOTCONTAINS);
+						break;
+					}
+					case TOKEN_AOREQUAL:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_ABOVEOREQUAL);
+						break;
+					}
+					case TOKEN_BOREQUAL:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_BELOWOREQUAL);
+						break;
+					}
+					case TOKEN_NOTEQUAL:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_NOTEQUALS);
+						break;
+					}
+					case TOKEN_BEGINS:
+					{
+						state = 2;
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_BEGINS);
+					}
+					break;
+					case TOKEN_ENDS:
+					{
+						state = 2;
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_ENDS);
+					}
+					break;
+					case TOKEN_LIKE:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_LIKE);
+						break;
+					}
+					case TOKEN_BEGINSLIKE:
+					{
+						state = 2;
+						// set filter op
+						Filter *f = GetLastFilter();
+						f->SetOp(FILTER_BEGINSLIKE);
+						break;
+					}
+					case TOKEN_ISNOTEMPTY:
+					case TOKEN_ISEMPTY:
+					{
+						state = 3;
+						Filter *f = GetLastFilter();
+						f->SetOp(t==TOKEN_ISEMPTY ? FILTER_ISEMPTY : FILTER_ISNOTEMPTY);
+						// pop all operators in this level beginning by the last inserted
+						entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+						while (entry)
+						{
+							if (entry->GetVal().Level == pcount)
+							{
+								AddFilterOp(entry->GetVal().Op);
+								VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+								pstack.RemoveEntry(entry);
+								entry = _entry;
+							}
+							else
+								break;
+						}
+					}
+					break;
+					default:
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+				}
+				break;
+			case 2:
+				if (t == TOKEN_SQBRACKETOPEN)
+				{
+					state = 3;
+					const wchar_t *s = wcschr(p, L']');
+					if (!s)
+					{
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+					}
+					p = Query_EatSpace(p);
+					if (p && *p == L'[') p++;
+
+					wchar_t *format = ndestring_wcsndup(p, s-p);
+
+					Filter *f = GetLastFilter();
+					int id = f->GetId();
+					ColumnField *c = GetColumnById(id);
+					int tt = c ? c->GetDataType() : -1;
+					if (disable_date_resolution || !c || (tt != FIELD_INTEGER && tt != FIELD_DATETIME && tt != FIELD_LENGTH && tt != FIELD_BOOLEAN))
+					{
+						if (disable_date_resolution)
+						{
+							StringField *field = (StringField *)f->Data();
+
+							if (!field)
+							{
+								// format was used without a value, assume value is 0
+								f->SetData(new StringField(L""));
+								entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+								while (entry)
+								{
+									if (entry->GetVal().Level == pcount)
+									{
+										AddFilterOp(entry->GetVal().Op);
+										VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+										pstack.RemoveEntry(entry);
+										entry = _entry;
+									}
+									else
+										break;
+								}
+								field = (StringField*)f->Data();
+							}
+
+							field->SetNDEString(format);
+							ndestring_release(format);
+							p = s+1;
+							continue;
+						}
+
+						ndestring_release(format);
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+					}
+
+					IntegerField *field = (IntegerField *)f->Data();
+
+					if (!field)
+					{
+						// format was used without a value, assume value is 0
+						f->SetData(new IntegerField(0));
+						entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+						while (entry)
+						{
+							if (entry->GetVal().Level == pcount)
+							{
+								AddFilterOp(entry->GetVal().Op);
+								VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+								pstack.RemoveEntry(entry);
+								entry = _entry;
+							}
+							else
+								break;
+						}
+						field = (IntegerField *)f->Data();
+					}
+					int r = field->ApplyConversion(format);
+					ndestring_release(format);
+					if (!r)
+					{
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+					}
+					p = s+1;
+					continue;
+				}
+	//        switch (t) {
+				//      case TOKEN_IDENTIFIER:
+				else   // JF> we make this relaxed, so anything is valid as a value
+				{
+					state = 3;
+					// set filter data
+					Filter *f = GetLastFilter();
+					int id = f->GetId();
+					ColumnField *c = GetColumnById(id);
+					switch (c ? c->GetDataType() : -1)
+					{
+						case FIELD_DATETIME:
+							if (disable_date_resolution)
+								goto field_string_override;
+						case FIELD_LENGTH:
+						{
+							IntegerField *i_f = new IntegerField();
+							int i = Query_ParseLength(token);
+							i_f->SetValue(i);
+							f->SetData(i_f);
+						}
+						break;
+						case FIELD_BOOLEAN:
+						case FIELD_INTEGER:
+						{
+							IntegerField *i_f = new IntegerField();
+							int i = _wtoi(token);
+							i_f->SetValue(i);
+							f->SetData(i_f);
+						}
+						break;
+						case FIELD_INT64:
+						{
+							Int64Field *i_f = new Int64Field();
+							int64_t i = _wtoi64(token); // todo: Replace with own conversion and error checking
+							i_f->SetValue(i);
+							f->SetData(i_f);
+						}
+						break;
+						case FIELD_FILENAME:
+							{
+								FilenameField *s_f = new FilenameField();
+								s_f->SetNDEString(token);
+								f->SetData(s_f);
+							}
+							break;
+						case FIELD_STRING:
+field_string_override:
+							{
+								StringField *s_f = new StringField();
+								s_f->SetNDEString(token);
+								f->SetData(s_f);
+							}
+							break;
+						default:
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+							break;
+						}
+						// pop all operators in this level beginning by the last inserted
+						entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+						while (entry)
+						{
+							if (entry->GetVal().Level == pcount)
+							{
+								AddFilterOp(entry->GetVal().Op);
+								VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+								pstack.RemoveEntry(entry);
+								entry = _entry;
+							}
+							else
+								break;
+						}
+						break;
+					}
+		//          default:
+					//          Query_SyntaxError(p-query);
+					//        return FALSE;
+	//        }
+				break;
+			case 3:
+				switch (t)
+				{
+					case TOKEN_SQBRACKETOPEN:
+					{
+						const wchar_t *s = wcschr(p, L']');
+						if (!s)
+						{
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+						p = Query_EatSpace(p);
+						if (p && *p == L'[') p++;
+						wchar_t *format = ndestring_wcsndup(p, s-p);
+						Filter *f = GetLastFilter();
+						int id = f->GetId();
+						ColumnField *c = GetColumnById(id);
+						int tt = c ? c->GetDataType() : -1;
+						if (disable_date_resolution || !c || (tt != FIELD_INTEGER && tt != FIELD_DATETIME && tt != FIELD_LENGTH && tt != FIELD_BOOLEAN))
+						{
+							if (disable_date_resolution)
+							{
+								StringField *field = (StringField *)f->Data();
+
+								if (!field)
+								{
+									// format was used without a value, assume value is 0
+									f->SetData(new StringField(L""));
+									entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+									while (entry)
+									{
+										if (entry->GetVal().Level == pcount)
+										{
+											AddFilterOp(entry->GetVal().Op);
+											VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+											pstack.RemoveEntry(entry);
+											entry = _entry;
+										}
+										else
+											break;
+									}
+									field = (StringField *)f->Data();
+								}
+
+								field->SetNDEString(format);
+								ndestring_release(format);
+								p = s+1;
+								continue;
+							}
+							ndestring_release(format);
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+
+						IntegerField *field = (IntegerField *)f->Data();
+
+						if (!field)
+						{
+							// format was used without a value, assume value is 0
+							f->SetData(new IntegerField(0));
+							entry = (VListEntry<OpLevel> *)pstack.GetFoot();
+							while (entry)
+							{
+								if (entry->GetVal().Level == pcount)
+								{
+									AddFilterOp(entry->GetVal().Op);
+									VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+									pstack.RemoveEntry(entry);
+									entry = _entry;
+								}
+								else
+									break;
+							}
+							field = (IntegerField *)f->Data();
+						}
+						int r = field->ApplyConversion(format);
+						ndestring_release(format);
+						if (!r)
+						{
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+						p = s+1;
+						continue;
+					}
+					break;
+
+					case TOKEN_PARCLOSE:
+						state = 4;
+						// check parenthesis count
+						if (pcount == 0)
+						{
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+						// down one level
+						pcount--;
+						// pop all operators in this level, beginning by the last inserted
+						while (entry)
+						{
+							if (entry->GetVal().Level == pcount)
+							{
+								AddFilterOp(entry->GetVal().Op);
+								VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+								pstack.RemoveEntry(entry);
+								entry = _entry;
+							}
+							else
+								break;
+						}
+						break;
+					case TOKEN_AND:
+					{
+						state = 0;
+						// push and
+						OpLevel o;
+						o.Op = FILTER_AND;
+						o.Level = pcount;
+						entry = new VListEntry<OpLevel>;
+						entry->SetVal(o);
+						pstack.AddEntry(entry, TRUE);
+						break;
+					}
+					case TOKEN_OR:
+					{
+						state = 0;
+						// push or
+						OpLevel o;
+						o.Op = FILTER_OR;
+						o.Level = pcount;
+						entry = new VListEntry<OpLevel>;
+						entry->SetVal(o);
+						pstack.AddEntry(entry, TRUE);
+						break;
+					}
+					default:
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+				}
+				break;
+			case 4:
+				switch (t)
+				{
+					case TOKEN_AND:
+					{
+						state = 0;
+						// push and
+						OpLevel o;
+						o.Op = FILTER_AND;
+						o.Level = pcount;
+						entry = new VListEntry<OpLevel>;
+						entry->SetVal(o);
+						pstack.AddEntry(entry, TRUE);
+						break;
+					}
+					case TOKEN_OR:
+					{
+						state = 0;
+						// push or
+						OpLevel o;
+						o.Op = FILTER_OR;
+						o.Level = pcount;
+						entry = new VListEntry<OpLevel>;
+						entry->SetVal(o);
+						pstack.AddEntry(entry, TRUE);
+						break;
+					}
+					case TOKEN_PARCLOSE:
+						state = 4;
+						// check parenthesis count
+						if (pcount == 0)
+						{
+							Query_SyntaxError((int)(p-query));
+							return FALSE;
+						}
+						// down one level
+						pcount--;
+						// pop all operators in this level, beginning by the last inserted
+						while (entry)
+						{
+							if (entry->GetVal().Level == pcount)
+							{
+								AddFilterOp(entry->GetVal().Op);
+								VListEntry<OpLevel> *_entry = (VListEntry<OpLevel> *)entry->GetPrevious();
+								pstack.RemoveEntry(entry);
+								entry = _entry;
+							}
+							else
+								break;
+						}
+						break;
+					default:
+						Query_SyntaxError((int)(p-query));
+						return FALSE;
+				}
+				break;
+			default:
+				// Ahem... :/
+				break;
+		}
+		p += size;
+	}
+	if (pcount > 0)
+	{
+		Query_SyntaxError((int)(p-query));
+		return FALSE;
+	}
+	return TRUE;
+}
+
+//---------------------------------------------------------------------------
+
+void Scanner::Query_SyntaxError(int c)
+{
+}
+
+//---------------------------------------------------------------------------
+void Scanner::Query_CleanUp()
+{
+	while (pstack.GetNElements() > 0)
+	{
+		VListEntry<int> *e;
+		e = (VListEntry<int> *)pstack.GetHead();
+		pstack.RemoveEntry(e);
+	}
+}
+
+
+//---------------------------------------------------------------------------
+const wchar_t *Scanner::Query_EatSpace(const wchar_t *p)
+{
+	while (p && *p && *p == L' ') p++;
+	return p;
+}
+
+//---------------------------------------------------------------------------
+const wchar_t *Scanner::Query_ProbeNonAlphaNum(const wchar_t *p)
+{
+	int inquote=0;
+	while (p && *p && (!Query_isControlChar(*p) || (inquote)))
+	{
+		if (*p == L'\"')
+		{
+			if (!inquote)
+				inquote = 1;
+			else
+				return p+1;
+		}
+		p++;
+	}
+	return p;
+}
+
+//---------------------------------------------------------------------------
+int Scanner::Query_isControlChar(wchar_t p)
+{
+	switch (p)
+	{
+		case L'&':
+		case L'|':
+		case L'!':
+		case L'(':
+		case L'[':
+		case L')':
+		case L']':
+		case L'>':
+		case L'<':
+		case L'=':
+		case L',':
+		case L' ':
+			return TRUE;
+	}
+	return FALSE;
+}
+
+//---------------------------------------------------------------------------
+wchar_t *Scanner::Query_ProbeAlphaNum(wchar_t *p)
+{
+	while (p && *p && Query_isControlChar(*p)) p++;
+	return p;
+}
+
+//---------------------------------------------------------------------------
+wchar_t *Scanner::Query_ProbeSpace(wchar_t *p)
+{
+	while (p && *p && *p != ' ') p++;
+	return p;
+}
+
+//---------------------------------------------------------------------------
+int Scanner::Query_LookupToken(const wchar_t *t)
+{
+	for (int i=0;i<sizeof(Tokens)/sizeof(tokenstruct);i++)
+	{
+		if (!_wcsicmp(Tokens[i].token, t))
+			return Tokens[i].tid;
+	}
+	return TOKEN_IDENTIFIER;
+}
+
+//---------------------------------------------------------------------------
+
+int Scanner::Query_GetNextToken(const wchar_t *p, int *size, wchar_t **_token, int tokentable)
+{
+	int t = TOKEN_EOQ;
+	const wchar_t *startptr = p;
+
+	if (!p || !*p) return TOKEN_EOQ;
+
+	p = Query_EatSpace(p);
+
+	const wchar_t *e = Query_ProbeNonAlphaNum(p);
+
+	if (e != p)   // We have a word
+	{
+		size_t token_length = e-p;
+		if (*_token) ndestring_release(*_token);
+		*_token = ndestring_wcsndup(p, token_length);
+		if (*(*_token) == L'\"' && (*_token)[token_length-1] == L'\"') // check for quoted string
+		{
+			int l=(int)token_length-2;
+			if (l>0)
+			{
+				memcpy(*_token,(*_token)+1,l*sizeof(wchar_t));
+				(*_token)[l]=0;
+				Query_Unescape(*_token); 
+			}
+			else
+				(*_token)[0]=0;// we have an empty string
+		}
+
+		switch (tokentable)
+		{
+			case -1:
+				t = TOKEN_IDENTIFIER;
+				break;
+			case 0:
+				t = Query_LookupToken(*_token);
+				break;
+			case 1:
+				t = IntegerField::LookupToken(*_token);
+		}
+		p = e;
+	}
+	else   // We have a symbol
+	{
+		switch (*p)
+		{
+			case L'&':
+				if (*(p+1) == L'&') p++;
+				t = TOKEN_AND;
+				break;
+			case L'|':
+				if (*(p+1) == L'|') p++;
+				t = TOKEN_OR;
+				break;
+			case L'!':
+				if (*(p+1) == L'=')
+				{
+					p++;
+					t = TOKEN_NOTEQUAL;
+					break;
+				}
+				t = TOKEN_NOT;
+				break;
+			case L'(':
+				t = TOKEN_PAROPEN;
+				break;
+			case L')':
+				t = TOKEN_PARCLOSE;
+				break;
+			case L'[':
+				t = TOKEN_SQBRACKETOPEN;
+				break;
+			case L']':
+				t = TOKEN_SQBRACKETCLOSE;
+				break;
+			case L',':
+				t = TOKEN_COMMA;
+				break;
+			case L'>':
+				if (*(p+1) == L'=')
+				{
+					p++;
+					t = TOKEN_AOREQUAL;
+					break;
+				}
+				if (*(p+1) == L'<')
+				{
+					p++;
+					t = TOKEN_NOTEQUAL;
+					break;
+				}
+				t = TOKEN_ABOVE;
+				break;
+			case L'<':
+				if (*(p+1) == L'=')
+				{
+					p++;
+					t = TOKEN_BOREQUAL;
+					break;
+				}
+				if (*(p+1) == L'>')
+				{
+					p++;
+					t = TOKEN_NOTEQUAL;
+					break;
+				}
+				t = TOKEN_BELOW;
+				break;
+			case L'=':
+				if (*(p+1) == L'>')
+				{
+					p++;
+					t = TOKEN_AOREQUAL;
+					break;
+				}
+				if (*(p+1) == L'<')
+				{
+					p++;
+					t = TOKEN_BOREQUAL;
+					break;
+				}
+				if (*(p+1) == L'!')
+				{
+					p++;
+					t = TOKEN_NOTEQUAL;
+					break;
+				}
+				if (*(p+1) == L'=') p++;
+				t = TOKEN_EQUAL;
+				break;
+			default:
+				t = TOKEN_UNKNOWN;
+				break;
+		}
+		p++;
+	}
+
+	*size = (int)(p - startptr);
+	return t;
+}
+
+static uint8_t quickhex(wchar_t c)
+{
+	int hexvalue = c;
+	if (hexvalue & 0x10)
+		hexvalue &= ~0x30;
+	else
+	{
+		hexvalue &= 0xF;
+		hexvalue += 9;
+	}
+	return hexvalue;
+}
+
+static uint8_t DecodeEscape(const wchar_t *&str)
+{
+	uint8_t a = quickhex(*++str);
+	uint8_t b = quickhex(*++str);
+	str++;
+	return a * 16 + b;
+}
+
+static void DecodeEscapedUTF8(wchar_t *&output, const wchar_t *&input)
+{
+	uint8_t utf8_data[1024] = {0}; // hopefully big enough!!
+	int num_utf8_words=0;
+	bool error=false;
+
+	while (input && *input == '%' && num_utf8_words < sizeof(utf8_data))
+	{
+		if (iswxdigit(input[1]) && iswxdigit(input[2]))
+		{
+			utf8_data[num_utf8_words++]=DecodeEscape(input);
+		}
+		else if (input[1] == '%')
+		{
+			input+=2;
+			utf8_data[num_utf8_words++]='%';
+		}
+		else
+		{
+			error = true;
+			break;
+		}
+	}
+
+	int len = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)utf8_data, num_utf8_words, 0, 0);
+	MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)utf8_data, num_utf8_words, output, len);
+	output += len;
+
+	if (error)
+	{
+		*output++ = *input++;
+	}
+}
+
+// benski> We have the luxury of knowing that decoding will ALWAYS produce smaller strings
+// so we can do it in-place
+void Query_Unescape(wchar_t *str)
+{
+	const wchar_t *itr = str;
+	while (itr && *itr)
+	{
+		switch (*itr)
+		{
+			case '%':
+				DecodeEscapedUTF8(str, itr);
+				break;
+			default:
+				*str++ = *itr++;
+				break;
+		}
+	}
+	*str = 0;
+}

Src/nde/win/Query.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#define TOKEN_UNKNOWN        -1 // BLAAAAA!!!!!!!!
+#define TOKEN_EOQ             0 // End of Query
+#define TOKEN_IDENTIFIER      1 // Column name or data to match against
+#define TOKEN_EQUAL           2 // =, ==, IS
+#define TOKEN_NOTEQUAL        3 // !=, =!, <>, ><
+#define TOKEN_BELOW           4 // <
+#define TOKEN_ABOVE           5 // >
+#define TOKEN_BOREQUAL        6 // <=, =<
+#define TOKEN_AOREQUAL        7 // >=, =>
+#define TOKEN_NOT             8 // !, NOT
+#define TOKEN_AND             9 // &, &&, AND
+#define TOKEN_OR             10 // |, ||, OR
+#define TOKEN_PAROPEN        11 // (
+#define TOKEN_PARCLOSE       12 // )
+#define TOKEN_CONTAINS       13 // HAS
+#define TOKEN_BEGINS         14	// string starts with...
+#define TOKEN_ENDS           15	// string ends with...
+#define TOKEN_LIKE           16 // string is nearly (excluding "the " and whitespace etc)
+#define TOKEN_ISEMPTY        17	// field does not exists
+#define TOKEN_SQBRACKETOPEN  18 // [
+#define TOKEN_SQBRACKETCLOSE 19 // ]
+#define TOKEN_COMMA          20 // ,
+#define TOKEN_NOTCONTAINS    21    // NOTHAS
+#define TOKEN_ISNOTEMPTY     22	// field does not exists
+#define TOKEN_BEGINSLIKE     23 // string is nearly starts with (excluding "the " and whitespace etc)
+
+// in-place
+void Query_Unescape(wchar_t *p);

Src/nde/win/Record.cpp

@@ -0,0 +1,133 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+Record Class
+
+--------------------------------------------------------------------------- */
+
+//#include "record.h"
+#include "../nde.h"
+#include <stdio.h>
+
+void RecordBase::Retain()
+{
+	ref_count++;
+}
+
+void RecordBase::Release()
+{
+	if (--ref_count == 0)
+		delete this;
+}
+
+RecordBase::RecordBase()
+{
+	ref_count = 1;
+	InsertionPoint = 0;
+}
+
+//---------------------------------------------------------------------------
+Record::Record(int RecordPos,  int insertionPoint, VFILE *TableHandle, Table *ParentTable)
+{
+	InsertionPoint = insertionPoint;
+	Record *columns = ParentTable->GetColumns();
+	int max=columns ? (int)columns->Fields.size() : 128;
+	if (RecordPos != 0)
+	{
+		int n=0;
+		uint32_t ThisPos = RecordPos;
+		while (ThisPos)
+		{
+			if (n >= max)
+				break;
+			Vfseek(TableHandle, ThisPos, SEEK_SET);
+			Field Entry (ThisPos);
+			Field *TypedEntry = Entry.ReadField(ParentTable, ThisPos, &ThisPos);
+
+			if (!TypedEntry) break; // some db error?
+
+			AddField(TypedEntry);
+			n++;
+		}
+	}
+}
+
+//---------------------------------------------------------------------------
+RecordBase::~RecordBase()
+{
+	//Fields.deleteAll();
+	for (Field* field : Fields)
+	{
+		if (field)
+		{
+			delete field;
+		}
+	}
+	Fields.clear();
+}
+
+ColumnField *Record::GetColumnByName(const wchar_t *name)
+{
+	for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+	{
+		ColumnField *p = (ColumnField *)*itr;
+		if (!_wcsicmp(p->GetFieldName(), name))
+			return p;
+	}
+	return NULL;
+}
+
+//---------------------------------------------------------------------------
+int Record::WriteFields(Table *ParentTable, int RecordIndex)
+{
+	Field *previous = 0;
+	for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+	{
+		Field *p = *itr;
+
+		Field* nextField = (itr + 1) != Fields.end() ? *(itr + 1) : nullptr;
+
+		//p->WriteField(ParentTable, previous, (Field*)p->next);
+		p->WriteField(ParentTable, previous, nextField);
+		
+		previous = p;
+	}
+	return WriteIndex(ParentTable, RecordIndex);
+}
+
+//---------------------------------------------------------------------------
+int Record::WriteIndex(Table *ParentTable, int RecordIndex)
+{
+	int P=0;
+	if (RecordIndex == NEW_RECORD)
+		RecordIndex = ParentTable->index->Insert(InsertionPoint);
+	if (!Fields.empty())
+	{
+		Field *f = *Fields.begin();
+		P=f->GetFieldPos();
+	}
+	return ParentTable->index->Update(RecordIndex, P, this, FALSE);
+}
+
+//---------------------------------------------------------------------------
+void Record::Delete(Table *ParentTable, int RecordIndex)
+{
+	ParentTable->index->Delete(RecordIndex, ParentTable->index->Get(RecordIndex), this);
+}
+
+void Record::WalkFields(FieldsWalker callback, void *context)
+{
+	if (callback)
+	{
+		for (FieldList::iterator itr=Fields.begin();itr!=Fields.end();itr++)
+		{
+			if (!callback(this, *itr, context))
+				break;
+		}
+	}
+}

Src/nde/win/Record.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "../nde.h"
+#include "../Field.h"
+#include "Vfs.h"
+#include <stdio.h>
+#include <deque>
+
+class ColumnField;
+
+class RecordBase
+{
+public:
+	RecordBase();
+	virtual ~RecordBase();
+	Field *GetField(unsigned char ID);
+	void Retain();
+	void Release();
+	void RemoveField(Field *field);
+	void AddField(Field *field);
+	Field* GetLastField()
+	{
+		if (Fields.empty())
+		{
+			return nullptr;
+		}
+		return Fields[Fields.size() - 1];
+	}
+protected:
+	void Undelete(void);
+
+	int InsertionPoint; // TODO: benski> might be able to pass this into WriteFields/WriteIndex
+	int ref_count;
+	typedef std::deque<Field*> FieldList;
+	FieldList Fields;
+};
+
+class Record : public RecordBase
+{
+public:
+	Record(int RecordPos, int insertionPoint, VFILE *FileHandle,Table *p);
+	bool InCache() { return ref_count > 1; }
+
+	ColumnField *GetColumnByName(const wchar_t *name);
+	int WriteFields(Table *ParentTable, int RecordIndex);
+	int WriteIndex(Table *ParentTable, int RecordIndex);
+	void Delete(Table *ParentTable, int RecordIndex);
+	typedef bool (__cdecl *FieldsWalker)(Record *record, Field *entry, void *context);
+	NDE_API void WalkFields(FieldsWalker callback, void *context);
+};

Src/nde/win/Scanner.h

@@ -0,0 +1,154 @@
+/* ---------------------------------------------------------------------------
+ Nullsoft Database Engine
+ --------------------
+ codename: Near Death Experience
+ --------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+ 
+ Scanner Class Prototypes
+ 
+ --------------------------------------------------------------------------- */
+
+#ifndef __SCANNER_H
+#define __SCANNER_H
+
+#include <vector>
+#include "record.h"
+#include "Table.h"
+#include "index.h"
+#include <vector>
+#include <set>
+#ifdef __APPLE__
+#include <CoreFoundation/CoreFoundation.h>
+#endif
+class Table;
+class Index;
+#pragma warning( disable: 4251 )
+class Scanner;
+class Record;
+class Scanner : public LinkedListEntry 
+{
+public:
+	Record *GetRecord(int Idx);
+	Scanner(Table *parentTable);
+	void IndexModified(void);
+	Index *GetIndex() { return index; }
+	Index *index; // TODO: make protected
+
+protected:
+	~Scanner();
+
+	Table *pTable;
+	BOOL iModified;
+	typedef std::vector<StringField *> SearchStrings;
+	SearchStrings search_strings;
+	typedef std::set<unsigned char> SearchFields;
+	SearchFields search_fields;
+	bool search_any;
+
+	void GetCurrentRecord(void);
+	bool MatchFilters(void);
+	bool MatchSearches();
+	bool MatchSearch(const SearchFields &fields, StringField *search_field);
+	//BOOL MatchJoins(void);
+	int CheckFilters(void);
+	void CacheLastLocate(int Id, int From, Field *field, Index *i, int j);
+
+	static int Query_LookupToken(const wchar_t *token);
+	void Query_CleanUp(void);
+	void Query_SyntaxError(int c);
+public:
+	static int Query_GetNextToken(const wchar_t *p, int *size, wchar_t **token, int tokentable=0);
+	static const wchar_t *Query_EatSpace(const wchar_t *p);
+	static wchar_t *Query_ProbeSpace(wchar_t *p);
+	static const wchar_t *Query_ProbeNonAlphaNum(const wchar_t *p);
+	static wchar_t *Query_ProbeAlphaNum(wchar_t *p);
+	static int Query_isControlChar(wchar_t p);
+
+	BOOL Query(const wchar_t *query);
+	BOOL Query_Parse(const wchar_t *query);
+
+	const wchar_t *GetLastQuery();
+
+public://fucko: protected
+	LinkedList pstack;
+	wchar_t *token;
+	wchar_t *last_query;
+	int last_query_failed;
+
+protected:
+	Record *CurrentRecord;
+	int CurrentRecordIdx;
+	LinkedList FilterList;
+	Index *lastLocateIndex;
+	int lastLocateIdx;
+	Field *lastLocateFieldClone;
+	int lastLocateFrom;
+	int lastLocateId;
+	BOOL Edition;
+	int ResultPtr;
+	BOOL FiltersOK;
+
+public:
+	bool MatchFilter(Filter *filter);
+	typedef bool (*FilterWalker)(Scanner *scanner, Filter *filter, void *context);
+	void WalkFilters(FilterWalker walker, void *context);
+
+	ColumnField *GetColumnByName(const wchar_t *FieldName);
+	ColumnField *GetColumnById(unsigned char id);
+
+	Field *NewFieldByName(const wchar_t *fieldName, unsigned char Perm);
+	Field *NewFieldByType(unsigned char Type, unsigned char Id, unsigned char Perm);
+	Field *NewFieldById(unsigned char Id, unsigned char Perm);
+	void DeleteField(Field *field);
+	void DeleteFieldByName(const wchar_t *name); 
+	void DeleteFieldById(unsigned char Id);
+
+	void Cancel(void);
+	void Insert(void);
+	void Edit(void);
+	void Post(void);
+	void Delete(void); 
+
+	Field *GetFieldByName(const wchar_t *FieldName);
+	Field *GetFieldById(unsigned char Id);
+
+	void First(int *killswitch=0);
+	void Last(int *killswitch=0);
+	int Next(int *killswitch=0);
+	int Previous(int *killswitch=0);
+	BOOL Eof(void);
+	BOOL Bof(void);
+	void New(void);
+	int GetRecordsCount(void);
+	void GetRecordById(int Id, BOOL checkFilters=TRUE);
+	int GetRecordId(void);
+	void Sync(void);
+	BOOL LocateByName(const wchar_t *column, int From, Field *field, int *nskip=NULL);
+	BOOL LocateById(int Id, int From, Field *field, int *nskip=NULL);
+	BOOL LocateByIdEx(int Id, int From, Field *field, int *nskip, int comp_mode);
+
+	// Filters
+	int AddFilterByName(const wchar_t *name, Field *Data, unsigned char Op);
+	int AddFilterById(unsigned char Id, Field *Data, unsigned char Op);
+	int AddFilterOp(unsigned char Op);
+	void RemoveFilters(void);
+	Filter *GetLastFilter(void);
+
+	BOOL SetWorkingIndexByName(const wchar_t *desc);
+	BOOL SetWorkingIndexById(unsigned char Id);
+
+	void Search(const wchar_t *search_string);
+	BOOL HasIndexChanged(void) { return iModified; }
+	void ClearDirtyBit(void);
+	float FragmentationLevel(void);
+
+	void WalkFields(Record::FieldsWalker callback, void *context);
+
+	Table *GetTable();
+	int in_query_parser;
+	int disable_date_resolution;
+};
+
+#endif

Src/nde/win/StringField.cpp

@@ -0,0 +1,341 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+StringField Class
+Windows specific version
+
+Field data layout:
+[2 bytes] string length (bytes)
+[length bytes] String data.  UTF-16 data will start with a BOM 
+--------------------------------------------------------------------------- */
+
+#include "../nde.h"
+#include "StringField.h"
+#include "../../nu/AutoChar.h"
+#include "../../nu/AutoWide.h"
+
+static wchar_t CharSwap(wchar_t value)
+{
+	return (value >> 8) | (value << 8);
+}
+
+//---------------------------------------------------------------------------
+StringField::StringField(const wchar_t *Str, int strkind)
+{
+	InitField();
+	Type = FIELD_STRING;
+	if (Str)
+	{
+		if (strkind == STRING_IS_WCHAR)
+			StringW = ndestring_wcsdup(Str);
+		else
+		{
+			StringW = const_cast<wchar_t *>(Str);
+			ndestring_retain(StringW);
+		}
+	}
+}
+
+//---------------------------------------------------------------------------
+void StringField::InitField(void)
+{
+	Type = FIELD_STRING;
+	StringW = NULL;
+	optimized_the = 0;
+}
+
+//---------------------------------------------------------------------------
+StringField::StringField()
+{
+	InitField();
+}
+
+//---------------------------------------------------------------------------
+StringField::~StringField()
+{
+	ndestring_release(StringW);
+	StringW=0;
+}
+
+//---------------------------------------------------------------------------
+void StringField::ReadTypedData(const uint8_t *data, size_t len)
+{
+	unsigned short c;
+
+	CHECK_SHORT(len);
+	c = (unsigned short)(data[0]|(data[1]<<8));
+	data+=2;
+	if (c)
+	{
+		bool unicode=false;
+		bool reverseEndian=false;
+		if (c >= 2 // enough room for BOM
+			&& (c % 2) == 0) // can't be unicode if it's not an even multiple of 2
+		{
+			wchar_t BOM=0;
+			memcpy(&BOM, data, 2);
+			if (BOM == 0xFEFF)
+			{
+				data+=2;
+				c-=2;	
+				unicode=true;
+			}
+			else if (BOM == 0xFFFE)
+			{
+				data+=2;
+				c-=2;	
+				unicode=true;
+				reverseEndian=true;
+			}
+		}
+
+		CHECK_BIN(len, c);
+		if (unicode)
+		{
+			ndestring_release(StringW);
+			StringW = ndestring_malloc(c+sizeof(wchar_t));
+
+			memcpy(StringW, data, c);
+			StringW[c/2]=0;
+			if (reverseEndian)
+			{
+				for (unsigned short i=0;i<c;i++)
+					StringW[i]=CharSwap(StringW[i]);
+			}
+		}
+		else
+		{
+			int len = MultiByteToWideChar(CP_ACP, 0, (LPCSTR)data, c, 0, 0);
+			StringW = ndestring_malloc((len+1)*sizeof(wchar_t));
+			MultiByteToWideChar(CP_ACP, 0, (LPCSTR)data, c, StringW, len);
+			StringW[len]=0;
+		}
+	}
+}
+
+//---------------------------------------------------------------------------
+void StringField::WriteTypedData(uint8_t *data, size_t len)
+{
+	int pos=0;
+
+	if (StringW)
+	{
+		unsigned short c = (unsigned short)wcslen(StringW) * sizeof(wchar_t) + 2 /* for BOM */;
+		// write size
+		CHECK_SHORT(len);
+		PUT_SHORT(c); pos+=2;
+
+		// write byte order mark
+		CHECK_BIN(len, 2);
+		wchar_t BOM = 0xFEFF;
+		PUT_BINARY(data, (uint8_t *)&BOM, 2, pos);
+		pos+=2;
+		c-=2;
+
+		// write string
+		CHECK_BIN(len, c);
+		PUT_BINARY(data, (uint8_t *)StringW, c, pos);
+	}
+	else
+	{
+		CHECK_SHORT(len);
+		PUT_SHORT(0); /*pos+=2;*/
+	}
+}
+
+//---------------------------------------------------------------------------
+wchar_t *StringField::GetStringW(void)
+{
+	return StringW;
+}
+
+//---------------------------------------------------------------------------
+void StringField::SetStringW(const wchar_t *Str)
+{
+	if (!Str) return;
+
+	ndestring_release(StringW);
+	StringW = NULL;
+	StringW = ndestring_wcsdup(Str);
+	optimized_the=0;
+}
+
+//---------------------------------------------------------------------------
+void StringField::SetNDEString(wchar_t *Str)
+{
+	if (!Str) return;
+
+	// copy and then release, just in case we're copying into ourselves
+	wchar_t *oldStr = StringW;
+	StringW = Str;
+	ndestring_retain(StringW);
+	ndestring_release(oldStr);
+	optimized_the=0;
+}
+
+//---------------------------------------------------------------------------
+size_t StringField::GetDataSize(void)
+{
+	if (StringW)
+	{
+		return wcslen(StringW)*2 +2 /*for BOM*/ + 2 /*for byte length*/;
+	}
+	else
+	{
+		return 2;
+	}
+}
+
+//---------------------------------------------------------------------------
+int StringField::Compare(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (Entry->GetType() != GetType()) return 0;
+	return mywcsicmp(GetStringW(), ((StringField*)Entry)->GetStringW());
+}
+
+//---------------------------------------------------------------------------
+int StringField::Starts(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (Entry->GetType() != GetType()) return 0;
+	const wchar_t *p = ((StringField*)Entry)->GetStringW();
+	const wchar_t *d = GetStringW();
+	if (!d || !p) return 0;
+	return nde_wcsbegins(d, p);
+}
+
+//---------------------------------------------------------------------------
+int StringField::Contains(Field *Entry)
+{
+	if (!Entry) return -1;
+	if (Entry->GetType() != GetType()) return 0;
+	const wchar_t *p = ((StringField*)Entry)->GetStringW();
+	const wchar_t *d = GetStringW();
+	if (!d || !p) return 0;
+	return nde_wcscontains(GetStringW(), ((StringField*)Entry)->GetStringW());
+}
+
+Field *StringField::Clone(Table *pTable)
+{
+	StringField *clone = new StringField(StringW, STRING_IS_NDESTRING);
+	clone->Pos = FIELD_CLONE;
+	clone->ID = ID;
+	clone->MaxSizeOnDisk = (uint32_t)GetDataSize();
+	return clone;
+}
+
+// todo: make configurable words to skip, as well as trailing whitespace removal
+#define IsCharSpaceW(c) (c == L' ' || c == L'\t')
+inline bool IsTheW(const wchar_t *str) { if (str && (str[0] == L't' || str[0] == L'T') && (str[1] == L'h' || str[1] == L'H') && (str[2] == L'e' || str[2] == L'E') && (str[3] == L' ')) return true; else return false; }
+#define SKIP_THE_AND_WHITESPACEW(x) { wchar_t *save##x=(wchar_t*)x; while (IsCharSpaceW(*x) && *x) x++; if (IsTheW(x)) x+=4; while (IsCharSpaceW(*x)) x++; if (!*x) x=save##x; }
+
+bool StringField::ApplyFilter(Field *Data, int op)
+{
+	// TODO: maybe do this?
+
+	if (op == FILTER_ISEMPTY || op == FILTER_ISNOTEMPTY)
+	{
+		bool r = (op == FILTER_ISEMPTY);
+		if (!StringW)
+			return r;
+
+		if (StringW && StringW[0] == 0)
+			return r;
+
+		return !r;
+	}
+	//
+	bool r;
+	StringField *compField = (StringField *)Data;
+
+	const wchar_t *p = compField->GetStringW();
+	const wchar_t *d = GetStringW();
+	if (!p)
+		p = L"";
+	if (!d)
+		d = L"";
+
+	switch (op)
+	{
+		case FILTER_EQUALS:
+			r = !nde_wcsicmp(d, p);
+			break;
+		case FILTER_NOTEQUALS:
+			r = !!nde_wcsicmp(d, p);
+			break;
+		case FILTER_CONTAINS:
+			r = nde_wcscontains(d, p);
+			break;
+		case FILTER_NOTCONTAINS:
+			r = !nde_wcscontains(d, p);
+			break;
+		case FILTER_ABOVE:
+			r = (bool)(nde_wcsicmp(d, p) > 0);
+			break;
+		case FILTER_ABOVEOREQUAL:
+			r = (bool)(nde_wcsicmp(d, p) >= 0);
+			break;
+		case FILTER_BELOW:
+			r = (bool)(nde_wcsicmp(d, p) < 0);
+			break;
+		case FILTER_BELOWOREQUAL:
+			r = (bool)(nde_wcsicmp(d, p) <= 0);
+			break;
+		case FILTER_BEGINS:
+			r = nde_wcsbegins(d, p);
+			break;
+		case FILTER_ENDS:
+				r = nde_wcsends(d, p);
+			break;
+		case FILTER_LIKE:
+
+			if (compField->optimized_the)
+				p = compField->optimized_the;
+			else
+			{
+				SKIP_THE_AND_WHITESPACEW(p);
+				compField->optimized_the = p;				
+			}
+
+			if (optimized_the)
+				d = optimized_the;
+			else
+			{
+				SKIP_THE_AND_WHITESPACEW(d);
+				optimized_the=d;
+			}
+
+			r = (bool)(nde_wcsicmp(d, p) == 0);
+			break;
+		case FILTER_BEGINSLIKE:
+
+			if (compField->optimized_the)
+				p = compField->optimized_the;
+			else
+			{
+				SKIP_THE_AND_WHITESPACEW(p);
+				compField->optimized_the = p;				
+			}
+
+			if (optimized_the)
+				d = optimized_the;
+			else
+			{
+				SKIP_THE_AND_WHITESPACEW(d);
+				optimized_the=d;
+			}
+
+			r = nde_wcsbegins(d, p);
+			break;
+		default:
+			r = true;
+			break;
+	}
+	return r;
+}

Src/nde/win/StringField.h

@@ -0,0 +1,50 @@
+/* 
+---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+---------------------------------------------------------------------------
+*/
+
+/* 
+---------------------------------------------------------------------------
+
+StringField Class Prototypes
+
+---------------------------------------------------------------------------
+*/
+
+#ifndef __STRINGFIELD_H
+#define __STRINGFIELD_H
+#include "../NDEString.h"
+
+class StringField : public Field
+{
+public:
+	StringField();
+	~StringField();
+	
+
+	StringField(const wchar_t *Str, int strkind=STRING_IS_WCHAR);
+	wchar_t *GetStringW(void);
+	void SetStringW(const wchar_t *Str);
+	void SetNDEString(wchar_t *Str);
+
+protected:
+	virtual void ReadTypedData(const uint8_t *, size_t len);
+	virtual void WriteTypedData(uint8_t *, size_t len);
+	virtual size_t GetDataSize(void);
+	virtual int Compare(Field *Entry);
+	virtual int Starts(Field *Entry);
+	virtual int Contains(Field *Entry);
+
+	virtual bool ApplyFilter(Field *Data, int op);
+	virtual Field *Clone(Table *pTable);
+	void InitField(void);
+	
+	wchar_t *StringW;
+	const wchar_t *optimized_the;
+};
+
+#endif
+

Src/nde/win/Table.cpp

@@ -0,0 +1,886 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+Table Class
+
+--------------------------------------------------------------------------- */
+#include "../nde.h"
+//#include <direct.h>
+#include <stdio.h>
+#include <string.h>
+#include "../CRC.H"
+#include <strsafe.h>
+
+const char *tSign="NDETABLE";
+
+//---------------------------------------------------------------------------
+Table::Table( const wchar_t *TableName, const wchar_t *Idx, BOOL Create, Database *_db, BOOL _Cached ) : Scanner( this )
+{
+	memset( column_ids, FIELD_UNKNOWN, 255 );
+
+	Cached     = _Cached;
+	db         = _db;
+	AutoCreate = Create;
+	Name       = ndestring_wcsdup( TableName );
+	IdxName    = ndestring_wcsdup( Idx );
+
+	Init();
+}
+
+//---------------------------------------------------------------------------
+void Table::Init()
+{
+	numErrors       = 0;
+	Scanners        = new LinkedList();
+	// benski> cut: Handle=NULL;
+	IdxHandle       = NULL;
+	FieldsRecord    = NULL;
+	IndexList       = NULL;
+	GLocateUpToDate = FALSE;
+}
+
+//---------------------------------------------------------------------------
+Table::~Table()
+{
+	Reset();
+	if (Handle) // Reset doesn'l_data_type completely destroy Handle
+		Vfdestroy(Handle);
+	Handle = 0;
+	
+	ndestring_release(Name);
+	ndestring_release(IdxName);
+}
+
+//---------------------------------------------------------------------------
+void Table::Reset()
+{
+	if ( IndexList )
+		IndexList->Release();
+
+	IndexList = 0;
+
+	if ( FieldsRecord )
+		FieldsRecord->Release();
+
+	FieldsRecord = 0;
+
+	delete Scanners;
+	Scanners = NULL;
+
+	if ( Handle )
+		Vfclose( Handle ); // close (but don'l_data_type destroy) to keep mutex open.
+	if ( IdxHandle )
+		Vfdestroy( IdxHandle );
+
+	IdxHandle = NULL;
+
+	for ( RowCache::iterator itr = rowCache.begin(); itr != rowCache.end(); ++itr )
+	{
+		if ( itr->second )
+			itr->second->Release();
+	}
+
+	rowCache.clear();
+
+	memset( column_ids, FIELD_UNKNOWN, 255 );
+	columns_cached = false;
+}
+
+struct IndexNewWalkerContext
+{
+	IndexNewWalkerContext(Table *_table)
+	{
+		N = -1;
+		table = _table;
+	}
+	int N;
+	Table *table;
+};
+
+bool Table::IndexNewWalker(IndexRecord *record, Field *entry, void *context_in)
+{
+	IndexNewWalkerContext *context = (IndexNewWalkerContext *)context_in;
+	IndexField *p = (IndexField *)entry;
+	p->index = new Index(context->table->IdxHandle, p->ID, context->N++, p->Type, FALSE, 0, context->table);
+	return true;
+}
+
+//---------------------------------------------------------------------------
+BOOL Table::Open(void)
+{
+	int justcreated = 0;
+
+	if (!Handle)
+		Handle = Vfnew(Name, "r+b", Cached);
+	if (!Handle) return FALSE;
+	if (!Vflock(Handle))
+	{
+		Vfdestroy(Handle);
+		Handle = 0;
+		return FALSE;
+	}
+
+	Handle = Vfopen(Handle, Name, "r+b", Cached);
+	IdxHandle = Vfopen(0, IdxName, "r+b", TRUE);
+	BOOL Valid = (Handle && IdxHandle);
+
+	// unlock
+	if (Valid || !AutoCreate)
+	{
+		//if (Handle)
+			//Vfunlock(Handle);
+	}
+	else
+	{
+		if (Handle)
+		{
+			Vfclose(Handle);
+			if (IdxHandle)
+				Vfdestroy(IdxHandle);
+			IdxHandle = 0;
+		}
+		else
+		{
+			if (IdxHandle)
+				Vfdestroy(IdxHandle);
+			IdxHandle = 0;
+			Handle = Vfnew(Name, "w+b", Cached);
+			if (!Vflock(Handle))
+			{
+				Vfdestroy(Handle);
+				return FALSE;
+			}
+		}
+
+		Handle = Vfopen(Handle, Name, "w+b", Cached);
+		IdxHandle = Vfopen(0, IdxName, "w+b", TRUE);
+		Valid = (Handle && IdxHandle);
+
+		if (Valid)
+		{
+			Vfwrite(__TABLE_SIGNATURE__, strlen(__TABLE_SIGNATURE__), Handle);
+			Vfwrite(__INDEX_SIGNATURE__, strlen(__TABLE_SIGNATURE__), IdxHandle);
+			// TODO bensk> change if NUM_SPECIAL_RECORDS ever increases
+			int v=NUM_SPECIAL_RECORDS;//strlen(__TABLE_SIGNATURE__);
+			Vfwrite(&v, sizeof(v), IdxHandle);
+			//    v = 0; fwrite(&v, sizeof(v), 1, IdxHandle);
+			v = -1; Vfwrite(&v, sizeof(v), IdxHandle); // write ID
+			v = 0; 
+			for (int i=0;i<NUM_SPECIAL_RECORDS;i++)
+			{
+				Vfwrite(&v, sizeof(v), IdxHandle);
+				Vfwrite(&v, sizeof(v), IdxHandle);
+			}
+			Sync();
+			justcreated = 1;
+		}
+	}
+
+	if (!Valid)
+	{
+		if (Handle) Vfdestroy(Handle);
+		if (IdxHandle) Vfdestroy(IdxHandle);
+		Handle = NULL;
+		IdxHandle = NULL;
+	}
+	else
+	{
+		char test1[9]={0};
+		char test2[9]={0};
+
+		Vfseek(Handle, 0, SEEK_SET);
+		Vfread(test1, strlen(__TABLE_SIGNATURE__), Handle);
+		Vfseek(IdxHandle, 0, SEEK_SET);
+		Vfread(test2, strlen(__INDEX_SIGNATURE__), IdxHandle);
+		test1[8]=0;
+		test2[8]=0;
+		if (strcmp(test1, __TABLE_SIGNATURE__) || strcmp(test2, __INDEX_SIGNATURE__))
+		{
+			if (Handle) Vfdestroy(Handle);
+			Handle = 0;
+			if (IdxHandle) Vfdestroy(IdxHandle);
+			IdxHandle = 0;
+			return FALSE;
+		}
+
+		// Load default index
+		IndexField *l_index_field = new IndexField(PRIMARY_INDEX, -1, -1, L"None");
+		l_index_field->index = new Index(IdxHandle, PRIMARY_INDEX, -1, -1, FALSE, 0, this);
+
+		// Get indexes
+		int Ptr = l_index_field->index->Get(INDEX_RECORD_NUM);
+		IndexList = new IndexRecord(Ptr, INDEX_RECORD_NUM, Handle, this);
+		if (!IndexList)
+		{
+			delete l_index_field;
+			if (Handle) Vfdestroy(Handle);
+			Handle = 0;
+			if (IdxHandle) Vfdestroy(IdxHandle);
+			IdxHandle = 0;
+			return FALSE;
+		}
+
+		// Init them
+		IndexNewWalkerContext newContext(this);
+		IndexList->WalkFields(IndexNewWalker, &newContext);
+
+		// Add default in case its not there (if it is it won'l_data_type be added by addfield)
+		IndexList->AddField(l_index_field);
+
+		// Get the default index (whether loaded or preloaded)
+		Scanner::index = ((IndexField*)IndexList->GetField(PRIMARY_INDEX))->index;
+
+		// If it's different from preloaded, delete preloaded
+		if (l_index_field->index != Scanner::index)
+		{
+			delete l_index_field;
+			l_index_field = NULL;
+		}
+
+		// Set up colaboration
+		IndexList->BuildCollaboration();
+
+		// Get columns
+		Ptr = Scanner::index->Get(FIELDS_RECORD_NUM);
+		FieldsRecord = new Record(Ptr, FIELDS_RECORD_NUM, Handle, this);
+		if (!FieldsRecord)
+		{
+			IndexList->Release();
+			IndexList=0;
+			if (Handle) Vfdestroy(Handle);
+			Handle = 0;
+			if (IdxHandle) Vfdestroy(IdxHandle);
+			IdxHandle = 0;
+			return FALSE;
+		}
+
+		// update the column cache
+		FieldsRecord->WalkFields(BuildColumnCache, this);
+		columns_cached=true;
+	}
+
+	if (Valid && !justcreated)
+	{
+		if (IndexList->NeedFix())
+			Compact();
+	}
+
+	if (Valid) First();
+	if (Handle)
+		Vfunlock(Handle);
+	return Valid;
+}
+
+//---------------------------------------------------------------------------
+void Table::Close(void)
+{
+	int v=0;
+
+	if (Handle && IndexList && Vflock(Handle, 0))
+	{
+		IndexList->WalkFields(IndexWriteWalker, 0);
+	}
+
+	delete Scanners;
+	Scanners = NULL;
+
+	Vsync(Handle);
+	if (IdxHandle)
+	{
+		Vfdestroy(IdxHandle);
+		IdxHandle = NULL;
+		v |= 2;
+	}
+	if (Handle)
+	{
+		Vfdestroy(Handle);
+		Handle = NULL;
+		v |= 1;
+	}
+
+	if (v != 3)
+		return;
+}
+
+bool Table::IndexWriteWalker(IndexRecord *record, Field *entry, void *context)
+{
+	IndexField *field = (IndexField *)entry;
+	field->index->WriteIndex();
+	return true;
+}
+
+//---------------------------------------------------------------------------
+void Table::Sync(void)
+{
+	if (!Vflock(Handle))
+		return;
+
+	if (IndexList)
+		IndexList->WalkFields(IndexWriteWalker, 0);
+
+	int err=0;
+	if (!err && Handle) err|=Vsync(Handle);
+	if (!err && IdxHandle) Vsync(IdxHandle);
+
+	Vfunlock(Handle);
+}
+
+//---------------------------------------------------------------------------
+ColumnField *Table::NewColumn( unsigned char FieldID, const wchar_t *FieldName, unsigned char FieldType, BOOL indexUnique )
+{
+	columns_cached = false; // if they start writing new columns, kill the columns cache until they PostColumns()
+	ColumnField *l_column_field = GetColumnById( FieldID );
+	if ( l_column_field )
+	{
+		int l_data_type = l_column_field->GetDataType();
+		if ( l_data_type != FieldType )
+		{
+			OutputDebugStringW( L"column " );
+			OutputDebugStringW( FieldName );
+			OutputDebugStringW( L" already exists but is of the wrong type\n" );
+
+			if ( CompatibleFields( l_data_type, FieldType ) )
+			{
+				OutputDebugStringW( L"going from one equivalent type to another, converting column\n" );
+				l_column_field->SetDataType( FieldType );
+				goto aok;
+			}
+		}
+
+		l_column_field->SetFieldName( (wchar_t *)FieldName );
+		return NULL;
+	}
+aok:
+	if ( GetColumnByName( FieldName ) )
+		return NULL;
+
+	ColumnField *l_new_column_field = new ColumnField( FieldID, FieldName, FieldType, this );
+	column_ids[ FieldID ] = FieldType;
+	FieldsRecord->AddField( l_new_column_field );
+
+	return l_new_column_field;
+}
+
+void Table::SetFieldSearchableById(unsigned char field_id, bool searchable)
+{
+	ColumnField *column = GetColumnById(field_id);
+	if (column)
+		column->SetSearchable(searchable);
+	if (searchable)
+	{
+		search_fields.insert(field_id);
+	}
+	else
+	{
+		search_fields.erase(field_id);
+	}
+}
+
+//---------------------------------------------------------------------------
+bool Table::BuildColumnCache(Record *record, Field *entry, void *context)
+{
+	Table *table = (Table *)context;
+	ColumnField *column = (ColumnField *)entry;
+	unsigned char field_id=column->GetFieldId();
+	table->column_ids[field_id] = column->GetDataType();
+
+	if (column->IsSearchableField())
+	{
+		table->search_fields.insert(field_id);
+	}
+	else
+	{
+		table->search_fields.erase(field_id);
+	}
+	return true;
+}
+
+//---------------------------------------------------------------------------
+void Table::PostColumns(void)
+{
+	FieldsRecord->WriteFields(this, FIELDS_RECORD_NUM);
+	memset(column_ids, FIELD_UNKNOWN, 255);
+	FieldsRecord->WalkFields(BuildColumnCache, this);
+	columns_cached=true;
+}
+
+unsigned char Table::GetColumnType(unsigned char Id)
+{
+	if (columns_cached)
+	{
+		return column_ids[Id];
+	}
+	else
+	{
+		return GetColumnById(Id)->GetDataType();
+	}
+}
+
+//---------------------------------------------------------------------------
+IndexField *Table::GetIndexByName(const wchar_t *name)
+{
+	if (!IndexList)
+		return NULL;
+	return IndexList->GetIndexByName(name);
+}
+
+//---------------------------------------------------------------------------
+IndexField *Table::GetIndexById(unsigned char Id)
+{
+	if (!IndexList)
+		return NULL;
+	return (IndexField *)IndexList->GetField(Id);
+}
+
+//---------------------------------------------------------------------------
+void Table::AddIndexByName(const wchar_t *name, const wchar_t *desc)
+{
+	ColumnField *header = GetColumnByName(name);
+	if (header)
+	{
+		unsigned char Idx = header->ID;
+		AddIndexById(Idx, desc);
+	}
+}
+
+//---------------------------------------------------------------------------
+void Table::AddIndexById( unsigned char Id, const wchar_t *desc )
+{
+	if ( GetIndexById( Id ) )
+		return;
+
+	ColumnField *l_column_field = GetColumnById( Id );
+	if ( !l_column_field )
+		return;
+
+	IndexField *l_new_index_field = new IndexField( Id, IndexList->GetColumnCount(), l_column_field->GetDataType(), desc );
+	l_new_index_field->index      = new Index( IdxHandle, Id, IndexList->GetColumnCount(), l_column_field->GetDataType(), TRUE, Scanner::index->NEntries, this );
+	
+	IndexField* l_previous_index_field = dynamic_cast<IndexField*>(IndexList->GetLastField());
+
+	IndexList->AddField( l_new_index_field );
+
+	//IndexField* l_previous_index_field = (IndexField*)l_new_index_field->prev;
+	if (l_previous_index_field)
+	{
+		l_previous_index_field->index->Colaborate(l_new_index_field);
+	}
+
+	IndexField *l_primary_index_field = (IndexField *)IndexList->GetField( PRIMARY_INDEX );
+	if (l_primary_index_field)
+	{
+		l_new_index_field->index->Colaborate(l_primary_index_field);
+	}
+
+	if (l_previous_index_field)
+	{
+		l_previous_index_field->index->Propagate();
+	}
+
+	IndexList->WriteFields( this );
+}
+
+//---------------------------------------------------------------------------
+BOOL Table::CheckIndexing(void)
+{
+	if (IndexList->GetColumnCount() == 0) return TRUE;
+
+	for (int i=0;i<Scanner::index->NEntries;i++)
+	{		
+		if (!IndexList->CheckIndexing(i))
+			return FALSE;
+	}
+	return TRUE;
+}
+
+struct IndexWalkerThunkContext
+{
+	void *context;
+	Table *_this;
+	Table::IndexWalker callback;
+};
+
+bool Table::IndexWalkerThunk(IndexRecord *record, Field *entry, void *context_in)
+{
+	IndexWalkerThunkContext *context = (IndexWalkerThunkContext *)context_in;
+	return context->callback(context->_this, (IndexField *)entry, context->context);
+}
+
+//---------------------------------------------------------------------------
+void Table::WalkIndices(IndexWalker callback, void *context)
+{
+	if (IndexList && callback)
+	{
+		IndexWalkerThunkContext walkerContext = { context, this, callback };
+		IndexList->WalkFields(IndexWalkerThunk, &walkerContext);
+	}
+}
+
+//---------------------------------------------------------------------------
+void Table::DropIndex(IndexField *Ptr)
+{
+	if (!Ptr || Ptr->Type != FIELD_INDEX) return;
+	if (Scanner::index == Ptr->index)
+	{
+		Scanner::index = ((IndexField*)IndexList->GetField(PRIMARY_INDEX))->index;
+		IndexList->BuildCollaboration();
+	}
+	IndexList->RemoveField(Ptr);
+	if (Scanner::index->SecIndex == Ptr)
+		Scanner::index->SecIndex = 0;
+	IndexList->WriteFields(this);
+}
+
+//---------------------------------------------------------------------------
+void Table::DropIndexByName(const wchar_t *desc)
+{
+	IndexField *indx = GetIndexByName(desc);
+	if (!_wcsicmp(desc, L"None")) return;
+
+	if (indx)
+		DropIndex(indx);
+}
+
+//---------------------------------------------------------------------------
+void Table::DropIndexById(unsigned char Id)
+{
+	if (!IndexList)
+		return;
+	if (Id == (unsigned char)PRIMARY_INDEX) return;
+	IndexField *indx=(IndexField *)IndexList->GetField(Id);
+	if (indx)
+		DropIndex(indx);
+}
+
+//---------------------------------------------------------------------------
+BOOL Table::LocateByIdEx(int Id, int From, Field *field, int comp_mode)
+{
+	return Scanner::LocateByIdEx(Id, From, field, NULL, comp_mode);
+}
+
+//---------------------------------------------------------------------------
+Record *Table::GetColumns(void)
+{
+	if (!FieldsRecord)
+		return NULL;
+	return FieldsRecord;
+}
+
+//---------------------------------------------------------------------------
+Scanner *Table::NewScanner()
+{
+	Scanner *s = new Scanner(this);
+	/*if (Scanners->GetNElements() > 0)*/
+	s->index = Scanner::index;
+	Scanners->AddEntry(s, TRUE);
+	return s;
+}
+
+//---------------------------------------------------------------------------
+Scanner *Table::GetDefaultScanner()
+{
+	return this;
+}
+
+//---------------------------------------------------------------------------
+void Table::DeleteScanner(Scanner *scan)
+{
+	if (!scan) return;
+		Scanners->RemoveEntry(scan);
+}
+
+//---------------------------------------------------------------------------
+void Table::IndexModified(void)
+{
+	Scanner *s = (Scanner *)Scanners->GetHead();
+	while (s)
+	{
+		s->IndexModified();
+		s = (Scanner *)s->GetNext();
+	}
+}
+
+//---------------------------------------------------------------------------
+void Table::SetGlobalLocateUpToDate(BOOL is) {
+	GLocateUpToDate = is;
+}
+
+struct ColumnWalkContext
+{
+	Table *ctable;
+};
+
+bool Table::Compact_ColumnWalk(Record *record, Field *entry, void *context_in)
+{
+	ColumnField *field = static_cast<ColumnField *>(entry);
+	ColumnWalkContext *context = (ColumnWalkContext *)context_in;
+	Table *ctable = context->ctable;
+
+	ctable->NewColumn(field->GetFieldId(), field->GetFieldName(), field->GetDataType(), FALSE);
+	return true;
+}
+
+struct ColumnWalk2Context
+{
+	Table *ctable;
+	Table *thisTable;
+	uint8_t *data;
+	size_t data_size;
+	int gotstuff;
+};
+
+bool Table::Compact_ColumnWalk2(Record *record, Field *entry, void *context_in)
+{
+	ColumnField *colfield = (ColumnField *)entry;
+	ColumnWalk2Context *context = (ColumnWalk2Context *)context_in;
+
+	unsigned char fieldid = colfield->GetFieldId();
+	//char *fieldname = colfield->GetFieldName();
+	Field *mfield = context->thisTable->GetFieldById(fieldid);
+	//Field *mfield = GetFieldByName(fieldname);
+	if (mfield != NULL) {
+		if (!context->gotstuff) {
+			context->ctable->New();
+			context->gotstuff = 1;
+		}
+		Field *cfield = context->ctable->NewFieldById(fieldid, 0);
+		//Field *cfield = ctable->NewFieldByName(fieldname, mfield->GetPerm());
+		size_t len = mfield->GetDataSize();
+		if (len > context->data_size)
+		{
+			size_t old_data_size = context->data_size;
+			context->data_size = len;
+			uint8_t *new_data = (uint8_t *)realloc(context->data, context->data_size);
+			if (new_data)
+			{
+				context->data = new_data;
+			}
+			else
+			{
+				new_data = (uint8_t *)malloc(context->data_size);
+				if (new_data)
+				{
+					memcpy(new_data, context->data, old_data_size);
+					free(context->data);
+					context->data = new_data;
+				}
+				else
+				{
+					context->data_size = old_data_size;
+					return false;
+				}
+			}
+		}
+		mfield->WriteTypedData(context->data, len);
+		cfield->ReadTypedData(context->data, len);
+	}
+
+	return true;
+}
+
+bool Table::Compact_IndexWalk(Table *table, IndexField *field, void *context)
+{
+	Table *ctable = (Table *)context;
+
+	if (_wcsicmp(field->GetIndexName(), L"None"))
+		ctable->AddIndexById(field->GetFieldId(), field->GetIndexName());
+	return true;
+}
+//---------------------------------------------------------------------------
+void Table::Compact(int *progress) {
+	// ok so we're gonna be cheating a bit, instead of figuring out how to safely modify all those
+	// nifty indexes that cross reference themselves and blablabla, we're just gonna duplicate the
+	// whole table from scratch, overwrite ourselves, and reopen the table. duh.
+
+	if (!Vflock(Handle))
+	{
+		if (progress != NULL) *progress = 100;
+		return;
+	}
+	// create a temporary table in windows temp dir
+	wchar_t temp_table[MAX_PATH+12] = {0};
+	wchar_t temp_index[MAX_PATH+12] = {0};
+	wchar_t old_table[MAX_PATH+12] = {0};
+	wchar_t old_index[MAX_PATH+12] = {0};
+	DWORD pid=GetCurrentProcessId();
+
+	StringCbPrintfW(temp_table, sizeof(temp_table), L"%s.new%08X", Name,pid);
+	StringCbPrintfW(temp_index, sizeof(temp_index),L"%s.new%08X", IdxName,pid);
+	StringCbPrintfW(old_table, sizeof(old_table),L"%s.old%08X", Name,pid);
+	StringCbPrintfW(old_index, sizeof(old_index),L"%s.old%08X", IdxName,pid);
+
+	// delete them, in case we crashed while packing
+
+	DeleteFileW(temp_table);
+	DeleteFileW(temp_index);
+	DeleteFileW(old_table);
+	DeleteFileW(old_index);
+
+	// create a brand new db and a brand new table
+	Table *ctable = db->OpenTable(temp_table, temp_index, NDE_OPEN_ALWAYS, Cached);
+
+	// duplicate the columns
+	Record *record = GetColumns();
+	if (record != NULL) 
+	{
+		ColumnWalkContext context;
+		context.ctable = ctable;
+		record->WalkFields(Compact_ColumnWalk, &context);
+	}
+	ctable->PostColumns();
+
+	// duplicate the indexes
+	WalkIndices(Compact_IndexWalk, (void *)ctable);
+
+	// duplicate the data
+	int reccount = GetRecordsCount();
+
+	int count = 0;
+	First();
+	ColumnWalk2Context context;
+	context.data_size = 65536;
+	context.data = (uint8_t *)calloc(65536, sizeof(uint8_t));
+	context.ctable = ctable;
+	context.thisTable = this;
+
+	while (1) {
+		int lasterr = NumErrors();
+		GetDefaultScanner()->GetRecordById(count, FALSE);
+		count++;
+
+		if (Eof() || count > reccount) break;
+
+		if (NumErrors() > lasterr) 
+			continue;
+
+		Index *idx = GetDefaultScanner()->GetIndex();
+		int pos = idx->Get(GetDefaultScanner()->GetRecordId());
+
+		if (pos == 0) continue;
+
+		int pr = (int)((float)GetRecordId()/(float)reccount*100.0f);
+		if (progress != NULL) *progress = pr;
+		context.gotstuff = 0;
+
+		if (record != NULL)
+			record->WalkFields(Compact_ColumnWalk2, &context);
+
+		if (context.gotstuff) 
+			ctable->Post(); 
+	}
+	free(context.data);
+
+	// done creating temp table
+	db->CloseTable(ctable);
+
+	// reset the data structures and close underlying file handles
+	Reset();
+
+	if (MoveFileW(Name,old_table))
+	{
+		if (MoveFileW(IdxName,old_index))
+		{
+			if (!MoveFileW(temp_table,Name) || !MoveFileW(temp_index,IdxName))
+			{
+				// failed, try to copy back
+				DeleteFileW(Name);
+				DeleteFileW(IdxName);
+				MoveFileW(old_table,Name); // restore old file
+				MoveFileW(old_index,IdxName); // restore old file
+			}
+		}
+		else
+		{
+			MoveFileW(old_table,Name); // restore old file
+		}
+	}
+
+	// clean up our temp files
+	DeleteFileW(temp_table);
+	DeleteFileW(temp_index);
+	DeleteFileW(old_table);
+	DeleteFileW(old_index);
+
+	if (progress != NULL) *progress = 100;
+
+	// reopen our table
+	Init();
+	Open();
+}
+
+ColumnField *Table::GetColumnById(unsigned char Idx)
+{
+	if (!FieldsRecord)
+		return NULL;
+	return (ColumnField *)FieldsRecord->GetField(Idx);
+}
+
+ColumnField *Table::GetColumnByName(const wchar_t *FieldName)
+{
+	return FieldsRecord->GetColumnByName(FieldName);
+}
+
+void Table::RowCache_Delete(int position)
+{
+	if (use_row_cache)
+	{
+		RowCache::iterator found = rowCache.find(position);
+		if (found != rowCache.end())
+		{
+			if (found->second)
+				found->second->Release();
+			rowCache.erase(found);
+		}
+	}
+}
+
+void Table::RowCache_Remove(int position)
+{
+	if (use_row_cache)
+	{
+		Record *&row = rowCache[position];
+		if (row)
+		{
+			row->Release();
+		}
+
+		rowCache[position] = 0;
+	}
+}
+
+void Table::RowCache_Add(Record *record, int position)
+{
+	if (use_row_cache)
+	{
+		record->Retain();
+
+		Record *&row = rowCache[position];
+		if (row)
+		{
+			row->Release();
+		}
+
+		rowCache[position] = record;
+	}
+}
+
+Record *Table::RowCache_Get(int position)
+{
+	if (!use_row_cache || 0 == rowCache.count(position))
+		return 0;
+
+	Record *row = rowCache[position];
+	if (row)
+		row->Retain();
+	return row;
+}
+
+void Table::EnableRowCache()
+{
+	use_row_cache=true;
+}

Src/nde/win/Table.h

@@ -0,0 +1,170 @@
+/* ---------------------------------------------------------------------------
+                           Nullsoft Database Engine
+                             --------------------
+                        codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+ Table Class Prototypes
+ Windows implementation
+--------------------------------------------------------------------------- */
+
+#ifndef __TABLE_H
+#define __TABLE_H
+
+#include <stdio.h>
+//#include <io.h>
+#include "../Scanner.h"
+#include <map>
+#include "../IndexRecord.h"
+#include <assert.h>
+
+class Table : private Scanner
+{
+public:
+	// TODO: move these back to protected
+	VFILE *Handle = NULL;
+	using Scanner::index;
+	bool use_row_cache   = false;
+	BOOL GLocateUpToDate = FALSE;
+
+private:
+	void Init();
+	void Reset();
+
+private:
+	LinkedList *Scanners;
+
+protected:
+	wchar_t *Name;
+	wchar_t *IdxName;
+
+	VFILE        *IdxHandle = NULL;
+	BOOL          AutoCreate;
+	Record       *FieldsRecord = NULL;
+	IndexRecord  *IndexList = NULL;
+	Database     *db;
+	BOOL          Cached;
+	int           numErrors = 0;
+	using Scanner::Edition;
+	bool          columns_cached = false;
+	unsigned char column_ids[256];
+	typedef std::map<int, Record*> RowCache;
+	RowCache rowCache;
+
+	// Tables
+	static bool Compact_ColumnWalk(Record *record, Field *entry, void *context_in);
+	static bool Compact_ColumnWalk2(Record *record, Field *entry, void *context_in);
+	static bool Compact_IndexWalk(Table *table, IndexField *entry, void *context);
+	static bool IndexWriteWalker(IndexRecord *record, Field *entry, void *context);
+	static bool IndexWalkerThunk(IndexRecord *record, Field *entry, void *context);
+	static bool IndexNewWalker(IndexRecord *record, Field *entry, void *context);
+	static bool BuildColumnCache(Record *record, Field *entry, void *context);
+
+public:
+	typedef bool (*IndexWalker)(Table *table, IndexField *entry, void *context);
+	Table(const wchar_t *TableName, const wchar_t *IdxName, BOOL Create, Database *db, BOOL Cached);
+	~Table();
+	BOOL Open(void);
+	void Close(void);
+
+	// Columns
+	ColumnField *NewColumn(unsigned char Id, const wchar_t *name, unsigned char type, BOOL indexUniques);
+
+	void DeleteColumn(ColumnField *field); // todo
+	void DeleteColumnByName(const wchar_t *name); // todo
+	void DeleteColumnById(unsigned char Id); // todo
+	void PostColumns(void);
+	NDE_API Record *GetColumns(void);
+	ColumnField *GetColumnByName(const wchar_t *FieldName);
+	ColumnField *GetColumnById(unsigned char Idx);
+	unsigned char GetColumnType(unsigned char Id);
+
+	// Fields
+	using Scanner::NewFieldByName;
+	using Scanner::NewFieldById;
+	using Scanner::GetFieldByName;
+	using Scanner::GetFieldById;
+	using Scanner::DeleteField;
+	using Scanner::DeleteFieldByName;
+	using Scanner::DeleteFieldById;
+
+	// Records
+	using Scanner::First;
+	using Scanner::Last;
+	using Scanner::Next;
+	using Scanner::Previous;
+	using Scanner::Eof;
+	using Scanner::Bof;
+	using Scanner::New;
+	using Scanner::Insert;
+	using Scanner::Edit;
+	using Scanner::Cancel;
+	using Scanner::Post;
+	using Scanner::Delete;
+	using Scanner::GetRecordsCount;
+	using Scanner::GetRecordById;
+	using Scanner::GetRecordId;
+	void Sync(void);
+	using Scanner::LocateByName;
+	using Scanner::LocateById;
+	BOOL LocateByIdEx(int Id, int From, Field *field, int comp_mode);
+
+	// Indexes
+	void AddIndexByName(const wchar_t *FieldName, const wchar_t *KeyName);
+	void AddIndexById(unsigned char Id, const wchar_t *KeyName);
+
+	void WalkIndices(IndexWalker callback, void *context);
+
+	IndexField *GetIndexByName(const wchar_t *name);
+	IndexField *GetIndexById(unsigned char Id);
+	using Scanner::SetWorkingIndexByName;
+	using Scanner::SetWorkingIndexById;
+	NDE_API BOOL CheckIndexing(void);
+	void DropIndexByName(const wchar_t *desc);
+	void DropIndexById(unsigned char Id);
+	void DropIndex(IndexField *Ptr);
+	void IndexModified(void);
+
+	// Filters
+	using Scanner::AddFilterByName;
+	using Scanner::AddFilterById;
+	using Scanner::AddFilterOp;
+	using Scanner::RemoveFilters;
+
+	// Scanners
+	Scanner *NewScanner();
+	Scanner *GetDefaultScanner();
+	void DeleteScanner(Scanner *scan);
+
+	// Misc
+	using Scanner::FragmentationLevel;
+	void Compact(int *progress = NULL);
+	void SetGlobalLocateUpToDate(BOOL is);
+
+	// Row Cache
+	void RowCache_Delete(int position);
+	void RowCache_Remove(int position);
+	void RowCache_Add(Record *record, int position);
+	Record *RowCache_Get(int position);
+	NDE_API void EnableRowCache();
+
+	// Searching
+	void SetFieldSearchableById(unsigned char field_id, bool searchable);
+
+	int HasErrors()
+	{
+		return numErrors > 0;
+	}
+	int NumErrors()
+	{
+		return numErrors;
+	}
+	void IncErrorCount()
+	{
+		numErrors++;
+	}
+};
+
+#endif

Src/nde/win/Vfs.cpp

@@ -0,0 +1,599 @@
+/* ---------------------------------------------------------------------------
+Nullsoft Database Engine
+--------------------
+codename: Near Death Experience
+--------------------------------------------------------------------------- */
+
+/* ---------------------------------------------------------------------------
+
+Virtual File System
+
+--------------------------------------------------------------------------- */
+#include "../nde.h"
+
+#include "vfs.h"
+#include <malloc.h>
+#ifndef EOF
+#define EOF -1
+#endif
+#include <Sddl.h>
+#include <strsafe.h>
+
+#if defined(NDE_ALLOW_NONCACHED)
+size_t ReadFileN(void *buffer, size_t size, VFILE *f)
+{
+	uint8_t *b = (uint8_t *) buffer;
+	size_t total_size=0;
+	while (size)
+	{
+		DWORD bytesRead = 0;
+		DWORD toRead = min(0xffffffffUL, size);
+		ReadFile(f->hfile, b, toRead, &bytesRead, NULL);
+		if (bytesRead != toRead)
+		{
+			f->endoffile=true;
+			// TODO: rewind
+			return total_size+bytesRead;
+		}
+		size-=toRead;
+		b+=toRead;
+		total_size+=toRead;
+	}
+	return total_size;
+}
+
+size_t WriteFileN(void *buffer, size_t size,  VFILE *f)
+{
+	uint8_t *b = (uint8_t *) buffer;
+	size_t total_size=0;
+		while (size)
+		{
+			DWORD bytesRead = 0;
+			DWORD toRead = min(0xffffffffUL, size);
+			WriteFile(f->hfile, b, toRead, &bytesRead, NULL);
+			if (bytesRead != toRead)
+			{
+				f->endoffile=true;
+				// TODO: rewind
+				return total_size+bytesRead;
+			}
+			size-=toRead;
+			b+=toRead;
+			total_size+=toRead;
+		}
+	
+	return total_size;
+}
+#endif
+
+// benski> i havn't the slightest fucking clue why this works, it's copypasta code from the internets
+static LPCWSTR LOW_INTEGRITY_SDDL_SACL_W = L"S:(ML;;NW;;;LW)";
+static bool GetLowIntegrity(SECURITY_ATTRIBUTES *attributes)
+{
+	PSECURITY_DESCRIPTOR pSD = NULL;
+
+	if ( ConvertStringSecurityDescriptorToSecurityDescriptorW (
+		LOW_INTEGRITY_SDDL_SACL_W, SDDL_REVISION_1, &pSD, NULL ) )
+	{
+		attributes->nLength = sizeof(SECURITY_ATTRIBUTES);
+		attributes->lpSecurityDescriptor = pSD;
+		attributes->bInheritHandle = FALSE;
+
+		//LocalFree ( pSD );
+		return true;
+	}
+
+	return false;
+}
+
+VFILE *Vfnew(const wchar_t *fl, const char *mode, BOOL Cached)
+{
+	if (!fl) return NULL;
+	VFILE *f = (VFILE *)calloc(1, sizeof(VFILE));
+	if (!f) return NULL;
+
+#ifdef NDE_ALLOW_NONCACHED
+	if (!Cached)
+	{
+		f->r.reserve(256); // heuristically determined
+	}
+	f->hfile = INVALID_HANDLE_VALUE;
+#endif
+
+#ifndef NO_TABLE_WIN32_LOCKING
+	// TODO: should we retrieve a better filename, e.g. GetLongPathName, or GetFinalPathNameByHandle  on vista+
+	wchar_t mutex_name[1024] = {0};
+	StringCbPrintfW(mutex_name, sizeof(mutex_name), L"Global\\nde-%s", fl);
+
+	CharLowerW(mutex_name+7);
+	wchar_t *sw = mutex_name+7;
+	wchar_t *has_extension=0;
+	while (sw && *sw)
+	{
+		if (*sw == L'\\')
+		{
+			has_extension=0;
+			*sw = L'/';
+		}
+		else if (*sw == L'.')
+			has_extension=sw;
+		sw++;
+	}
+	if (has_extension)
+		*has_extension = 0;
+
+	SECURITY_ATTRIBUTES attr = {0};
+	if (GetLowIntegrity(&attr))
+	{
+		f->mutex = CreateMutexW(&attr, FALSE, mutex_name);
+		LocalFree(attr.lpSecurityDescriptor);
+	}
+	else
+		f->mutex = CreateMutexW(0, FALSE, mutex_name);
+
+#endif
+	return f;
+}
+
+//----------------------------------------------------------------------------
+VFILE *Vfopen(VFILE *f, wchar_t *fl, const char *mode, BOOL Cached)
+{
+	if (!fl) return NULL;
+	if (!f)
+	{
+		f = Vfnew(fl, mode, Cached);
+		if (!f)
+			return NULL;
+	}
+
+#ifdef NDE_ALLOW_NONCACHED
+	f->cached = Cached;
+#else
+	f->cached = TRUE;
+#endif
+
+	if (!strchr(mode, '+'))
+	{
+		if (strchr(mode, 'r'))
+			f->mode = VFS_READ | VFS_MUSTEXIST;
+		if (strchr(mode, 'w'))
+			f->mode = VFS_WRITE | VFS_CREATE | VFS_NEWCONTENT;
+		if (strchr(mode, 'a'))
+			f->mode = VFS_WRITE | VFS_CREATE | VFS_SEEKEOF;
+	}
+	else
+	{
+		if (strstr(mode, "r+"))
+			f->mode = VFS_WRITE | VFS_MUSTEXIST;
+		if (strstr(mode, "w+"))
+			f->mode = VFS_WRITE | VFS_CREATE | VFS_NEWCONTENT;
+		if (strstr(mode, "a+"))
+			f->mode = VFS_WRITE | VFS_CREATE | VFS_SEEKEOF;
+	}
+
+	if (f->mode == 0 || ((f->mode & VFS_READ) && (f->mode & VFS_WRITE)))
+	{
+		Vfdestroy(f);
+		return NULL;
+	}
+
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		f->endoffile=false;
+		int readFlags=GENERIC_READ, openFlags=0;
+		if (f->mode & VFS_WRITE)			readFlags|=GENERIC_WRITE;
+		if (f->mode & VFS_MUSTEXIST)			openFlags=OPEN_EXISTING;
+		if (f->mode & VFS_CREATE) openFlags = OPEN_ALWAYS;
+		if (f->mode & VFS_NEWCONTENT) openFlags = CREATE_ALWAYS;
+		f->hfile=CreateFile(fl,readFlags,FILE_SHARE_READ,0,openFlags,0,0);
+		if (f->hfile!=INVALID_HANDLE_VALUE)
+			f->filename = _strdup(fl);
+		else
+		{
+			Vfdestroy(f);
+			return NULL;
+		}
+		return f;
+	}
+#endif
+
+	if (f->mode & VFS_MUSTEXIST)
+	{
+		if (GetFileAttributesW(fl) == INVALID_FILE_ATTRIBUTES)
+		{
+			Vfdestroy(f);
+			return NULL;
+		}
+	}
+
+	if (!(f->mode & VFS_NEWCONTENT))
+	{
+		int attempts=0;
+		HANDLE hFile=INVALID_HANDLE_VALUE;
+again:
+		if (attempts<100) // we'll try for 10 seconds
+		{
+			hFile=CreateFileW(fl,GENERIC_READ,FILE_SHARE_READ/*|FILE_SHARE_WRITE*/,0,OPEN_EXISTING,FILE_FLAG_SEQUENTIAL_SCAN,0);
+			if (hFile == INVALID_HANDLE_VALUE && GetLastError() == ERROR_SHARING_VIOLATION)
+			{
+				Sleep(100); // let's try again
+				goto again;
+			}
+		}
+		else if (hFile == INVALID_HANDLE_VALUE && GetLastError() == ERROR_SHARING_VIOLATION)
+		{
+			// screwed up STILL? eeergh I bet it's another program locking it, let's try with more sharing flags
+			hFile=CreateFileW(fl,GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE,0,OPEN_EXISTING,FILE_FLAG_SEQUENTIAL_SCAN,0);
+		}
+
+		if (hFile==INVALID_HANDLE_VALUE)
+		{
+			f->data = (uint8_t *)calloc(VFILE_INC, 1);
+			if (f->data == NULL)
+			{
+				Vfdestroy(f);
+				return NULL;
+			}
+			f->filesize = 0;
+			f->maxsize = VFILE_INC;
+		}
+		else
+		{
+			size_t fsize_ret_value=GetFileSize(hFile,NULL);
+			if (fsize_ret_value==INVALID_FILE_SIZE)
+			{
+				Vfdestroy(f);
+				return NULL;
+			}
+			f->filesize = (uint32_t)fsize_ret_value;
+			f->data = (uint8_t *)calloc(f->filesize, 1);
+			if (f->data == NULL)
+			{
+				CloseHandle(hFile);
+				Vfdestroy(f);
+				return NULL;
+			}
+			f->maxsize = f->filesize;
+			DWORD r = 0;
+			// TODO: benski> I think we should switch this to overlapped I/O (to allow I/O to happen as we're parsing)
+			//               or switch to a memory mapped file... but we'll need to check with the profiler
+			if (!ReadFile(hFile,f->data,f->filesize,&r,NULL) || r != f->filesize)
+			{
+				CloseHandle(hFile);
+				Vfdestroy(f);
+				return NULL;
+			}
+			CloseHandle(hFile);
+		}
+	}
+
+	if (f->mode & VFS_SEEKEOF)
+		f->ptr = f->filesize;
+
+	f->filename = fl;
+	ndestring_retain(f->filename);
+	return f;
+}
+
+//----------------------------------------------------------------------------
+void Vfclose(VFILE *f)
+{
+	if (!f) return;
+
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		if (f->hfile!=INVALID_HANDLE_VALUE)
+			CloseHandle(f->hfile);
+		f->hfile=INVALID_HANDLE_VALUE;
+	}
+	else
+#endif
+	{
+		if (f->mode & VFS_WRITE)
+		{
+			Vsync(f);
+		}
+	}
+
+	ndestring_release(f->filename);
+	f->filename=0;
+	free(f->data);
+	f->data = 0;
+	f->ptr=0;
+	f->filesize=0;
+	f->maxsize=0;
+	f->dirty=0;
+}
+
+void Vfdestroy(VFILE *f)
+{
+	// benski> TODO: 
+	if (f)
+	{
+		Vfclose(f);
+
+		while (f->locks)
+			Vfunlock(f, 1);
+
+		if (f->mutex) CloseHandle(f->mutex);
+		free(f);
+	}
+}
+
+//----------------------------------------------------------------------------
+size_t Vfread( void *ptr, size_t size, VFILE *f )
+{
+	assert( ptr && f );
+#ifdef NDE_ALLOW_NONCACHED
+	if ( !f->cached )
+	{
+		size_t read = f->r.read( ptr, size );
+		ptr = (uint8_t *)ptr + read;
+		size -= read;
+		if ( size == 0 ) return 1; // yay fully buffered read
+		// if we got here, the ring buffer is empty
+		f->r.clear(); // reset back to normal
+		if ( size > f->r.avail() )
+		{
+			return ReadFileN( ptr, size, f ) == size;
+		}
+		void *data = f->r.LockBuffer();
+		size_t bytes_read = ReadFileN( data, f->r.avail(), f );
+		f->r.UnlockBuffer( bytes_read );
+		read = f->r.read( ptr, size );
+		return read == size;
+	}
+#endif
+	//if (!size) return 0;
+	if ( size + f->ptr > f->filesize )
+	{
+		//FUCKO: remove this
+		if ( !( f->ptr < f->filesize ) )
+		{
+#ifdef _DEBUG
+			char buf[ 128 ] = { 0 };
+			StringCbPrintfA( buf, sizeof( buf ), "NDE/VFS: VFS read at %d/%d (%d bytes) is bad\n", f->ptr, f->filesize, size );
+			OutputDebugStringA( buf );
+#endif
+
+			//      if (!f->flushtable) // this would be ideal, if we could figure out f->flushtable
+			//      f->flushtable=MessageBox(g_hwnd,"DB read failed, DB may be corrupted.\r\n\r\n"
+			//    "Hit Retry to continue, or Cancel to clear the DB and start over","Winamp Library Error",MB_RETRYCANCEL) == IDCANCEL; //fucko
+			//MessageBox(g_hwnd,"DB read failed, DB may be corrupted. If this error persists, remove all files from the library.",
+			//        "Winamp Library Error",MB_OK);
+			return 0;
+		}
+
+		size = f->filesize - f->ptr;
+	}
+
+	memcpy( ptr, f->data + f->ptr, size );
+	f->ptr += (uint32_t)size;
+
+	return 1;
+}
+
+//----------------------------------------------------------------------------
+void Vfwrite(const void *ptr, size_t size, VFILE *f)
+{
+	if (!ptr || !f) return;
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		// TODO: with some cleverness we might be able to make this write to the read cache
+
+		// if we're cached, then our file position is off and we need to adjust
+		if (!f->r.empty())
+			Vfseek(f, -(f->r.size()), SEEK_CUR);
+
+		f->r.clear(); 
+		WriteFileN(ptr, size, f);
+		return;
+	}
+#endif
+	f->dirty=1;
+	size_t s = (size);
+	if (s + f->ptr > f->maxsize)
+	{
+		// grow f->data,f->maxsize to be (s + f->ptr + VFILE_INC-1)&~(VFILE_INC-1)
+		// instead of calling Vgrow again which gets kinda slow
+		size_t newsize=(s + f->ptr + VFILE_INC-1)&~(VFILE_INC-1);
+		uint8_t *newdata=(uint8_t *)realloc(f->data,newsize);
+		if (newdata == NULL) return;
+		f->data = newdata;
+		memset(f->data+f->maxsize,0,newsize-f->maxsize);
+		f->maxsize=(uint32_t)newsize;
+	}
+	memcpy(f->data + f->ptr, ptr, s);
+	f->ptr += (uint32_t)s;
+	if (f->ptr > f->filesize)
+		f->filesize = f->ptr;
+}
+
+//----------------------------------------------------------------------------
+void Vgrow(VFILE *f)
+{
+	if (!f) return;
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached) return;
+#endif
+	uint8_t *newdata=(uint8_t *)realloc(f->data, f->maxsize + VFILE_INC);
+	if (newdata == NULL) return;
+	f->data = newdata;
+	f->maxsize += VFILE_INC;
+}
+
+//----------------------------------------------------------------------------
+uint32_t Vftell(VFILE *f)
+{
+	if (!f) return (unsigned)-1;
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		return SetFilePointer(f->hfile, 0, NULL, FILE_CURRENT);
+	}
+#endif
+	return f->ptr;
+}
+
+//----------------------------------------------------------------------------
+void Vfseek(VFILE *f, uint32_t i, int whence)
+{
+	if (!f) return;
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		if (whence == SEEK_CUR && i > 0 && i <f->r.size())
+		{
+			f->r.advance(i);
+		}
+		else
+		{
+			f->r.clear();
+			SetFilePointer(f->hfile, i, NULL, whence);
+			f->endoffile = false;
+		}
+		return;
+	}
+#endif
+	switch (whence)
+	{
+		case SEEK_SET:
+			f->ptr = i;
+			break;
+		case SEEK_CUR:
+			f->ptr += i;
+			break;
+		case SEEK_END:
+			f->ptr = f->filesize+i;
+			break;
+	}
+}
+
+//----------------------------------------------------------------------------
+int Vfeof(VFILE *f)
+{
+	if (!f) return -1;
+#ifdef NDE_ALLOW_NONCACHED
+	if (!f->cached)
+	{
+		return !!f->endoffile;
+	}
+#endif
+	return (f->ptr >= f->filesize);
+}
+
+//----------------------------------------------------------------------------
+int Vsync(VFILE *f)
+{
+	if (!f) return 0;
+	if (!f->dirty) return 0;
+
+	if (f->mode & VFS_WRITE)
+	{
+#ifdef NDE_ALLOW_NONCACHED
+		if (!f->cached)
+		{
+			LONG p = SetFilePointer(f->hfile, 0, NULL, FILE_CURRENT);
+			CloseHandle(f->hfile);
+			f->hfile = CreateFileW(f->filename,GENERIC_READ|GENERIC_WRITE,0,0,OPEN_EXISTING,0,NULL);
+			if (f->hfile == INVALID_HANDLE_VALUE)
+				return 1;
+			SetFilePointer(f->hfile, p, NULL, SEEK_SET);
+			f->endoffile=false;
+
+			return 0;
+		}
+#endif
+
+		wchar_t newfn[MAX_PATH] = {0};
+		wchar_t oldfn[MAX_PATH] = {0};
+		DWORD mypid=GetCurrentProcessId();
+
+		StringCchPrintfW(newfn, MAX_PATH, L"%s.n3w%08X",f->filename,mypid);
+		StringCchPrintfW(oldfn, MAX_PATH, L"%s.o1d%08X",f->filename,mypid);
+
+		DeleteFileW(newfn);
+		DeleteFileW(oldfn);
+
+		HANDLE hFile = CreateFileW(newfn,GENERIC_WRITE,0,0,CREATE_ALWAYS,0,NULL);
+		int success=0;
+		if (hFile != INVALID_HANDLE_VALUE)
+		{
+			DWORD o = 0;
+			if (WriteFile(hFile,f->data,f->filesize,&o,NULL) && o == f->filesize) success++;
+			CloseHandle(hFile);
+		}
+		if (!success)
+		{
+			DeleteFileW(newfn);
+			return 1;
+		}
+
+		// TODO use this to keep a backup of the database file for edit fails, etc
+		if (MoveFileW(f->filename,oldfn) == 0) // if the function fails
+		{
+			CopyFileW(f->filename,oldfn, FALSE);
+			DeleteFileW(f->filename);
+		}
+
+		int rv=0;
+		if (MoveFileW(newfn,f->filename) == 0 && CopyFileW(newfn,f->filename, FALSE) == 0)
+		{
+			MoveFileW(oldfn,f->filename); // restore old file
+			rv=1;
+		}
+		else
+		{
+			f->dirty=0;
+		}
+
+		// clean up our temp files
+		DeleteFileW(oldfn);
+		DeleteFileW(newfn);
+
+		return rv;
+	}
+	f->dirty=0;
+	return 0;
+}
+
+// returns 0 on failure
+int Vflock(VFILE *fl, BOOL is_sync)
+{	
+#ifndef NO_TABLE_WIN32_LOCKING
+	if (!fl) return 0;
+	if (!is_sync && fl->cached)
+		return 1;
+	// try for 10 seconds
+	if (fl->locks++ == 0)
+	{
+		if (WaitForSingleObject(fl->mutex, 10000) != WAIT_OBJECT_0)
+		{
+			fl->locks--;
+			return 0;
+		}
+	}
+#endif
+	return 1;
+}
+
+void Vfunlock(VFILE *fl, BOOL is_sync)
+{
+#ifndef NO_TABLE_WIN32_LOCKING
+	if (!is_sync && fl->cached)
+		return;
+
+	if (fl && fl->locks == 0)
+		DebugBreak();
+	if (--fl->locks == 0)
+	{
+		if (fl && fl->mutex)
+		{
+			ReleaseMutex(fl->mutex);
+		}
+	}
+#endif
+}

Src/nde/win/Vfs.h

@@ -0,0 +1,98 @@
+#ifndef __NDE_VFS_H
+#define __NDE_VFS_H
+
+#include <bfc/platform/types.h>
+#include <windows.h>
+
+//#define NDE_ALLOW_NONCACHED
+
+/*
+#ifdef NDE_ALLOW_NONCACHED
+  #ifndef NDE_NOWIN32FILEIO
+    #error NDE_ALLOW_NONCACHED at least for now requires NDE_NOWIN32FILEIO
+  #endif
+#endif
+*/
+
+#define VFILE_INC		65536
+
+#define VFS_READ		1
+#define VFS_WRITE		2
+#define VFS_SEEKEOF		4
+#define VFS_CREATE		8
+#define VFS_NEWCONTENT	16
+#define VFS_MUSTEXIST	32
+#if defined(NDE_ALLOW_NONCACHED)
+#include "../nu/RingBuffer.h"
+#endif
+typedef struct VFILEStruct
+{
+	void VFILE()
+	{
+		data = 0;
+		ptr = 0;
+		filesize = 0;
+		maxsize = 0;
+		filename = 0;
+		mode = 0;
+		cached = FALSE;
+		dirty = 0;
+
+	#ifdef NDE_ALLOW_NONCACHED
+	  #ifdef NDE_NOWIN32FILEIO
+		rfile = 0;
+	  #else
+		hfile = NULL;
+		endoffile = false;
+		r = 0;
+	  #endif
+	#endif
+
+		mutex = NULL;
+		locks = 0;
+	}
+
+    uint8_t *data;
+    uint32_t ptr;
+    uint32_t filesize;
+    uint32_t maxsize;
+    wchar_t *filename;
+    char mode;
+    BOOL cached;
+    int dirty;
+
+#ifdef NDE_ALLOW_NONCACHED
+  #ifdef NDE_NOWIN32FILEIO
+    FILE *rfile;
+  #else
+    HANDLE hfile;
+	bool endoffile;
+	RingBuffer r;
+  #endif
+#endif
+	HANDLE mutex;
+	int locks;
+} VFILE;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+VFILE *Vfnew(const wchar_t *filename, const char *mode, BOOL Cached);
+VFILE *Vfopen(VFILE *f, wchar_t *filename, const char *mode, BOOL Cached); // filename must be an NDE string
+size_t Vfread(void *ptr, size_t size, VFILE *buf);
+void Vfseek(VFILE *fl, uint32_t i, int whence);
+uint32_t Vftell(VFILE *fl);
+void Vfclose(VFILE *fl);
+void Vfdestroy(VFILE *fl); // benski> TODO: 
+void Vfwrite(const void *ptr, size_t size, VFILE *f);
+int Vfeof(VFILE *fl);
+int Vsync(VFILE *fl); // 1 on error updating
+int Vflock(VFILE *fl, BOOL is_sync=TRUE); // returns 0 on failure
+void Vfunlock(VFILE *fl, BOOL is_sync=TRUE);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

Src/nde/win/nde_c.cpp

@@ -0,0 +1,561 @@
+#include "nde_c.h"
+#include "../nde.h"
+#include "../../nu/AutoCharFn.h"
+#include "../../nu/AutoWide.h"
+
+/* Database */
+nde_database_t NDE_CreateDatabase(HINSTANCE hInstance)
+{
+	return (nde_database_t)new Database(hInstance);
+}
+
+void NDE_DestroyDatabase(nde_database_t db)
+{
+	delete (Database *)db;
+}
+
+nde_table_t NDE_Database_OpenTable(nde_database_t db, const wchar_t *filename, const wchar_t *indexname, int create, int cache)
+{
+	Database *database = (Database *)db;
+	if (database && filename)
+		return (nde_table_t)database->OpenTable(filename, indexname, (BOOL)create, (BOOL)cache);
+	else
+		return 0;
+}
+
+void NDE_Database_CloseTable(nde_database_t db, nde_table_t t)
+{
+	Database *database = (Database *)db;
+	Table *table = (Table *)t;
+	if (database && table)
+	{
+		database->CloseTable(table);
+	}
+}
+/* Table */
+
+nde_field_t NDE_Table_NewColumn(nde_table_t t, unsigned char id, const char *name, unsigned char type)
+{
+	Table *table = (Table *)t;
+	if (table)
+		return (nde_field_t)table->NewColumn(id, AutoWideDup(name), type, FALSE);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Table_NewColumnW(nde_table_t t, unsigned char id, const wchar_t *name, unsigned char type)
+{
+	Table *table = (Table *)t;
+	if (table)
+		return (nde_field_t)table->NewColumn(id, name, type, FALSE);
+	else
+		return 0;
+}
+
+void NDE_Table_PostColumns(nde_table_t t)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->PostColumns();
+}
+
+void NDE_Table_AddIndexByID(nde_table_t t, unsigned char id, const char *name)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->AddIndexById(id, AutoWide(name));
+}
+
+void NDE_Table_AddIndexByIDW(nde_table_t t, unsigned char id, const wchar_t *name)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->AddIndexById(id, name);
+}
+
+nde_scanner_t NDE_Table_CreateScanner(nde_table_t t)
+{
+	Table *table = (Table *)t;
+	if (table)
+		return (nde_scanner_t)table->NewScanner();
+	else
+		return 0;
+}
+
+void NDE_Table_DestroyScanner(nde_table_t t, nde_scanner_t s)
+{
+	Table *table = (Table *)t;
+	Scanner *scanner = (Scanner *)s;
+	if (table && scanner)
+		table->DeleteScanner(scanner);
+}
+
+void NDE_Table_Sync(nde_table_t t)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->Sync();
+}
+
+void NDE_Table_Compact(nde_table_t t, int *progress)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->Compact(progress);
+}
+
+int NDE_Table_GetRecordsCount(nde_table_t t)
+{
+	Table *table = (Table *)t;
+	if (table)
+		return table->GetRecordsCount();
+	else
+		return 0;
+}
+
+nde_field_t NDE_Table_GetColumnByID(nde_table_t t, unsigned char id)
+{
+	Table *table = (Table *)t;
+	if (table)
+		return (nde_field_t)table->GetColumnById(id);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Table_GetColumnByName(nde_table_t t, const char *name)
+{
+	Table *table = (Table *)t;
+	if (table && name)
+		return (nde_field_t)table->GetColumnByName(AutoWide(name));
+	else
+		return 0;
+}
+
+void NDE_Table_SetColumnSearchableByID(nde_table_t t, unsigned char id, int searchable)
+{
+	Table *table = (Table *)t;
+	if (table)
+		table->SetFieldSearchableById(id, !!searchable);
+}
+
+/* Scanner */
+int NDE_Scanner_Query(nde_scanner_t s, const wchar_t *query)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->Query(query);
+	else
+		return 0;
+}
+
+void NDE_Scanner_Search(nde_scanner_t s, const wchar_t *search_term)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->Search(search_term);
+}
+
+const wchar_t *NDE_Scanner_GetLastQuery(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->GetLastQuery();
+	else
+		return 0;
+}
+
+int NDE_Scanner_GetRecordsCount(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->GetRecordsCount();
+	else
+		return 0;
+}
+
+void NDE_Scanner_New(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->New();
+}
+
+void NDE_Scanner_Post(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->Post();
+}
+
+void NDE_Scanner_First(nde_scanner_t s, int *killswitch)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->First(killswitch);
+}
+
+void NDE_Scanner_Delete(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->Delete();
+}
+
+void NDE_Scanner_Edit(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->Edit();
+}
+
+int NDE_Scanner_EOF(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->Eof();
+	else
+		return 1;
+}
+
+int NDE_Scanner_BOF(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->Bof();
+	else
+		return 1;
+}
+
+nde_field_t NDE_Scanner_NewFieldByID(nde_scanner_t s, unsigned char id)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return (nde_field_t)scanner->NewFieldById(id, PERM_READWRITE);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Scanner_NewFieldByType(nde_scanner_t s, unsigned char type, unsigned char id)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return (nde_field_t)scanner->NewFieldByType(type, id, PERM_READWRITE);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Scanner_NewFieldByName(nde_scanner_t s, const char *name)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return (nde_field_t)scanner->NewFieldByName(AutoWide(name), PERM_READWRITE);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Scanner_GetFieldByID(nde_scanner_t s, unsigned char id)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return (nde_field_t)scanner->GetFieldById(id);
+	else
+		return 0;
+}
+
+nde_field_t NDE_Scanner_GetFieldByName(nde_scanner_t s, const char *name)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner && name)
+		return (nde_field_t)scanner->GetFieldByName(AutoWide(name));
+	else
+		return 0;
+}
+
+void NDE_Scanner_AddFilterByID(nde_scanner_t s, unsigned char id, nde_field_t f, unsigned char filter_operation)
+{
+	Scanner *scanner = (Scanner *)s;
+	Field *field = (Field *)f;
+	if (scanner && field)
+		scanner->AddFilterById(id, field, filter_operation);
+}
+
+int NDE_Scanner_LocateInteger(nde_scanner_t s, unsigned char id, int from, int value, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+	{
+		IntegerField field(value);
+		return scanner->LocateByIdEx(id, from, &field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+int NDE_Scanner_LocateString(nde_scanner_t s, unsigned char id, int from, const wchar_t *value, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+	{
+		StringField field(value);
+		return scanner->LocateByIdEx(id, from, &field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+
+int NDE_Scanner_LocateNDEString(nde_scanner_t s, unsigned char id, int from, wchar_t *value, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+	{
+		StringField field(value, STRING_IS_NDESTRING);
+		return scanner->LocateByIdEx(id, from, &field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+int NDE_Scanner_LocateFilename(nde_scanner_t s, unsigned char id, int from, const wchar_t *value, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+	{
+		FilenameField field(value);
+		return scanner->LocateByIdEx(id, from, &field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+int NDE_Scanner_LocateNDEFilename(nde_scanner_t s, unsigned char id, int from, wchar_t *value, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+	{
+		FilenameField field(value, STRING_IS_NDESTRING);
+		return scanner->LocateByIdEx(id, from, &field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+int NDE_Scanner_LocateField(nde_scanner_t s, unsigned char id, int from, nde_field_t f, int *nskip, int compare_mode)
+{
+	Scanner *scanner = (Scanner *)s;
+	Field *field = (Field *)f;
+	if (scanner && field)
+	{
+		return scanner->LocateByIdEx(id, from, field, nskip, compare_mode);
+	}
+	else
+		return 0;
+}
+
+void NDE_Scanner_DeleteField(nde_scanner_t s, nde_field_t f)
+{
+	Scanner *scanner = (Scanner *)s;
+	Field *field = (Field *)f;
+	if (scanner && field)
+	{
+		scanner->DeleteField(field);
+	}
+}
+
+void NDE_Scanner_RemoveFilters(nde_scanner_t s)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		scanner->RemoveFilters();
+}
+
+int NDE_Scanner_Next(nde_scanner_t s, int *killswitch)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->Next(killswitch);
+	else
+		return 0;
+}
+
+void NDE_Scanner_WalkFields(nde_scanner_t s, FieldEnumerator enumerator, void *context)
+{
+	Scanner *scanner = (Scanner *)s;
+	if (scanner)
+		return scanner->WalkFields((Record::FieldsWalker)enumerator, context);
+}
+
+/* Filter functions */
+unsigned char NDE_Filter_GetID(nde_filter_t f)
+{
+	Filter *filter = (Filter *)f;
+	if (filter)
+		return filter->GetId();
+	else
+		return FILTERS_INVALID; // right value but I'm not sure if it's the best constant name to use
+}
+
+NDE_API unsigned char NDE_Filter_GetOp(nde_filter_t f)
+{
+	Filter *filter = (Filter *)f;
+	if (filter)
+		return filter->GetOp();
+	else
+		return FILTERS_INVALID; // right value but I'm not sure if it's the best constant name to use
+}
+
+NDE_API nde_field_t NDE_Filter_GetData(nde_filter_t f)
+{
+	Filter *filter = (Filter *)f;
+	if (filter)
+		return (nde_field_t)filter->Data();
+	else
+		return 0;
+}
+
+/* Field functions */
+unsigned char NDE_Field_GetType(nde_field_t f)
+{
+	Field *field = (Field *)f;
+	if (field)
+		return field->GetType();
+	else
+		return FIELD_UNKNOWN;
+}
+
+unsigned char NDE_Field_GetID(nde_field_t f)
+{
+	Field *field = (Field *)f;
+	if (field)
+		return field->GetFieldId();
+	else
+		return FIELD_UNKNOWN;
+}
+
+/* StringField functions */
+
+void NDE_StringField_SetNDEString(nde_field_t f, wchar_t *nde_string)
+{
+	StringField *field = (StringField *)(Field *)f;
+	if (field)
+		field->SetNDEString(nde_string);
+}
+
+wchar_t *NDE_StringField_GetString(nde_field_t f)
+{
+	StringField *field = (StringField *)(Field *)f;
+	if (field)
+		return field->GetStringW();
+	else
+		return 0;
+}
+
+void NDE_StringField_SetString(nde_field_t f, const wchar_t *str)
+{
+	StringField *field = (StringField *)(Field *)f;
+	if (field)
+		field->SetStringW(str);
+}
+
+/* IntegerField functions */
+void NDE_IntegerField_SetValue(nde_field_t f, int value)
+{
+	IntegerField *field = (IntegerField *)(Field *)f;
+	if (field)
+		field->SetValue(value);
+}
+
+int NDE_IntegerField_GetValue(nde_field_t f)
+{
+	IntegerField *field = (IntegerField *)(Field *)f;
+	if (field)
+		return field->GetValue();
+	else
+		return 0;
+}
+
+nde_field_t NDE_IntegerField_Create(int value)
+{
+	return (nde_field_t)new IntegerField(value);
+}
+
+/* Int64Field functions */
+void NDE_Int64Field_SetValue(nde_field_t f, __int64 value)
+{
+	Int64Field *field = (Int64Field *)(Field *)f;
+	if (field)
+		field->SetValue(value);
+}
+
+__int64 NDE_Int64Field_GetValue(nde_field_t f)
+{
+	Int64Field *field = (Int64Field *)(Field *)f;
+	if (field)
+		return field->GetValue();
+	else
+		return 0;
+}
+
+nde_field_t NDE_Int64Field_Create(__int64 value)
+{
+	return (nde_field_t)new Int64Field(value);
+}
+
+/* BinaryField */
+void *NDE_BinaryField_GetData(nde_field_t f, size_t *length)
+{
+	BinaryField *field = (BinaryField *)(Field *)f;
+	if (field)
+		return (void *)field->GetData(length);
+	else
+		return 0;
+}
+void NDE_BinaryField_SetData(nde_field_t f, const void *data, size_t length)
+{
+	BinaryField *field = (BinaryField *)(Field *)f;
+	if (field)
+		field->SetData((const uint8_t *)data, length);
+}
+
+/* Int128Field */
+void NDE_Int128Field_SetValue(nde_field_t f, const void *value)
+{
+	Int128Field *field = (Int128Field *)(Field *)f;
+	if (field && value)
+		field->SetValue(value);
+}
+
+/* ColumnField */
+const wchar_t *NDE_ColumnField_GetFieldName(nde_field_t f)
+{
+	ColumnField *field = (ColumnField *)(Field *)f;
+	if (field)
+		return field->GetFieldName();
+	else
+		return 0;
+}
+
+unsigned char NDE_ColumnField_GetDataType(nde_field_t f)
+{
+	ColumnField *field = (ColumnField *)(Field *)f;
+	if (field)
+		return field->GetDataType();
+	else
+		return FIELD_UNKNOWN;
+}
+
+unsigned char NDE_ColumnField_GetFieldID(nde_field_t f)
+{
+	ColumnField *field = (ColumnField *)(Field *)f;
+	if (field)
+		return field->GetFieldId();
+	else
+		return FIELD_UNKNOWN;
+}
+
+__time64_t NDE_Time_ApplyConversion(__time64_t value, const wchar_t *str, class TimeParse *tp)
+{
+	IntegerField f((int)value);
+	f.ApplyConversion(str, tp);
+	return f.GetValue();
+}

Src/nde/win/nde_c.h

@@ -0,0 +1,131 @@
+#pragma once
+/* C style API.
+
+We'll eventually deprecate the C++ API as it presents a lot of linking challenges
+*/
+#include "../nde_defines.h" 
+#include "../NDEString.h"
+#include <bfc/platform/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+	typedef struct nde_database_struct_t { } *nde_database_t;
+	typedef struct nde_table_struct_t { } *nde_table_t;
+	typedef struct nde_scanner_t_struct_t { } *nde_scanner_t;
+	typedef struct nde_field_t_struct_t { } *nde_field_t;
+	typedef struct nde_filter_struct_t { } *nde_filter_t;
+	
+	/* Database functions */
+
+	// Windows API
+#ifdef __cplusplus
+#define NDE_DEFAULT_PARAMETER(x) =x
+#else
+#define NDE_DEFAULT_PARAMETER(x)
+#endif
+
+NDE_API void NDE_Init();
+NDE_API void NDE_Quit();
+NDE_API nde_database_t NDE_CreateDatabase(HINSTANCE hInstance NDE_DEFAULT_PARAMETER(0)); 
+NDE_API void NDE_DestroyDatabase(nde_database_t db);
+NDE_API nde_table_t NDE_Database_OpenTable(nde_database_t db, const wchar_t *filename, const wchar_t *filename_index, int create, int cache);
+
+NDE_API void NDE_Database_CloseTable(nde_database_t db, nde_table_t table);
+
+/* Table functions */
+NDE_API nde_field_t NDE_Table_NewColumn(nde_table_t table, unsigned char id, const char *name, unsigned char type);
+NDE_API nde_field_t NDE_Table_NewColumnW(nde_table_t table, unsigned char id, const wchar_t *name, unsigned char type);
+NDE_API void NDE_Table_PostColumns(nde_table_t table);
+NDE_API void NDE_Table_AddIndexByID(nde_table_t table, unsigned char id, const char *name);
+NDE_API void NDE_Table_AddIndexByIDW(nde_table_t table, unsigned char id, const wchar_t *name);
+NDE_API nde_scanner_t NDE_Table_CreateScanner(nde_table_t table);
+NDE_API void NDE_Table_DestroyScanner(nde_table_t table, nde_scanner_t scanner);
+NDE_API void NDE_Table_Sync(nde_table_t table);
+NDE_API void NDE_Table_Compact(nde_table_t table, int *progress NDE_DEFAULT_PARAMETER(0));
+NDE_API int NDE_Table_GetRecordsCount(nde_table_t table);
+NDE_API nde_field_t NDE_Table_GetColumnByID(nde_table_t table, unsigned char id);
+
+NDE_API nde_field_t NDE_Table_GetColumnByName(nde_table_t table, const char *name);
+
+NDE_API void NDE_Table_SetColumnSearchableByID(nde_table_t table, unsigned char id, int searchable);
+
+/* Scanner functions */
+NDE_API int NDE_Scanner_Query(nde_scanner_t scanner, const wchar_t *query);
+NDE_API void NDE_Scanner_Search(nde_scanner_t scanner, const wchar_t *search_term);
+NDE_API const wchar_t *NDE_Scanner_GetLastQuery(nde_scanner_t scanner);
+NDE_API int NDE_Scanner_GetRecordsCount(nde_scanner_t scanner);
+NDE_API void NDE_Scanner_New(nde_scanner_t scanner);
+NDE_API void NDE_Scanner_Post(nde_scanner_t scanner);
+NDE_API void NDE_Scanner_First(nde_scanner_t scanner, int *killswitch NDE_DEFAULT_PARAMETER(0));
+NDE_API int NDE_Scanner_Next(nde_scanner_t scanner, int *killswitch NDE_DEFAULT_PARAMETER(0));
+NDE_API void NDE_Scanner_Delete(nde_scanner_t scanner);
+NDE_API void NDE_Scanner_Edit(nde_scanner_t scanner);
+NDE_API int NDE_Scanner_EOF(nde_scanner_t scanner);
+NDE_API int NDE_Scanner_BOF(nde_scanner_t scanner);
+NDE_API nde_field_t NDE_Scanner_NewFieldByID(nde_scanner_t scanner, unsigned char id);
+NDE_API nde_field_t NDE_Scanner_NewFieldByType(nde_scanner_t s, unsigned char type, unsigned char id);
+NDE_API nde_field_t NDE_Scanner_NewFieldByName(nde_scanner_t scanner, const char *name);
+NDE_API nde_field_t NDE_Scanner_GetFieldByID(nde_scanner_t scanner, unsigned char id);
+NDE_API nde_field_t NDE_Scanner_GetFieldByName(nde_scanner_t scanner, const char *name);
+NDE_API void NDE_Scanner_AddFilterByID(nde_scanner_t scanner, unsigned char id, nde_field_t field, unsigned char filter_operation);
+
+NDE_API int NDE_Scanner_LocateInteger(nde_scanner_t scanner, unsigned char id, int from, int value,
+																			int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+NDE_API int NDE_Scanner_LocateString(nde_scanner_t scanner, unsigned char id, int from, const wchar_t *value, 
+																		 int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+NDE_API int NDE_Scanner_LocateNDEString(nde_scanner_t scanner, unsigned char id, int from, wchar_t *value,
+																				int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+NDE_API int NDE_Scanner_LocateFilename(nde_scanner_t scanner, unsigned char id, int from, const wchar_t *value, 
+																			 int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+NDE_API int NDE_Scanner_LocateNDEFilename(nde_scanner_t scanner, unsigned char id, int from, wchar_t *value, 
+																					int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+NDE_API int NDE_Scanner_LocateField(nde_scanner_t scanner, unsigned char id, int from, nde_field_t field,
+																		int *nskip NDE_DEFAULT_PARAMETER(0), int compare_mode NDE_DEFAULT_PARAMETER(COMPARE_MODE_EXACT));
+
+NDE_API void NDE_Scanner_DeleteField(nde_scanner_t scanner, nde_field_t field);
+NDE_API void NDE_Scanner_RemoveFilters(nde_scanner_t scanner);
+typedef int (__cdecl *FieldEnumerator)(void *record, nde_field_t field, void *context);
+NDE_API void NDE_Scanner_WalkFields(nde_scanner_t scanner, FieldEnumerator enumerator, void *context);
+/* Filter functions */
+NDE_API unsigned char NDE_Filter_GetID(nde_filter_t filter);
+NDE_API unsigned char NDE_Filter_GetOp(nde_filter_t filter);
+NDE_API nde_field_t NDE_Filter_GetData(nde_filter_t filter);
+
+/* Field functions */
+NDE_API unsigned char NDE_Field_GetType(nde_field_t field);
+NDE_API unsigned char NDE_Field_GetID(nde_field_t field);
+
+/* String Field functions */
+NDE_API wchar_t *NDE_StringField_GetString(nde_field_t field); /* returns non-const because it's an NDE string (reference counted, see ndestring.h) */
+NDE_API void NDE_StringField_SetNDEString(nde_field_t field, wchar_t *nde_string);
+NDE_API void NDE_StringField_SetString(nde_field_t field, const wchar_t *str);
+
+/* IntegerField functions */
+NDE_API void NDE_IntegerField_SetValue(nde_field_t field, int value);
+NDE_API int NDE_IntegerField_GetValue(nde_field_t field);
+NDE_API nde_field_t NDE_IntegerField_Create(int value); /* mainly used for NDE_Scanner_AddFilterByID */
+
+/* Int64Field functions */
+NDE_API void NDE_Int64Field_SetValue(nde_field_t field, __int64 value);
+NDE_API __int64 NDE_Int64Field_GetValue(nde_field_t field);
+NDE_API nde_field_t NDE_Int64Field_Create(__int64 value);
+
+/* BinaryField functions */
+// on windows, the data pointer is optionally reference counted via ndestring (ndestring_retain if you plan on keeping it)
+NDE_API void *NDE_BinaryField_GetData(nde_field_t field, size_t *length); 
+NDE_API void NDE_BinaryField_SetData(nde_field_t field, const void *data, size_t length);
+
+/* Int128Field functions */
+NDE_API void NDE_Int128Field_SetValue(nde_field_t field, const void *value);
+
+/* ColumnField functions */
+NDE_API const wchar_t *NDE_ColumnField_GetFieldName(nde_field_t field);
+NDE_API unsigned char NDE_ColumnField_GetDataType(nde_field_t field);
+NDE_API unsigned char NDE_ColumnField_GetFieldID(nde_field_t);
+
+NDE_API __time64_t NDE_Time_ApplyConversion(__time64_t value, const wchar_t *str, class TimeParse *tp); 
+
+#ifdef __cplusplus
+}
+#endif

Src/nde/win/nde_init.cpp

@@ -0,0 +1,36 @@
+#include "../nde_c.h"
+#include "../DBUtils.h"
+
+#include <atomic>
+
+extern "C" void NDE_HeapInit();
+extern "C" void NDE_HeapQuit();
+static volatile std::atomic<std::size_t> _init_count = 0;
+
+/* NDE_Init isn't thread safe, be aware
+best to call on the main thread during initialization
+*/
+void NDE_Init()
+{
+	if ( _init_count.load() == 0 )
+	{
+		NDE_HeapInit();
+		HMODULE klib = LoadLibraryW( L"Kernel32.dll" );
+		if ( klib )
+		{
+			void *nls = GetProcAddress( klib, "FindNLSString" );
+			if ( nls )
+				*( (void **)&findNLSString ) = nls;
+		}
+
+		FreeModule( klib );
+	}
+
+	_init_count.fetch_add( 1 );
+}
+
+void NDE_Quit()
+{
+	if ( _init_count.fetch_sub( 1 ) == 0 )
+		NDE_HeapQuit();
+}