swish-e/src/db_native.h

/*
**
** This program and library is free software; you can redistribute it and/or
** modify it under the terms of the GNU (Library) General Public License
** as published by the Free Software Foundation; either version 2
** of the License, or any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU (Library) General Public License for more details.
**
** You should have received a copy of the GNU (Library) General Public License
** along with this program; if not, write to the Free Software
** Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
**
**
**
** 2001-01  jose   initial coding
**
*/


#ifndef __HasSeenModule_DBNative
#define __HasSeenModule_DBNative    1


#ifdef USE_BTREE
#include "btree.h"
#include "array.h"
#include "worddata.h"
#define MAXCHARS 6            /* Only 5 are needed when BTREE is used */

#else

#define MAXCHARS 266            /* 255 for chars plus ten more for other data */

#endif

#define FILELISTPOS (MAXCHARS - 1)
#define FILEOFFSETPOS (MAXCHARS - 2)
#define HEADERPOS (MAXCHARS - 3)
#define WORDPOS (MAXCHARS - 4)
#define SORTEDINDEX (MAXCHARS - 5)

#ifdef USE_BTREE
#define TOTALWORDSPERFILEPOS (MAXCHARS - 6)
#endif


struct Handle_DBNative
{
       /* values used by the index proccess */
       /* points to the start of offsets to words in the file */
   int offsetstart;

   SWISH *sw;  /* for reporting errors back */

#ifndef USE_BTREE
       /* points to the start of hashoffsets to words in the file */
   int hashstart;
#endif
       /* File Offsets to words */
   long offsets[MAXCHARS];

#ifndef USE_BTREE
   long hashoffsets[VERYBIGHASHSIZE];

   int lasthashval[VERYBIGHASHSIZE];
   int wordhash_counter;
#endif

   long nextwordoffset;
   long lastsortedindex;
   long next_sortedindex;
   
   int worddata_counter;

#ifndef USE_BTREE
   long *wordhashdata;

      /* Hash array to improve wordhashdata performance */
   struct numhash
   {
      int index;
      struct numhash *next;
   } *hash[BIGHASHSIZE];
   MEM_ZONE *hashzone;

#endif

   int num_words;

   DB_OPEN_MODE mode; 

   char *dbname;

#ifndef USE_BTREE
       /* ramdisk to store words */
   struct ramdisk *rd;
#endif

       /* Index FILE handle as returned from fopen */

       /* Pointers to words write/read functions */ 
   long    (*w_tell)(FILE *);
   size_t  (*w_write)(const void *, size_t, size_t, FILE *);
   int     (*w_seek)(FILE *, long, int);
   size_t  (*w_read)(void *, size_t, size_t, FILE *);
   int     (*w_close)(FILE *);
   int     (*w_putc)(int , FILE *);
   int     (*w_getc)(FILE *);

   FILE *fp;
   FILE *prop;

   int      tmp_index;      /* These indicates the file is opened as a temporary file */
   int      tmp_prop;
   char    *cur_index_file;
   char    *cur_prop_file;

   long     unique_ID;          /* just because it's called that doesn't mean it is! */

#ifdef USE_BTREE
   BTREE   *bt;
   FILE    *fp_btree;
   int      tmp_btree;
   char    *cur_btree_file;

   WORDDATA    *worddata;
   FILE    *fp_worddata;
   int      tmp_worddata;
   char    *cur_worddata_file;

   FILE    *fp_array;
   int      tmp_array;
   char    *cur_array_file;

   int      n_presorted_array;
   unsigned long *presorted_root_node;
   unsigned long *presorted_propid;
   ARRAY  **presorted_array;
   FILE    *fp_presorted;
   int      tmp_presorted;
   char    *cur_presorted_file;

   unsigned long cur_presorted_propid;
   ARRAY   *cur_presorted_array;

   ARRAY   *totwords_array;

   ARRAY   *props_array;
#endif
};


void initModule_DBNative (SWISH *);
void freeModule_DBNative (SWISH *);
int configModule_DBNative (SWISH *sw, StringList *sl);

void   *DB_Create_Native (SWISH *sw, char *dbname);
void   *DB_Open_Native (SWISH *sw, char *dbname, int mode);
void    DB_Close_Native(void *db);
void    DB_Remove_Native(void *db);


int     DB_InitWriteHeader_Native(void *db);
int     DB_EndWriteHeader_Native(void *db);
int     DB_WriteHeaderData_Native(int id, unsigned char *s, int len, void *db);

int     DB_InitReadHeader_Native(void *db);
int     DB_ReadHeaderData_Native(int *id, unsigned char **s, int *len, void *db);
int     DB_EndReadHeader_Native(void *db);


int     DB_InitWriteWords_Native(void *db);
long    DB_GetWordID_Native(void *db);
int     DB_WriteWord_Native(char *word, long wordID, void *db);

#ifdef USE_BTREE
int     DB_UpdateWordID_Native(char *word, long new_wordID, void *db);
int     DB_DeleteWordData_Native(long wordID, void *db);
#endif

int     DB_WriteWordHash_Native(char *word, long wordID, void *db);
long    DB_WriteWordData_Native(long wordID, unsigned char *worddata, int lendata, void *db);
int     DB_EndWriteWords_Native(void *db);

int     DB_InitReadWords_Native(void *db);
int     DB_ReadWordHash_Native(char *word, long *wordID, void *db);
int     DB_ReadFirstWordInvertedIndex_Native(char *word, char **resultword, long *wordID, void *db);
int     DB_ReadNextWordInvertedIndex_Native(char *word, char **resultword, long *wordID, void *db);
long    DB_ReadWordData_Native(long wordID, unsigned char **worddata, int *lendata, void *db);
int     DB_EndReadWords_Native(void *db);


int     DB_InitWriteFiles_Native(void *db);
int     DB_WriteFile_Native(int filenum, unsigned char *filedata,int sz_filedata, void *db);
int     DB_EndWriteFiles_Native(void *db);

int     DB_InitReadFiles_Native(void *db);
int     DB_ReadFile_Native(int filenum, unsigned char **filedata,int *sz_filedata, void *db);
int     DB_EndReadFiles_Native(void *db);


#ifdef USE_BTREE
int     DB_InitWriteSortedIndex_Native(void *db , int n_props);
int     DB_WriteSortedIndex_Native(int propID, int *data, int sz_data,void *db);
#else
int     DB_InitWriteSortedIndex_Native(void *db );
int     DB_WriteSortedIndex_Native(int propID, unsigned char *data, int sz_data,void *db);
#endif
int     DB_EndWriteSortedIndex_Native(void *db);
 
int     DB_InitReadSortedIndex_Native(void *db);
int     DB_ReadSortedIndex_Native(int propID, unsigned char **data, int *sz_data,void *db);
int     DB_ReadSortedData_Native(int *data,int index, int *value, void *db);
int     DB_EndReadSortedIndex_Native(void *db);

void    DB_WriteProperty_Native( IndexFILE *indexf, FileRec *fi, int propID, char *buffer, int buf_len, int uncompressed_len, void *db);
void    DB_WritePropPositions_Native(IndexFILE *indexf, FileRec *fi, void *db);
void    DB_ReadPropPositions_Native(IndexFILE *indexf, FileRec *fi, void *db);
char   *DB_ReadProperty_Native(IndexFILE *indexf, FileRec *fi, int propID, int *buf_len, int *uncompressed_len, void *db);
void    DB_Reopen_PropertiesForRead_Native(void *db);

#ifdef USE_BTREE
int        DB_InitWriteTotalWordsPerFile_Native(SWISH *sw, void *DB);
int    DB_WriteTotalWordsPerFile_Native(SWISH *sw, int idx, int wordcount, void *DB);
int    DB_EndWriteTotalWordsPerFile_Native(SWISH *sw, void *DB);
int        DB_InitReadTotalWordsPerFile_Native(SWISH *sw, void *DB);
int    DB_ReadTotalWordsPerFile_Native(SWISH *sw, int idx, int *wordcount, void *DB);
int    DB_EndReadTotalWordsPerFile_Native(SWISH *sw, void *DB);
#endif


/* 04/00 Jose Ruiz
** Functions to read/write longs from a file
*/
void    printlong(FILE * fp, unsigned long num, size_t (*f_write)(const void *, size_t, size_t, FILE *));
unsigned long    readlong(FILE * fp, size_t (*f_read)(void *, size_t, size_t, FILE *));


#endif
1	/*
2	**
3	** This program and library is free software; you can redistribute it and/or
4	** modify it under the terms of the GNU (Library) General Public License
5	** as published by the Free Software Foundation; either version 2
6	** of the License, or any later version.
7	**
8	** This program is distributed in the hope that it will be useful,
9	** but WITHOUT ANY WARRANTY; without even the implied warranty of
10	** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	** GNU (Library) General Public License for more details.
12	**
13	** You should have received a copy of the GNU (Library) General Public License
14	** along with this program; if not, write to the Free Software
15	** Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16	**
17	**
18	**
19	** 2001-01 jose initial coding
20	**
21	*/
22
23
24
25	#ifndef __HasSeenModule_DBNative
26	#define __HasSeenModule_DBNative 1
27
28
29	#ifdef USE_BTREE
30	#include "btree.h"
31	#include "array.h"
32	#include "worddata.h"
33	#define MAXCHARS 6 /* Only 5 are needed when BTREE is used */
34
35	#else
36
37	#define MAXCHARS 266 /* 255 for chars plus ten more for other data */
38
39	#endif
40
41	#define FILELISTPOS (MAXCHARS - 1)
42	#define FILEOFFSETPOS (MAXCHARS - 2)
43	#define HEADERPOS (MAXCHARS - 3)
44	#define WORDPOS (MAXCHARS - 4)
45	#define SORTEDINDEX (MAXCHARS - 5)
46
47	#ifdef USE_BTREE
48	#define TOTALWORDSPERFILEPOS (MAXCHARS - 6)
49	#endif
50
51
52
53	struct Handle_DBNative
54	{
55	/* values used by the index proccess */
56	/* points to the start of offsets to words in the file */
57	int offsetstart;
58
59	SWISH sw; / for reporting errors back */
60
61	#ifndef USE_BTREE
62	/* points to the start of hashoffsets to words in the file */
63	int hashstart;
64	#endif
65	/* File Offsets to words */
66	long offsets[MAXCHARS];
67
68	#ifndef USE_BTREE
69	long hashoffsets[VERYBIGHASHSIZE];
70
71	int lasthashval[VERYBIGHASHSIZE];
72	int wordhash_counter;
73	#endif
74
75	long nextwordoffset;
76	long lastsortedindex;
77	long next_sortedindex;
78
79	int worddata_counter;
80
81	#ifndef USE_BTREE
82	long *wordhashdata;
83
84	/* Hash array to improve wordhashdata performance */
85	struct numhash
86	{
87	int index;
88	struct numhash *next;
89	} *hash[BIGHASHSIZE];
90	MEM_ZONE *hashzone;
91
92	#endif
93
94	int num_words;
95
96	DB_OPEN_MODE mode;
97
98	char *dbname;
99
100	#ifndef USE_BTREE
101	/* ramdisk to store words */
102	struct ramdisk *rd;
103	#endif
104
105	/* Index FILE handle as returned from fopen */
106
107	/* Pointers to words write/read functions */
108	long (w_tell)(FILE );
109	size_t (w_write)(const void , size_t, size_t, FILE *);
110	int (w_seek)(FILE , long, int);
111	size_t (w_read)(void , size_t, size_t, FILE *);
112	int (w_close)(FILE );
113	int (w_putc)(int , FILE );
114	int (w_getc)(FILE );
115
116	FILE *fp;
117	FILE *prop;
118
119	int tmp_index; /* These indicates the file is opened as a temporary file */
120	int tmp_prop;
121	char *cur_index_file;
122	char *cur_prop_file;
123
124	long unique_ID; /* just because it's called that doesn't mean it is! */
125
126	#ifdef USE_BTREE
127	BTREE *bt;
128	FILE *fp_btree;
129	int tmp_btree;
130	char *cur_btree_file;
131
132	WORDDATA *worddata;
133	FILE *fp_worddata;
134	int tmp_worddata;
135	char *cur_worddata_file;
136
137	FILE *fp_array;
138	int tmp_array;
139	char *cur_array_file;
140
141	int n_presorted_array;
142	unsigned long *presorted_root_node;
143	unsigned long *presorted_propid;
144	ARRAY **presorted_array;
145	FILE *fp_presorted;
146	int tmp_presorted;
147	char *cur_presorted_file;
148
149	unsigned long cur_presorted_propid;
150	ARRAY *cur_presorted_array;
151
152	ARRAY *totwords_array;
153
154	ARRAY *props_array;
155	#endif
156	};
157
158
159	void initModule_DBNative (SWISH *);
160	void freeModule_DBNative (SWISH *);
161	int configModule_DBNative (SWISH sw, StringList sl);
162
163	void DB_Create_Native (SWISH sw, char *dbname);
164	void DB_Open_Native (SWISH sw, char *dbname, int mode);
165	void DB_Close_Native(void *db);
166	void DB_Remove_Native(void *db);
167
168
169
170	int DB_InitWriteHeader_Native(void *db);
171	int DB_EndWriteHeader_Native(void *db);
172	int DB_WriteHeaderData_Native(int id, unsigned char s, int len, void db);
173
174	int DB_InitReadHeader_Native(void *db);
175	int DB_ReadHeaderData_Native(int id, unsigned char s, int len, void *db);
176	int DB_EndReadHeader_Native(void *db);
177
178
179
180	int DB_InitWriteWords_Native(void *db);
181	long DB_GetWordID_Native(void *db);
182	int DB_WriteWord_Native(char word, long wordID, void db);
183
184	#ifdef USE_BTREE
185	int DB_UpdateWordID_Native(char word, long new_wordID, void db);
186	int DB_DeleteWordData_Native(long wordID, void *db);
187	#endif
188
189	int DB_WriteWordHash_Native(char word, long wordID, void db);
190	long DB_WriteWordData_Native(long wordID, unsigned char worddata, int lendata, void db);
191	int DB_EndWriteWords_Native(void *db);
192
193	int DB_InitReadWords_Native(void *db);
194	int DB_ReadWordHash_Native(char word, long wordID, void *db);
195	int DB_ReadFirstWordInvertedIndex_Native(char word, char resultword, long wordID, void *db);
196	int DB_ReadNextWordInvertedIndex_Native(char word, char resultword, long wordID, void *db);
197	long DB_ReadWordData_Native(long wordID, unsigned char *worddata, int lendata, void *db);
198	int DB_EndReadWords_Native(void *db);
199
200
201
202	int DB_InitWriteFiles_Native(void *db);
203	int DB_WriteFile_Native(int filenum, unsigned char filedata,int sz_filedata, void db);
204	int DB_EndWriteFiles_Native(void *db);
205
206	int DB_InitReadFiles_Native(void *db);
207	int DB_ReadFile_Native(int filenum, unsigned char *filedata,int sz_filedata, void *db);
208	int DB_EndReadFiles_Native(void *db);
209
210
211	#ifdef USE_BTREE
212	int DB_InitWriteSortedIndex_Native(void *db , int n_props);
213	int DB_WriteSortedIndex_Native(int propID, int data, int sz_data,void db);
214	#else
215	int DB_InitWriteSortedIndex_Native(void *db );
216	int DB_WriteSortedIndex_Native(int propID, unsigned char data, int sz_data,void db);
217	#endif
218	int DB_EndWriteSortedIndex_Native(void *db);
219
220	int DB_InitReadSortedIndex_Native(void *db);
221	int DB_ReadSortedIndex_Native(int propID, unsigned char *data, int sz_data,void *db);
222	int DB_ReadSortedData_Native(int data,int index, int value, void *db);
223	int DB_EndReadSortedIndex_Native(void *db);
224
225	void DB_WriteProperty_Native( IndexFILE indexf, FileRec fi, int propID, char buffer, int buf_len, int uncompressed_len, void db);
226	void DB_WritePropPositions_Native(IndexFILE indexf, FileRec fi, void *db);
227	void DB_ReadPropPositions_Native(IndexFILE indexf, FileRec fi, void *db);
228	char DB_ReadProperty_Native(IndexFILE indexf, FileRec fi, int propID, int buf_len, int uncompressed_len, void db);
229	void DB_Reopen_PropertiesForRead_Native(void *db);
230
231	#ifdef USE_BTREE
232	int DB_InitWriteTotalWordsPerFile_Native(SWISH sw, void DB);
233	int DB_WriteTotalWordsPerFile_Native(SWISH sw, int idx, int wordcount, void DB);
234	int DB_EndWriteTotalWordsPerFile_Native(SWISH sw, void DB);
235	int DB_InitReadTotalWordsPerFile_Native(SWISH sw, void DB);
236	int DB_ReadTotalWordsPerFile_Native(SWISH sw, int idx, int wordcount, void *DB);
237	int DB_EndReadTotalWordsPerFile_Native(SWISH sw, void DB);
238	#endif
239
240
241
242
243
244	/* 04/00 Jose Ruiz
245	** Functions to read/write longs from a file
246	*/
247	void printlong(FILE * fp, unsigned long num, size_t (f_write)(const void , size_t, size_t, FILE *));
248	unsigned long readlong(FILE * fp, size_t (f_read)(void , size_t, size_t, FILE *));
249
250
251	#endif