swish-e/src/db_native.h

/*
**
** This program and library is free software; you can redistribute it and/or
** modify it under the terms of the GNU (Library) General Public License
** as published by the Free Software Foundation; either version 2
** of the License, or any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
** GNU (Library) General Public License for more details.
**
** You should have received a copy of the GNU (Library) General Public License
** along with this program; if not, write to the Free Software
** Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
**
**
**
** 2001-01  jose   initial coding
**
*/


#ifndef __HasSeenModule_DBNative
#define __HasSeenModule_DBNative    1


#ifdef USE_BTREE
#include "btree.h"
#include "array.h"
#include "worddata.h"
#define MAXCHARS 6            /* Only 5 are needed when BTREE is used */

#else

#define MAXCHARS 266            /* 255 for chars plus ten more for other data */

#endif

#define FILELISTPOS (MAXCHARS - 1)
#define FILEOFFSETPOS (MAXCHARS - 2)
#define HEADERPOS (MAXCHARS - 3)
#define WORDPOS (MAXCHARS - 4)
#define SORTEDINDEX (MAXCHARS - 5)

#ifdef USE_BTREE
#define TOTALWORDSPERFILEPOS (MAXCHARS - 6)
#endif


struct Handle_DBNative
{
       /* values used by the index proccess */
       /* points to the start of offsets to words in the file */
   int offsetstart;

   SWISH *sw;  /* for reporting errors back */

#ifndef USE_BTREE
       /* points to the start of hashoffsets to words in the file */
   int hashstart;
#endif
       /* File Offsets to words */
   long offsets[MAXCHARS];

#ifndef USE_BTREE
   long hashoffsets[VERYBIGHASHSIZE];

   int lasthashval[VERYBIGHASHSIZE];
   int wordhash_counter;
#endif

   long nextwordoffset;
   long lastsortedindex;
   long next_sortedindex;
   
   int worddata_counter;

#ifndef USE_BTREE
   long *wordhashdata;

      /* Hash array to improve wordhashdata performance */
   struct numhash
   {
      int index;
      struct numhash *next;
   } *hash[BIGHASHSIZE];
   MEM_ZONE *hashzone;

#endif

   int num_words;

   DB_OPEN_MODE mode; 

   char *dbname;

#ifndef USE_BTREE
       /* ramdisk to store words */
   struct ramdisk *rd;
#endif

       /* Index FILE handle as returned from fopen */

       /* Pointers to words write/read functions */ 
   long    (*w_tell)(FILE *);
   size_t  (*w_write)(const void *, size_t, size_t, FILE *);
   int     (*w_seek)(FILE *, long, int);
   size_t  (*w_read)(void *, size_t, size_t, FILE *);
   int     (*w_close)(FILE *);
   int     (*w_putc)(int , FILE *);
   int     (*w_getc)(FILE *);

   FILE *fp;
   FILE *prop;

   int      tmp_index;      /* These indicates the file is opened as a temporary file */
   int      tmp_prop;
   char    *cur_index_file;
   char    *cur_prop_file;

   long     unique_ID;          /* just because it's called that doesn't mean it is! */

#ifdef USE_BTREE
   BTREE   *bt;
   FILE    *fp_btree;
   int      tmp_btree;
   char    *cur_btree_file;

   WORDDATA    *worddata;
   FILE    *fp_worddata;
   int      tmp_worddata;
   char    *cur_worddata_file;

   FILE    *fp_array;
   int      tmp_array;
   char    *cur_array_file;

   int      n_presorted_array;
   unsigned long *presorted_root_node;
   unsigned long *presorted_propid;
   ARRAY  **presorted_array;
   FILE    *fp_presorted;
   int      tmp_presorted;
   char    *cur_presorted_file;

   unsigned long cur_presorted_propid;
   ARRAY   *cur_presorted_array;

   ARRAY   *totwords_array;

   ARRAY   *props_array;
#endif
};


void initModule_DBNative (SWISH *);
void freeModule_DBNative (SWISH *);
int configModule_DBNative (SWISH *sw, StringList *sl);

void   *DB_Create_Native (SWISH *sw, char *dbname);
void   *DB_Open_Native (SWISH *sw, char *dbname, int mode);
void    DB_Close_Native(void *db);
void    DB_Remove_Native(void *db);


int     DB_InitWriteHeader_Native(void *db);
int     DB_EndWriteHeader_Native(void *db);
int     DB_WriteHeaderData_Native(int id, unsigned char *s, int len, void *db);

int     DB_InitReadHeader_Native(void *db);
int     DB_ReadHeaderData_Native(int *id, unsigned char **s, int *len, void *db);
int     DB_EndReadHeader_Native(void *db);


int     DB_InitWriteWords_Native(void *db);
long    DB_GetWordID_Native(void *db);
int     DB_WriteWord_Native(char *word, long wordID, void *db);

#ifdef USE_BTREE
int     DB_UpdateWordID_Native(char *word, long new_wordID, void *db);
int     DB_DeleteWordData_Native(long wordID, void *db);
#endif

int     DB_WriteWordHash_Native(char *word, long wordID, void *db);
long    DB_WriteWordData_Native(long wordID, unsigned char *worddata, int lendata, void *db);
int     DB_EndWriteWords_Native(void *db);

int     DB_InitReadWords_Native(void *db);
int     DB_ReadWordHash_Native(char *word, long *wordID, void *db);
int     DB_ReadFirstWordInvertedIndex_Native(char *word, char **resultword, long *wordID, void *db);
int     DB_ReadNextWordInvertedIndex_Native(char *word, char **resultword, long *wordID, void *db);
long    DB_ReadWordData_Native(long wordID, unsigned char **worddata, int *lendata, void *db);
int     DB_EndReadWords_Native(void *db);


int     DB_InitWriteFiles_Native(void *db);
int     DB_WriteFile_Native(int filenum, unsigned char *filedata,int sz_filedata, void *db);
int     DB_EndWriteFiles_Native(void *db);

int     DB_InitReadFiles_Native(void *db);
int     DB_ReadFile_Native(int filenum, unsigned char **filedata,int *sz_filedata, void *db);
int     DB_EndReadFiles_Native(void *db);


#ifdef USE_BTREE
int     DB_InitWriteSortedIndex_Native(void *db , int n_props);
int     DB_WriteSortedIndex_Native(int propID, int *data, int sz_data,void *db);
#else
int     DB_InitWriteSortedIndex_Native(void *db );
int     DB_WriteSortedIndex_Native(int propID, unsigned char *data, int sz_data,void *db);
#endif
int     DB_EndWriteSortedIndex_Native(void *db);
 
int     DB_InitReadSortedIndex_Native(void *db);
int     DB_ReadSortedIndex_Native(int propID, unsigned char **data, int *sz_data,void *db);
int     DB_ReadSortedData_Native(int *data,int index, int *value, void *db);
int     DB_EndReadSortedIndex_Native(void *db);

void    DB_WriteProperty_Native( IndexFILE *indexf, FileRec *fi, int propID, char *buffer, int buf_len, int uncompressed_len, void *db);
void    DB_WritePropPositions_Native(IndexFILE *indexf, FileRec *fi, void *db);
void    DB_ReadPropPositions_Native(IndexFILE *indexf, FileRec *fi, void *db);
char   *DB_ReadProperty_Native(IndexFILE *indexf, FileRec *fi, int propID, int *buf_len, int *uncompressed_len, void *db);
void    DB_Reopen_PropertiesForRead_Native(void *db);

#ifdef USE_BTREE
int        DB_InitWriteTotalWordsPerFile_Native(SWISH *sw, void *DB);
int    DB_WriteTotalWordsPerFile_Native(SWISH *sw, int idx, int wordcount, void *DB);
int    DB_EndWriteTotalWordsPerFile_Native(SWISH *sw, void *DB);
int        DB_InitReadTotalWordsPerFile_Native(SWISH *sw, void *DB);
int    DB_ReadTotalWordsPerFile_Native(SWISH *sw, int idx, int *wordcount, void *DB);
int    DB_EndReadTotalWordsPerFile_Native(SWISH *sw, void *DB);
#endif


/* 04/00 Jose Ruiz
** Functions to read/write longs from a file
*/
void    printlong(FILE * fp, unsigned long num, size_t (*f_write)(const void *, size_t, size_t, FILE *));
unsigned long    readlong(FILE * fp, size_t (*f_read)(void *, size_t, size_t, FILE *));


#endif
1	adcroft	1.1	/*
2			**
3			** This program and library is free software; you can redistribute it and/or
4			** modify it under the terms of the GNU (Library) General Public License
5			** as published by the Free Software Foundation; either version 2
6			** of the License, or any later version.
7			**
8			** This program is distributed in the hope that it will be useful,
9			** but WITHOUT ANY WARRANTY; without even the implied warranty of
10			** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11			** GNU (Library) General Public License for more details.
12			**
13			** You should have received a copy of the GNU (Library) General Public License
14			** along with this program; if not, write to the Free Software
15			** Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16			**
17			**
18			**
19			** 2001-01 jose initial coding
20			**
21			*/
22
23
24
25			#ifndef __HasSeenModule_DBNative
26			#define __HasSeenModule_DBNative 1
27
28
29			#ifdef USE_BTREE
30			#include "btree.h"
31			#include "array.h"
32			#include "worddata.h"
33			#define MAXCHARS 6 /* Only 5 are needed when BTREE is used */
34
35			#else
36
37			#define MAXCHARS 266 /* 255 for chars plus ten more for other data */
38
39			#endif
40
41			#define FILELISTPOS (MAXCHARS - 1)
42			#define FILEOFFSETPOS (MAXCHARS - 2)
43			#define HEADERPOS (MAXCHARS - 3)
44			#define WORDPOS (MAXCHARS - 4)
45			#define SORTEDINDEX (MAXCHARS - 5)
46
47			#ifdef USE_BTREE
48			#define TOTALWORDSPERFILEPOS (MAXCHARS - 6)
49			#endif
50
51
52
53			struct Handle_DBNative
54			{
55			/* values used by the index proccess */
56			/* points to the start of offsets to words in the file */
57			int offsetstart;
58
59			SWISH sw; / for reporting errors back */
60
61			#ifndef USE_BTREE
62			/* points to the start of hashoffsets to words in the file */
63			int hashstart;
64			#endif
65			/* File Offsets to words */
66			long offsets[MAXCHARS];
67
68			#ifndef USE_BTREE
69			long hashoffsets[VERYBIGHASHSIZE];
70
71			int lasthashval[VERYBIGHASHSIZE];
72			int wordhash_counter;
73			#endif
74
75			long nextwordoffset;
76			long lastsortedindex;
77			long next_sortedindex;
78
79			int worddata_counter;
80
81			#ifndef USE_BTREE
82			long *wordhashdata;
83
84			/* Hash array to improve wordhashdata performance */
85			struct numhash
86			{
87			int index;
88			struct numhash *next;
89			} *hash[BIGHASHSIZE];
90			MEM_ZONE *hashzone;
91
92			#endif
93
94			int num_words;
95
96			DB_OPEN_MODE mode;
97
98			char *dbname;
99
100			#ifndef USE_BTREE
101			/* ramdisk to store words */
102			struct ramdisk *rd;
103			#endif
104
105			/* Index FILE handle as returned from fopen */
106
107			/* Pointers to words write/read functions */
108			long (w_tell)(FILE );
109			size_t (w_write)(const void , size_t, size_t, FILE *);
110			int (w_seek)(FILE , long, int);
111			size_t (w_read)(void , size_t, size_t, FILE *);
112			int (w_close)(FILE );
113			int (w_putc)(int , FILE );
114			int (w_getc)(FILE );
115
116			FILE *fp;
117			FILE *prop;
118
119			int tmp_index; /* These indicates the file is opened as a temporary file */
120			int tmp_prop;
121			char *cur_index_file;
122			char *cur_prop_file;
123
124			long unique_ID; /* just because it's called that doesn't mean it is! */
125
126			#ifdef USE_BTREE
127			BTREE *bt;
128			FILE *fp_btree;
129			int tmp_btree;
130			char *cur_btree_file;
131
132			WORDDATA *worddata;
133			FILE *fp_worddata;
134			int tmp_worddata;
135			char *cur_worddata_file;
136
137			FILE *fp_array;
138			int tmp_array;
139			char *cur_array_file;
140
141			int n_presorted_array;
142			unsigned long *presorted_root_node;
143			unsigned long *presorted_propid;
144			ARRAY **presorted_array;
145			FILE *fp_presorted;
146			int tmp_presorted;
147			char *cur_presorted_file;
148
149			unsigned long cur_presorted_propid;
150			ARRAY *cur_presorted_array;
151
152			ARRAY *totwords_array;
153
154			ARRAY *props_array;
155			#endif
156			};
157
158
159			void initModule_DBNative (SWISH *);
160			void freeModule_DBNative (SWISH *);
161			int configModule_DBNative (SWISH sw, StringList sl);
162
163			void DB_Create_Native (SWISH sw, char *dbname);
164			void DB_Open_Native (SWISH sw, char *dbname, int mode);
165			void DB_Close_Native(void *db);
166			void DB_Remove_Native(void *db);
167
168
169
170			int DB_InitWriteHeader_Native(void *db);
171			int DB_EndWriteHeader_Native(void *db);
172			int DB_WriteHeaderData_Native(int id, unsigned char s, int len, void db);
173
174			int DB_InitReadHeader_Native(void *db);
175			int DB_ReadHeaderData_Native(int id, unsigned char s, int len, void *db);
176			int DB_EndReadHeader_Native(void *db);
177
178
179
180			int DB_InitWriteWords_Native(void *db);
181			long DB_GetWordID_Native(void *db);
182			int DB_WriteWord_Native(char word, long wordID, void db);
183
184			#ifdef USE_BTREE
185			int DB_UpdateWordID_Native(char word, long new_wordID, void db);
186			int DB_DeleteWordData_Native(long wordID, void *db);
187			#endif
188
189			int DB_WriteWordHash_Native(char word, long wordID, void db);
190			long DB_WriteWordData_Native(long wordID, unsigned char worddata, int lendata, void db);
191			int DB_EndWriteWords_Native(void *db);
192
193			int DB_InitReadWords_Native(void *db);
194			int DB_ReadWordHash_Native(char word, long wordID, void *db);
195			int DB_ReadFirstWordInvertedIndex_Native(char word, char resultword, long wordID, void *db);
196			int DB_ReadNextWordInvertedIndex_Native(char word, char resultword, long wordID, void *db);
197			long DB_ReadWordData_Native(long wordID, unsigned char *worddata, int lendata, void *db);
198			int DB_EndReadWords_Native(void *db);
199
200
201
202			int DB_InitWriteFiles_Native(void *db);
203			int DB_WriteFile_Native(int filenum, unsigned char filedata,int sz_filedata, void db);
204			int DB_EndWriteFiles_Native(void *db);
205
206			int DB_InitReadFiles_Native(void *db);
207			int DB_ReadFile_Native(int filenum, unsigned char *filedata,int sz_filedata, void *db);
208			int DB_EndReadFiles_Native(void *db);
209
210
211			#ifdef USE_BTREE
212			int DB_InitWriteSortedIndex_Native(void *db , int n_props);
213			int DB_WriteSortedIndex_Native(int propID, int data, int sz_data,void db);
214			#else
215			int DB_InitWriteSortedIndex_Native(void *db );
216			int DB_WriteSortedIndex_Native(int propID, unsigned char data, int sz_data,void db);
217			#endif
218			int DB_EndWriteSortedIndex_Native(void *db);
219
220			int DB_InitReadSortedIndex_Native(void *db);
221			int DB_ReadSortedIndex_Native(int propID, unsigned char *data, int sz_data,void *db);
222			int DB_ReadSortedData_Native(int data,int index, int value, void *db);
223			int DB_EndReadSortedIndex_Native(void *db);
224
225			void DB_WriteProperty_Native( IndexFILE indexf, FileRec fi, int propID, char buffer, int buf_len, int uncompressed_len, void db);
226			void DB_WritePropPositions_Native(IndexFILE indexf, FileRec fi, void *db);
227			void DB_ReadPropPositions_Native(IndexFILE indexf, FileRec fi, void *db);
228			char DB_ReadProperty_Native(IndexFILE indexf, FileRec fi, int propID, int buf_len, int uncompressed_len, void db);
229			void DB_Reopen_PropertiesForRead_Native(void *db);
230
231			#ifdef USE_BTREE
232			int DB_InitWriteTotalWordsPerFile_Native(SWISH sw, void DB);
233			int DB_WriteTotalWordsPerFile_Native(SWISH sw, int idx, int wordcount, void DB);
234			int DB_EndWriteTotalWordsPerFile_Native(SWISH sw, void DB);
235			int DB_InitReadTotalWordsPerFile_Native(SWISH sw, void DB);
236			int DB_ReadTotalWordsPerFile_Native(SWISH sw, int idx, int wordcount, void *DB);
237			int DB_EndReadTotalWordsPerFile_Native(SWISH sw, void DB);
238			#endif
239
240
241
242
243
244			/* 04/00 Jose Ruiz
245			** Functions to read/write longs from a file
246			*/
247			void printlong(FILE * fp, unsigned long num, size_t (f_write)(const void , size_t, size_t, FILE *));
248			unsigned long readlong(FILE * fp, size_t (f_read)(void , size_t, size_t, FILE *));
249
250
251			#endif