| 1 |
/* |
| 2 |
** NOTE: This implementation was originally part of the WAIS system |
| 3 |
** The main function, Stem(), was incorporated into Swish-E 1.1 |
| 4 |
** to provide a stemming function. |
| 5 |
** 11/24/98 Mark Gaulin |
| 6 |
** |
| 7 |
** Stem returns original word if words stems to empty string |
| 8 |
** Bill Moseley 10/11/99 |
| 9 |
** |
| 10 |
** Repeats stemming until word will stem no more |
| 11 |
** Bill Moseley 10/17/99 |
| 12 |
** |
| 13 |
** function: EndsWithCVC patched a bug. see below. Moseley 10/19/99 |
| 14 |
** |
| 15 |
** Added word length arg to ReplaceEnd and Stem to avoid strcat overflow |
| 16 |
** 11/17/99 - SRE |
| 17 |
** |
| 18 |
** fixed int cast, missing return value, braces around initializations: problems pointed out by "gcc -Wall" |
| 19 |
** SRE 2/22/00 |
| 20 |
** |
| 21 |
** Jose Ruiz 18/10/00 |
| 22 |
** Remove static word from end var and make the code thread safe |
| 23 |
** |
| 24 |
** Bill Moseley 20/05/01 |
| 25 |
** Rewrote to simplify. No more need to repeat stem (expandstar gone from search.c) |
| 26 |
** got rid of most of the reallocation of memory. |
| 27 |
*/ |
| 28 |
|
| 29 |
/* WIDE AREA INFORMATION SERVER SOFTWARE: |
| 30 |
No guarantees or restrictions. See the readme file for the full standard |
| 31 |
disclaimer. |
| 32 |
|
| 33 |
francois@welchgate.welch.jhu.edu |
| 34 |
*/ |
| 35 |
|
| 36 |
/* Copyright (c) CNIDR (see ../COPYRIGHT) */ |
| 37 |
|
| 38 |
|
| 39 |
|
| 40 |
/* |
| 41 |
* stems a word. |
| 42 |
* |
| 43 |
*/ |
| 44 |
|
| 45 |
/* the main functions are: |
| 46 |
* stemmer |
| 47 |
* |
| 48 |
*/ |
| 49 |
|
| 50 |
#include "swish.h" |
| 51 |
#include <ctype.h> |
| 52 |
#include <string.h> |
| 53 |
#include <stdio.h> |
| 54 |
|
| 55 |
|
| 56 |
#include "stemmer.h" |
| 57 |
#include "mem.h" |
| 58 |
|
| 59 |
|
| 60 |
#define FALSE 0 |
| 61 |
#define TRUE 1 |
| 62 |
|
| 63 |
|
| 64 |
|
| 65 |
/******************************* stem.c *********************************** |
| 66 |
|
| 67 |
Purpose: Implementation of the Porter stemming algorithm documented |
| 68 |
in: Porter, M.F., "An Algorithm For Suffix Stripping," |
| 69 |
Program 14 (3), July 1980, pp. 130-137. |
| 70 |
|
| 71 |
Provenance: Written by B. Frakes and C. Cox, 1986. |
| 72 |
Changed by C. Fox, 1990. |
| 73 |
- made measure function a DFA |
| 74 |
- restructured structs |
| 75 |
- renamed functions and variables |
| 76 |
- restricted function and variable scopes |
| 77 |
Changed by C. Fox, July, 1991. |
| 78 |
- added ANSI C declarations |
| 79 |
- branch tested to 90% coverage |
| 80 |
|
| 81 |
Notes: This code will make little sense without the the Porter |
| 82 |
article. The stemming function converts its input to |
| 83 |
lower case. |
| 84 |
**/ |
| 85 |
|
| 86 |
/************************ Standard Include Files *************************/ |
| 87 |
|
| 88 |
/*****************************************************************************/ |
| 89 |
/***************** Private Defines and Data Structures *******************/ |
| 90 |
|
| 91 |
#define IsVowel(c) ('a'==(c)||'e'==(c)||'i'==(c)||'o'==(c)||'u'==(c)) |
| 92 |
|
| 93 |
typedef struct |
| 94 |
{ |
| 95 |
int id; /* returned if rule fired */ |
| 96 |
char *old_end; /* suffix replaced */ |
| 97 |
char *new_end; /* suffix replacement */ |
| 98 |
int old_offset; /* from end of word to start of suffix */ |
| 99 |
int new_offset; /* from beginning to end of new suffix */ |
| 100 |
int min_root_size; /* min root word size for replacement */ |
| 101 |
int (*condition) (); /* the replacement test function */ |
| 102 |
} |
| 103 |
RuleList; |
| 104 |
|
| 105 |
#define LAMBDA "" /* the constant empty string */ |
| 106 |
|
| 107 |
/*****************************************************************************/ |
| 108 |
/******************** Private Function Declarations **********************/ |
| 109 |
|
| 110 |
#ifdef __STDC__ |
| 111 |
|
| 112 |
int WordSize(char *word); |
| 113 |
int ContainsVowel(char *word); |
| 114 |
int EndsWithCVC(char *word); |
| 115 |
int AddAnE(char *word); |
| 116 |
int RemoveAnE(char *word); |
| 117 |
int ReplaceEnd(char *word, RuleList * rule); |
| 118 |
|
| 119 |
#else |
| 120 |
|
| 121 |
int WordSize( /* word */ ); |
| 122 |
int ContainsVowel( /* word */ ); |
| 123 |
int EndsWithCVC( /* word */ ); |
| 124 |
int AddAnE( /* word */ ); |
| 125 |
int RemoveAnE( /* word */ ); |
| 126 |
int ReplaceEnd( /* word, rule */ ); |
| 127 |
|
| 128 |
#endif |
| 129 |
|
| 130 |
/******************************************************************************/ |
| 131 |
/***************** Initialized Private Data Structures ********************/ |
| 132 |
/* 2/22/00 - added braces around each line */ |
| 133 |
|
| 134 |
static RuleList step1a_rules[] = { |
| 135 |
{101, "sses", "ss", 3, 1, -1, NULL,}, |
| 136 |
{102, "ies", "i", 2, 0, -1, NULL,}, |
| 137 |
{103, "ss", "ss", 1, 1, -1, NULL,}, |
| 138 |
{104, "s", LAMBDA, 0, -1, -1, NULL,}, |
| 139 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 140 |
}; |
| 141 |
|
| 142 |
static RuleList step1b_rules[] = { |
| 143 |
{105, "eed", "ee", 2, 1, 0, NULL,}, |
| 144 |
{106, "ed", LAMBDA, 1, -1, -1, ContainsVowel,}, |
| 145 |
{107, "ing", LAMBDA, 2, -1, -1, ContainsVowel,}, |
| 146 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 147 |
}; |
| 148 |
|
| 149 |
static RuleList step1b1_rules[] = { |
| 150 |
{108, "at", "ate", 1, 2, -1, NULL,}, |
| 151 |
{109, "bl", "ble", 1, 2, -1, NULL,}, |
| 152 |
{110, "iz", "ize", 1, 2, -1, NULL,}, |
| 153 |
{111, "bb", "b", 1, 0, -1, NULL,}, |
| 154 |
{112, "dd", "d", 1, 0, -1, NULL,}, |
| 155 |
{113, "ff", "f", 1, 0, -1, NULL,}, |
| 156 |
{114, "gg", "g", 1, 0, -1, NULL,}, |
| 157 |
{115, "mm", "m", 1, 0, -1, NULL,}, |
| 158 |
{116, "nn", "n", 1, 0, -1, NULL,}, |
| 159 |
{117, "pp", "p", 1, 0, -1, NULL,}, |
| 160 |
{118, "rr", "r", 1, 0, -1, NULL,}, |
| 161 |
{119, "tt", "t", 1, 0, -1, NULL,}, |
| 162 |
{120, "ww", "w", 1, 0, -1, NULL,}, |
| 163 |
{121, "xx", "x", 1, 0, -1, NULL,}, |
| 164 |
{122, LAMBDA, "e", -1, 0, -1, AddAnE,}, |
| 165 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 166 |
}; |
| 167 |
|
| 168 |
static RuleList step1c_rules[] = { |
| 169 |
{123, "y", "i", 0, 0, -1, ContainsVowel,}, |
| 170 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 171 |
}; |
| 172 |
|
| 173 |
static RuleList step2_rules[] = { |
| 174 |
{203, "ational", "ate", 6, 2, 0, NULL,}, |
| 175 |
{204, "tional", "tion", 5, 3, 0, NULL,}, |
| 176 |
{205, "enci", "ence", 3, 3, 0, NULL,}, |
| 177 |
{206, "anci", "ance", 3, 3, 0, NULL,}, |
| 178 |
{207, "izer", "ize", 3, 2, 0, NULL,}, |
| 179 |
{208, "abli", "able", 3, 3, 0, NULL,}, |
| 180 |
{209, "alli", "al", 3, 1, 0, NULL,}, |
| 181 |
{210, "entli", "ent", 4, 2, 0, NULL,}, |
| 182 |
{211, "eli", "e", 2, 0, 0, NULL,}, |
| 183 |
{213, "ousli", "ous", 4, 2, 0, NULL,}, |
| 184 |
{214, "ization", "ize", 6, 2, 0, NULL,}, |
| 185 |
{215, "ation", "ate", 4, 2, 0, NULL,}, |
| 186 |
{216, "ator", "ate", 3, 2, 0, NULL,}, |
| 187 |
{217, "alism", "al", 4, 1, 0, NULL,}, |
| 188 |
{218, "iveness", "ive", 6, 2, 0, NULL,}, |
| 189 |
{219, "fulnes", "ful", 5, 2, 0, NULL,}, |
| 190 |
{220, "ousness", "ous", 6, 2, 0, NULL,}, |
| 191 |
{221, "aliti", "al", 4, 1, 0, NULL,}, |
| 192 |
{222, "iviti", "ive", 4, 2, 0, NULL,}, |
| 193 |
{223, "biliti", "ble", 5, 2, 0, NULL,}, |
| 194 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 195 |
}; |
| 196 |
|
| 197 |
static RuleList step3_rules[] = { |
| 198 |
{301, "icate", "ic", 4, 1, 0, NULL,}, |
| 199 |
{302, "ative", LAMBDA, 4, -1, 0, NULL,}, |
| 200 |
{303, "alize", "al", 4, 1, 0, NULL,}, |
| 201 |
{304, "iciti", "ic", 4, 1, 0, NULL,}, |
| 202 |
{305, "ical", "ic", 3, 1, 0, NULL,}, |
| 203 |
{308, "ful", LAMBDA, 2, -1, 0, NULL,}, |
| 204 |
{309, "ness", LAMBDA, 3, -1, 0, NULL,}, |
| 205 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 206 |
}; |
| 207 |
|
| 208 |
static RuleList step4_rules[] = { |
| 209 |
{401, "al", LAMBDA, 1, -1, 1, NULL,}, |
| 210 |
{402, "ance", LAMBDA, 3, -1, 1, NULL,}, |
| 211 |
{403, "ence", LAMBDA, 3, -1, 1, NULL,}, |
| 212 |
{405, "er", LAMBDA, 1, -1, 1, NULL,}, |
| 213 |
{406, "ic", LAMBDA, 1, -1, 1, NULL,}, |
| 214 |
{407, "able", LAMBDA, 3, -1, 1, NULL,}, |
| 215 |
{408, "ible", LAMBDA, 3, -1, 1, NULL,}, |
| 216 |
{409, "ant", LAMBDA, 2, -1, 1, NULL,}, |
| 217 |
{410, "ement", LAMBDA, 4, -1, 1, NULL,}, |
| 218 |
{411, "ment", LAMBDA, 3, -1, 1, NULL,}, |
| 219 |
{412, "ent", LAMBDA, 2, -1, 1, NULL,}, |
| 220 |
{423, "sion", "s", 3, 0, 1, NULL,}, |
| 221 |
{424, "tion", "t", 3, 0, 1, NULL,}, |
| 222 |
{415, "ou", LAMBDA, 1, -1, 1, NULL,}, |
| 223 |
{416, "ism", LAMBDA, 2, -1, 1, NULL,}, |
| 224 |
{417, "ate", LAMBDA, 2, -1, 1, NULL,}, |
| 225 |
{418, "iti", LAMBDA, 2, -1, 1, NULL,}, |
| 226 |
{419, "ous", LAMBDA, 2, -1, 1, NULL,}, |
| 227 |
{420, "ive", LAMBDA, 2, -1, 1, NULL,}, |
| 228 |
{421, "ize", LAMBDA, 2, -1, 1, NULL,}, |
| 229 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 230 |
}; |
| 231 |
|
| 232 |
static RuleList step5a_rules[] = { |
| 233 |
{501, "e", LAMBDA, 0, -1, 1, NULL,}, |
| 234 |
{502, "e", LAMBDA, 0, -1, -1, RemoveAnE,}, |
| 235 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 236 |
}; |
| 237 |
|
| 238 |
static RuleList step5b_rules[] = { |
| 239 |
{503, "ll", "l", 1, 0, 1, NULL,}, |
| 240 |
{000, NULL, NULL, 0, 0, 0, NULL,} |
| 241 |
}; |
| 242 |
|
| 243 |
|
| 244 |
static RuleList *all_steps[] = { |
| 245 |
step1a_rules, |
| 246 |
step1b_rules, |
| 247 |
/* step1b1_rules, -- conditionaly called below */ |
| 248 |
step1c_rules, |
| 249 |
step2_rules, |
| 250 |
step3_rules, |
| 251 |
step4_rules, |
| 252 |
step5a_rules, |
| 253 |
step5b_rules, |
| 254 |
}; |
| 255 |
|
| 256 |
|
| 257 |
|
| 258 |
|
| 259 |
/*****************************************************************************/ |
| 260 |
/******************** Private Function Declarations **********************/ |
| 261 |
|
| 262 |
/*FN*************************************************************************** |
| 263 |
|
| 264 |
WordSize( word ) |
| 265 |
|
| 266 |
Returns: int -- a weird count of word size in adjusted syllables |
| 267 |
|
| 268 |
Purpose: Count syllables in a special way: count the number |
| 269 |
vowel-consonant pairs in a word, disregarding initial |
| 270 |
consonants and final vowels. The letter "y" counts as a |
| 271 |
consonant at the beginning of a word and when it has a vowel |
| 272 |
in front of it; otherwise (when it follows a consonant) it |
| 273 |
is treated as a vowel. For example, the WordSize of "cat" |
| 274 |
is 1, of "any" is 1, of "amount" is 2, of "anything" is 3. |
| 275 |
|
| 276 |
Plan: Run a DFA to compute the word size |
| 277 |
|
| 278 |
Notes: The easiest and fastest way to compute this funny measure is |
| 279 |
with a finite state machine. The initial state 0 checks |
| 280 |
the first letter. If it is a vowel, then the machine changes |
| 281 |
to state 1, which is the "last letter was a vowel" state. |
| 282 |
If the first letter is a consonant or y, then it changes |
| 283 |
to state 2, the "last letter was a consonant state". In |
| 284 |
state 1, a y is treated as a consonant (since it follows |
| 285 |
a vowel), but in state 2, y is treated as a vowel (since |
| 286 |
it follows a consonant. The result counter is incremented |
| 287 |
on the transition from state 1 to state 2, since this |
| 288 |
transition only occurs after a vowel-consonant pair, which |
| 289 |
is what we are counting. |
| 290 |
**/ |
| 291 |
|
| 292 |
int WordSize(char *word) |
| 293 |
{ |
| 294 |
register int result; /* WordSize of the word */ |
| 295 |
register int state; /* current state in machine */ |
| 296 |
|
| 297 |
result = 0; |
| 298 |
state = 0; |
| 299 |
|
| 300 |
/* Run a DFA to compute the word size */ |
| 301 |
while ( *word ) |
| 302 |
{ |
| 303 |
switch (state) |
| 304 |
{ |
| 305 |
case 0: |
| 306 |
state = (IsVowel(*word)) ? 1 : 2; |
| 307 |
break; |
| 308 |
case 1: |
| 309 |
state = (IsVowel(*word)) ? 1 : 2; |
| 310 |
if (2 == state) |
| 311 |
result++; |
| 312 |
break; |
| 313 |
case 2: |
| 314 |
state = (IsVowel(*word) || ('y' == *word)) ? 1 : 2; |
| 315 |
break; |
| 316 |
} |
| 317 |
word++; |
| 318 |
} |
| 319 |
|
| 320 |
return (result); |
| 321 |
|
| 322 |
} /* WordSize */ |
| 323 |
|
| 324 |
/*FN************************************************************************** |
| 325 |
|
| 326 |
ContainsVowel( word, end ) |
| 327 |
|
| 328 |
Returns: int -- TRUE (1) if the word parameter contains a vowel, |
| 329 |
FALSE (0) otherwise. |
| 330 |
|
| 331 |
Purpose: Some of the rewrite rules apply only to a root containing |
| 332 |
a vowel, where a vowel is one of "aeiou" or y with a |
| 333 |
consonant in front of it. |
| 334 |
|
| 335 |
Plan: Obviously, under the definition of a vowel, a word contains |
| 336 |
a vowel iff either its first letter is one of "aeiou", or |
| 337 |
any of its other letters are "aeiouy". The plan is to |
| 338 |
test this condition. |
| 339 |
|
| 340 |
Notes: None |
| 341 |
**/ |
| 342 |
|
| 343 |
int ContainsVowel(char *word) |
| 344 |
{ |
| 345 |
|
| 346 |
/* This isn't needed, right? */ |
| 347 |
if ( !*word ) |
| 348 |
return FALSE; |
| 349 |
|
| 350 |
return (IsVowel(*word) || (NULL != strpbrk(word + 1, "aeiouy"))); |
| 351 |
|
| 352 |
|
| 353 |
} /* ContainsVowel */ |
| 354 |
|
| 355 |
/*FN************************************************************************** |
| 356 |
|
| 357 |
EndsWithCVC( word ) |
| 358 |
|
| 359 |
Returns: int -- TRUE (1) if the current word ends with a |
| 360 |
consonant-vowel-consonant combination, and the second |
| 361 |
consonant is not w, x, or y, FALSE (0) otherwise. |
| 362 |
|
| 363 |
Purpose: Some of the rewrite rules apply only to a root with |
| 364 |
this characteristic. |
| 365 |
|
| 366 |
Plan: Look at the last three characters. |
| 367 |
|
| 368 |
Notes: None |
| 369 |
**/ |
| 370 |
|
| 371 |
int EndsWithCVC(char *word) |
| 372 |
{ |
| 373 |
int length; /* for finding the last three characters */ |
| 374 |
|
| 375 |
if ((length = (int) strlen(word)) < 3) /* This was < 2 in original - Moseley 10/19/99 */ |
| 376 |
return (FALSE); |
| 377 |
|
| 378 |
|
| 379 |
return ( |
| 380 |
(NULL == strchr("aeiouwxy", (int)word[--length])) /* consonant */ |
| 381 |
&& (NULL != strchr("aeiouy", (int)word[--length])) /* vowel */ |
| 382 |
&& (NULL == strchr("aeiou", (int)word[--length])) |
| 383 |
); /* consonant */ |
| 384 |
|
| 385 |
} /* EndsWithCVC */ |
| 386 |
|
| 387 |
/*FN************************************************************************** |
| 388 |
|
| 389 |
AddAnE( word ) |
| 390 |
|
| 391 |
Returns: int -- TRUE (1) if the current word meets special conditions |
| 392 |
for adding an e. |
| 393 |
|
| 394 |
Purpose: Rule 122 applies only to a root with this characteristic. |
| 395 |
|
| 396 |
Plan: Check for size of 1 and a consonant-vowel-consonant ending. |
| 397 |
|
| 398 |
Notes: None |
| 399 |
**/ |
| 400 |
|
| 401 |
int AddAnE(char *word) |
| 402 |
{ |
| 403 |
|
| 404 |
return ((1 == WordSize(word)) && EndsWithCVC(word)); |
| 405 |
|
| 406 |
} /* AddAnE */ |
| 407 |
|
| 408 |
/*FN************************************************************************** |
| 409 |
|
| 410 |
RemoveAnE( word ) |
| 411 |
|
| 412 |
Returns: int -- TRUE (1) if the current word meets special conditions |
| 413 |
for removing an e. |
| 414 |
|
| 415 |
Purpose: Rule 502 applies only to a root with this characteristic. |
| 416 |
|
| 417 |
Plan: Check for size of 1 and no consonant-vowel-consonant ending. |
| 418 |
|
| 419 |
Notes: None |
| 420 |
**/ |
| 421 |
|
| 422 |
int RemoveAnE(char *word) |
| 423 |
{ |
| 424 |
|
| 425 |
return ((1 == WordSize(word)) && !EndsWithCVC(word)); |
| 426 |
|
| 427 |
} /* RemoveAnE */ |
| 428 |
|
| 429 |
/*FN************************************************************************** |
| 430 |
|
| 431 |
ReplaceEnd( word, rule, end) |
| 432 |
|
| 433 |
Returns: int -- the id for the rule fired, 0 is none is fired |
| 434 |
|
| 435 |
Purpose: Apply a set of rules to replace the suffix of a word |
| 436 |
|
| 437 |
Plan: Loop through the rule set until a match meeting all conditions |
| 438 |
is found. If a rule fires, return its id, otherwise return 0. |
| 439 |
Connditions on the length of the root are checked as part of this |
| 440 |
function's processing because this check is so often made. |
| 441 |
|
| 442 |
Notes: This is the main routine driving the stemmer. It goes through |
| 443 |
a set of suffix replacement rules looking for a match on the |
| 444 |
current suffix. When it finds one, if the root of the word |
| 445 |
is long enough, and it meets whatever other conditions are |
| 446 |
required, then the suffix is replaced, and the function returns. |
| 447 |
**/ |
| 448 |
|
| 449 |
int ReplaceEnd(char *word, RuleList *rule) |
| 450 |
{ |
| 451 |
char *ending; /* set to start of possible stemmed suffix */ |
| 452 |
char tmp_ch; /* save replaced character when testing */ |
| 453 |
char *end; /* pointer to last char of string */ |
| 454 |
|
| 455 |
end = word + strlen( word ) - 1; |
| 456 |
|
| 457 |
while ( rule->id ) |
| 458 |
{ |
| 459 |
/* point ending to the start of the test sufix */ |
| 460 |
ending = end - rule->old_offset; |
| 461 |
|
| 462 |
/* is word long enough to contain suffix, and does it exist? */ |
| 463 |
if ( (word <= ending ) && (0 == strcmp(ending, rule->old_end)) ) |
| 464 |
{ |
| 465 |
tmp_ch = *ending; /* in case we change our mind */ |
| 466 |
*ending = '\0'; |
| 467 |
|
| 468 |
if ( rule->min_root_size < WordSize(word) ) |
| 469 |
if (!rule->condition || (*rule->condition) ( word )) |
| 470 |
{ |
| 471 |
/* replace the ending */ |
| 472 |
if ( (strlen( word ) + rule->new_offset + 1 ) >= MAXWORDLEN ) |
| 473 |
return STEM_WORD_TOO_BIG; |
| 474 |
strcat( word, rule->new_end ); |
| 475 |
|
| 476 |
return rule->id; |
| 477 |
} |
| 478 |
*ending = tmp_ch; /* nope, put it back */ |
| 479 |
} |
| 480 |
|
| 481 |
rule++; |
| 482 |
} |
| 483 |
|
| 484 |
return 0; |
| 485 |
|
| 486 |
} /* ReplaceEnd */ |
| 487 |
|
| 488 |
/*****************************************************************************/ |
| 489 |
/********************* Public Function Declarations **********************/ |
| 490 |
|
| 491 |
/*FN*************************************************************************** |
| 492 |
|
| 493 |
Stem( word ) |
| 494 |
|
| 495 |
Returns: int -- FALSE (0) if the word contains non-alphabetic characters |
| 496 |
and hence is not stemmed, TRUE (1) otherwise |
| 497 |
|
| 498 |
Purpose: Stem a word |
| 499 |
|
| 500 |
Plan: Part 1: Check to ensure the word is all alphabetic |
| 501 |
Part 2: Run through the Porter algorithm |
| 502 |
Part 3: Return an indication of successful stemming |
| 503 |
|
| 504 |
Notes: This function implements the Porter stemming algorithm, with |
| 505 |
a few additions here and there. See: |
| 506 |
|
| 507 |
Porter, M.F., "An Algorithm For Suffix Stripping," |
| 508 |
Program 14 (3), July 1980, pp. 130-137. |
| 509 |
|
| 510 |
Porter's algorithm is an ad hoc set of rewrite rules with |
| 511 |
various conditions on rule firing. The terminology of |
| 512 |
"step 1a" and so on, is taken directly from Porter's |
| 513 |
article, which unfortunately gives almost no justification |
| 514 |
for the various steps. Thus this function more or less |
| 515 |
faithfully refects the opaque presentation in the article. |
| 516 |
Changes from the article amount to a few additions to the |
| 517 |
rewrite rules; these are marked in the RuleList data |
| 518 |
structures with comments. |
| 519 |
**/ |
| 520 |
|
| 521 |
int Stem(char **inword, int *lenword) |
| 522 |
{ |
| 523 |
char *end; /* pointer to the end of the word */ |
| 524 |
char word[MAXWORDLEN+1]; |
| 525 |
int length; |
| 526 |
int rule_result; /* which rule is fired in replacing an end */ |
| 527 |
// RuleList **rule; |
| 528 |
int i; |
| 529 |
|
| 530 |
|
| 531 |
|
| 532 |
/* Make sure the word is not too large from the start. */ |
| 533 |
if ( strlen( *inword ) >= MAXWORDLEN ) |
| 534 |
return STEM_WORD_TOO_BIG; |
| 535 |
|
| 536 |
|
| 537 |
/* make working copy */ |
| 538 |
strcpy( word, *inword ); |
| 539 |
|
| 540 |
|
| 541 |
|
| 542 |
/* Part 1: Check to ensure the word is all alphabetic */ |
| 543 |
/* no longer converts to lower case -- word should be lower before calling */ |
| 544 |
|
| 545 |
for ( end = word; *end; end++ ) |
| 546 |
if ( !isalpha( (unsigned int) *end ) ) |
| 547 |
return STEM_NOT_ALPHA; |
| 548 |
|
| 549 |
|
| 550 |
|
| 551 |
/* Part 2: Run through the Porter algorithm */ |
| 552 |
|
| 553 |
|
| 554 |
for (i = 0; i < sizeof(all_steps)/sizeof(all_steps[0]); i++) |
| 555 |
{ |
| 556 |
rule_result = ReplaceEnd(word, all_steps[i]); |
| 557 |
|
| 558 |
if ((rule_result == 106) || (rule_result == 107)) |
| 559 |
rule_result = ReplaceEnd(word, step1b1_rules); |
| 560 |
|
| 561 |
if ( rule_result == STEM_WORD_TOO_BIG ) |
| 562 |
return rule_result; |
| 563 |
|
| 564 |
} |
| 565 |
|
| 566 |
|
| 567 |
|
| 568 |
length = strlen( word ); |
| 569 |
|
| 570 |
/* Stem must be two chars or more in length */ |
| 571 |
if ( length <= 1 ) |
| 572 |
return STEM_TO_NOTHING; |
| 573 |
|
| 574 |
/* reallocate memory if need more room */ |
| 575 |
|
| 576 |
if ( length >= *lenword ) |
| 577 |
{ |
| 578 |
efree( *inword ); |
| 579 |
|
| 580 |
*lenword = length; |
| 581 |
*inword = emalloc( *lenword + 1 ); |
| 582 |
} |
| 583 |
|
| 584 |
strcpy( *inword, word ); |
| 585 |
return STEM_OK; |
| 586 |
} |
| 587 |
|
Parent Directory
| 
Revision Log
| 
Revision Graph