diff --git a/check/fixes.frm b/check/fixes.frm index 96783c8b..f9205bc5 100644 --- a/check/fixes.frm +++ b/check/fixes.frm @@ -2162,13 +2162,160 @@ Print; .end # Only for 64-bit systems. Otherwise "Output term too large". #require wordsize == 4 -# For now it fails because -# "Currently Stage 4 sorts are not allowed for function arguments or $ variables." -assert runtime_error? -#assert succeeded? -#assert result("test1") =~ expr("0") -#assert result("test2") =~ expr("g(0)") +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("test1") =~ expr("0") +assert result("test2") =~ expr("g(0)") *--#] Issue211 : +*--#[ Issue215_1 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol j,x; + +#define BLOWUP "1000" + +Local argument = f(x) + f(2*x); +Argument f; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("argument") =~ expr("3003/2") +*--#] Issue215_1 : +*--#[ Issue215_2 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +Symbol j,x; + +#define BLOWUP "1000" + +#$dol = x; +Local dollar = 1; +Inside $dol; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndInside; +Multiply $dol; +Identify x = 1; +Print; +ModuleOption local $dol; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("dollar") =~ expr("1001/2") +*--#] Issue215_2 : +*--#[ Issue215_3 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +Symbol j,x; + +#define BLOWUP "1000" + +Local term = x; +Term; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndTerm; +Identify x = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("term") =~ expr("1001/2") +*--#] Issue215_3 : +*--#[ Issue215_4 : +#- +#: TermsInSmall 16 +#: LargePatches 4 +#: FilePatches 4 + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol i,j,x; + +#define N "257" +#define BLOWUP "257" + +Local nested = {`N'*(`N'+1)/2} + #do i = 1,`N' + - f(x^`i') + #enddo + ; + +Argument f; + Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x^i? = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("nested") =~ expr("0") +*--#] Issue215_4 : +*--#[ Issue215_4b : +#- +#: TermsInSmall 16 +#: LargePatches 4 +#: FilePatches 4 + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol i,j,x; + +* Enough terms to have a stagesort in each thread of a -w5 test +#define N "{5*257}" +#define BLOWUP "257" + +Local nested = {`N'*(`N'+1)/2} + #do i = 1,`N' + - f(x^`i') + #enddo + ; +* Sort to distribute terms around workers +.sort + +Argument f; + Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x^i? = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("nested") =~ expr("0") +*--#] Issue215_4b : *--#[ Issue222 : * accessing #factdollar factors causes program termination Symbol x; diff --git a/doc/manual/setup.tex b/doc/manual/setup.tex index 12168922..5bf93670 100644 --- a/doc/manual/setup.tex +++ b/doc/manual/setup.tex @@ -370,12 +370,12 @@ \chapter{The setup} file operations are notoriously slow the combination of the small buffer, the small extension and the large buffer is used for caching\index{cache} purposes. Hence this space can be split in `FilePatches' caches. The -limitation is that each cache should be capable to contain at least two -terms of maximal size. This means that the sum of SmallExtension and -LargeSize must be at least FilePatches times 2*MaxTermSize*(bytes in short -integer). It is possible to set the size of these caches directly with the -variable {\sl SortIOsize\index{sortiosize}}. If the variable is too large, -the variable FilePatches may be adjusted by \FORM\@. If there are more than +limitation is that each cache should be capable of containing at least +three terms of maximal size. This means that the sum of SmallExtension +and LargeSize must be at least FilePatches times 3*MaxTermSize*(bytes in +short integer). FilePatches may be adjusted by \FORM\ to accomodate this, +and LargeSize if FilePatches is already minimal. +If there are more than FilePatches patches in the sort file, a second sort file is needed for the output of each `superpatch'\index{superpatch}. When the first sort file has been treated, the second sort file can be treated in exactly the same way @@ -409,14 +409,9 @@ \chapter{The setup} \rightvitem{13cm}{The size of the large buffer.} \leftvitem{3cm}{SortIOsize\index{setup!sortiosize}\index{sortiosize}} -\rightvitem{13cm}{The size of the buffer that is used to write to the -intermediate sorting file and to read from it. It should be noted that if -this buffer is not very large, the sorting of large files may become rather -slow, depending on the operating system. Hence we recommend a potential -fourth stage in the sorting over having this number too small to fit more -filepatches in the combined small and large buffer. Setting the small and -large buffers to a decent size may avoid all problems by a: making more -space for the caching, b: creating fewer file patches to start with.} +\rightvitem{13cm}{The size of the output buffer for the intermediate sorting +file, and of the buffers used for compressed file I/O. \FORM\ will ensure +that this is at least MaxTermSize.} There is a second set of the above setup parameters for sorts of subexpressions\index{subexpressions} as in function arguments or in the diff --git a/doc/manual/sorting.tex b/doc/manual/sorting.tex index ed5146b6..adfa26a1 100644 --- a/doc/manual/sorting.tex +++ b/doc/manual/sorting.tex @@ -254,10 +254,10 @@ \chapter{Sorting and statistics} the chapter on the setup \ref{setup}. This chapter gives also all default values. -There is one restriction on the sorting of function arguments and -\$-expressions: They are not allowed to go into the stage4 sorting. Any -such attempt will result in an error message and the suggestion to raise -the size of the buffers for this type of sorting. +Sorting these `sub-buffers' may also run into a stage 4 sort when necessary. +This usually implies that performance will be severely affected, hence the +first time this happens during a program a warning will be printed, with the +suggestion to increase the sizes of the sub-buffers. When \FORM\ is running in parallel mode (either \TFORM\ or \ParFORM) each worker will need its own buffers. In \ParFORM\ in which the processors each control diff --git a/sources/checkpoint.c b/sources/checkpoint.c index 343170d6..8216e77f 100644 --- a/sources/checkpoint.c +++ b/sources/checkpoint.c @@ -1184,7 +1184,6 @@ static void print_R() MesPrint("%d", AR.DeferFlag); MesPrint("%d", AR.TePos); MesPrint("%d", AR.sLevel); - MesPrint("%d", AR.Stage4Name); MesPrint("%d", AR.GetOneFile); MesPrint("%d", AR.PolyFun); MesPrint("%d", AR.PolyFunInv); @@ -2333,7 +2332,6 @@ int DoRecovery(int *moduletype) R_SET(AR.DeferFlag, WORD); R_SET(AR.TePos, WORD); R_SET(AR.sLevel, WORD); - R_SET(AR.Stage4Name, WORD); R_SET(AR.GetOneFile, WORD); R_SET(AR.PolyFun, WORD); R_SET(AR.PolyFunInv, WORD); @@ -2957,7 +2955,6 @@ static int DoSnapshot(int moduletype) S_WRITE_B(&AR.DeferFlag, sizeof(WORD)); S_WRITE_B(&AR.TePos, sizeof(WORD)); S_WRITE_B(&AR.sLevel, sizeof(WORD)); - S_WRITE_B(&AR.Stage4Name, sizeof(WORD)); S_WRITE_B(&AR.GetOneFile, sizeof(WORD)); S_WRITE_B(&AR.PolyFun, sizeof(WORD)); S_WRITE_B(&AR.PolyFunInv, sizeof(WORD)); diff --git a/sources/compress.c b/sources/compress.c index 8fc82ef9..f8e02e0d 100644 --- a/sources/compress.c +++ b/sources/compress.c @@ -712,16 +712,23 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer /* INTERNAL_ERROR_EXCL_STOP */ } else { + // The caches may be larger than the remaining patch data. Limit the read size: + if ( ISGEPOS(*position,S->fPatchesStop[numstream]) ) return(0); + toread = buffersize; + if ( !ISGEPOSINC(S->fPatchesStop[numstream],*position,toread) ) { + DIFPOS(pos,S->fPatchesStop[numstream],*position); + toread = (LONG)BASEPOSITION(pos); + } #ifdef GZIPDEBUG MLOCK(ErrorMessageLock); - MesPrint("%w++Reading %l bytes at position %10p",buffersize,position); + MesPrint("%w++Reading %l bytes at position %10p",toread,position); MUNLOCK(ErrorMessageLock); #endif #ifdef ALLLOCK LOCK(f->pthreadslock); #endif SeekFile(f->handle,position,SEEK_SET); - readsize = ReadFile(f->handle,buffer,buffersize); + readsize = ReadFile(f->handle,buffer,toread); SeekFile(f->handle,position,SEEK_CUR); #ifdef ALLLOCK UNLOCK(f->pthreadslock); @@ -730,7 +737,7 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>%wFillInputGZIP: Read error during uncompressed sort."); - MesPrint("%w++Reading %l bytes at position %10p",buffersize,position); + MesPrint("%w++Reading %l bytes at position %10p",toread,position); MUNLOCK(ErrorMessageLock); /* INTERNAL_ERROR_EXCL_STOP */ } diff --git a/sources/declare.h b/sources/declare.h index cd4fb2e4..43a676dd 100644 --- a/sources/declare.h +++ b/sources/declare.h @@ -688,7 +688,7 @@ extern void PrintFeatureList(void); extern void PrintRunningTime(void); extern LONG GetRunningTime(void); extern int PutBracket(PHEAD WORD *); -extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,int); +extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,LONG,int); extern int PutInStore(INDEXENTRY *,WORD); extern WORD PutOut(PHEAD WORD *,POSITION *,FILEHANDLE *,WORD); extern UWORD Quotient(UWORD *,WORD *,WORD); diff --git a/sources/fsizes.h b/sources/fsizes.h index bcd877a0..6c8f87c2 100644 --- a/sources/fsizes.h +++ b/sources/fsizes.h @@ -129,6 +129,7 @@ #endif #define MAXPATCHES 256 +#define MINFPATCHES 4 #define MAXFPATCHES 256 #define SORTIOSIZE 200000L diff --git a/sources/proces.c b/sources/proces.c index b0fc8f74..b7b11edf 100644 --- a/sources/proces.c +++ b/sources/proces.c @@ -205,6 +205,7 @@ int Processor(void) #endif AS.CollectOverFlag = 0; AR.expchanged = 0; + AT.SubSortStage4Warning = 0; if ( i == last ) LastExpression = 1; else LastExpression = 0; if ( e->inmem ) { diff --git a/sources/setfile.c b/sources/setfile.c index d5ecee9e..847c9f68 100644 --- a/sources/setfile.c +++ b/sources/setfile.c @@ -417,10 +417,10 @@ int AllocSetups(void) SETUPPARAMETERS *sp; LONG LargeSize, SmallSize, SmallEsize, TermsInSmall, IOsize; int MaxPatches, MaxFpatches, error = 0, i, size; - UBYTE *s; #ifndef WITHPTHREADS int j; #endif + UBYTE *s; sp = GetSetupPar((UBYTE *)"threads"); if ( sp->value > 0 ) AM.totalnumberofthreads = sp->value+1; @@ -585,22 +585,8 @@ int AllocSetups(void) AM.S0 = 0; AM.S0 = AllocSort(LargeSize,SmallSize,SmallEsize,TermsInSmall ,MaxPatches,MaxFpatches,IOsize,0); - /* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */ -#ifdef WITHZLIB - AM.S0->file.ziosize = IOsize; #ifndef WITHPTHREADS - AR.FoStage4[0].ziosize = IOsize; - AR.FoStage4[1].ziosize = IOsize; AT.S0 = AM.S0; -#endif -#else -#ifndef WITHPTHREADS - AT.S0 = AM.S0; -#endif -#endif -#ifndef WITHPTHREADS - AR.FoStage4[0].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.FoStage4[1].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); #endif sp = GetSetupPar((UBYTE *)"subsmallsize"); AM.SSmallSize = sp->value; @@ -877,11 +863,9 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i LONG TermsInSmall = inTermsInSmall; int MaxPatches = inMaxPatches; int MaxFpatches = inMaxFpatches; - LONG IOsize = inIOsize; LONG longer,terms2insmall,sortsize,longerp; - LONG IObuffersize = IOsize; - LONG IOtry; + LONG IObuffersize = inIOsize; SORTING *sort; int fname2Size = 0, j = 0; char *s; @@ -890,7 +874,7 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i while ( *s ) { s++; fname2Size++; } fname2Size += 16; } - if ( MaxFpatches < 4 ) MaxFpatches = 4; + if ( MaxFpatches < MINFPATCHES ) MaxFpatches = MINFPATCHES; longer = MaxPatches > MaxFpatches ? MaxPatches : MaxFpatches; longerp = longer; while ( (1 << j) < longerp ) j++; @@ -907,28 +891,24 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i if ( LargeSize < 0 ) LargeSize = 0; sortsize = sizeof(SORTING); sortsize = (sortsize+15)&(-16L); + if ( IObuffersize < AM.MaxTer ) IObuffersize = AM.MaxTer; IObuffersize = (IObuffersize+sizeof(WORD)-1)/sizeof(WORD); /* - The next statement fixes a bug. In the rare case that we have a - problem here, we expand the size of the large buffer or the - small extension + File patches are cached in the combined large+small buffer when reading. + Each cache needs to fit needs to fit a maximally sized term, plus headroom + of two maximally sized terms and COMPINC. COMPINC guarantees that the + two-WORD header of the following compressed term fits in the cache. These + cache spaces are independent of the file's own IO buffer (PObuffer) which + is controlled by "SortIOSize". */ - if ( (ULONG)( LargeSize+SmallEsize ) < MaxFpatches*((IObuffersize - +COMPINC)*sizeof(WORD)+2*AM.MaxTer) ) { + if ( (ULONG)( LargeSize+SmallEsize ) < MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)) ) { if ( LargeSize == 0 ) - SmallEsize = MaxFpatches*((IObuffersize+COMPINC)*sizeof(WORD)+2*AM.MaxTer); + SmallEsize = MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)); else - LargeSize = MaxFpatches*((IObuffersize+COMPINC)*sizeof(WORD)+2*AM.MaxTer) + LargeSize = MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)) - SmallEsize; } - IOtry = ((LargeSize+SmallEsize)/MaxFpatches-2*AM.MaxTer)/sizeof(WORD)-COMPINC; - - /* Here both IObuffersize and IOtry are in units of sizeof(WORD). */ - if ( IObuffersize < IOtry ) { - IObuffersize = IOtry; - } - #if DEBUGGING char *prefix; if ( level == 0 ) { prefix = ""; } @@ -939,8 +919,6 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i if ( TermsInSmall != inTermsInSmall ) { MesPrint("Warning: %sTermsInSmall adjusted: %l -> %l", prefix, inTermsInSmall, TermsInSmall); } if ( MaxPatches != inMaxPatches ) { MesPrint("Warning: MaxPatches adjusted: %d -> %d", inMaxPatches, MaxPatches); } if ( MaxFpatches != inMaxFpatches ) {MesPrint("Warning: MaxFPatches adjusted: %d -> %d", inMaxFpatches, MaxFpatches); } - /* This one is always changed if the LargeSize has not been... */ - /* if ( IObuffersize != inIOsize/(LONG)sizeof(WORD) ) { MesPrint("Warning: IOsize adjusted: %l -> %l", inIOsize/sizeof(WORD), IObuffersize); } */ #endif /* Allocate separate buffers for most struct members, for better testing and debugging with valgrind. */ @@ -998,6 +976,17 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i #ifdef WITHZLIB sort->file.ziosize = IObuffersize*sizeof(WORD); sort->file.ziobuffer = 0; +#endif + FILEHANDLE *stage4 = &(sort->FoStage4); + memset(stage4,0,sizeof(*stage4)); + stage4->POsize = sort->file.POsize; + stage4->handle = -1; +#ifdef WITHPTHREADS + stage4->pthreadslock = dummylock; +#endif +#ifdef WITHZLIB + stage4->ziosize = sort->file.ziosize; + stage4->ziobuffer = 0; #endif if ( AM.S0 != 0 ) { sort->file.name = (char *)(sort->file.PObuffer + IObuffersize); diff --git a/sources/sort.c b/sources/sort.c index 69de71e7..591be52f 100644 --- a/sources/sort.c +++ b/sources/sort.c @@ -872,7 +872,6 @@ LONG EndSort(PHEAD WORD *buffer, int par) #endif } } - AR.Stage4Name = 0; #ifdef WITHPTHREADS if ( AS.MasterSort && AC.ThreadSortFileSynch ) { if ( S->file.handle >= 0 ) { @@ -1045,7 +1044,7 @@ LONG EndSort(PHEAD WORD *buffer, int par) /* #] EndSort : - #[ PutIn : LONG PutIn(handle,position,buffer,take,npat) + #[ PutIn : LONG PutIn(handle,position,buffer,take,size,npat) */ /** * Reads a new patch from position in file handle. @@ -1057,19 +1056,22 @@ LONG EndSort(PHEAD WORD *buffer, int par) * @param position The position from which to read * @param buffer The buffer into which to read * @param take The unused tail should be moved before the buffer + * @param size The size of the input patch cache in bytes * @param npat The number of the patch. Is needed if the information * was compressed with gzip, because each patch has its * own independent gzip encoding. */ -LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, int npat) +LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, LONG size, int npat) { LONG i, RetCode; WORD *from, *to; #ifndef WITHZLIB - DUMMYUSE(npat); + GETIDENTITY + POSITION remaining; + LONG readsize = size; #endif - from = buffer + ( file->POsize * sizeof(UBYTE) )/sizeof(WORD); + from = buffer + size/sizeof(WORD); i = from - *take; if ( i*((LONG)(sizeof(WORD))) > AM.MaxTer ) { /* INTERNAL_ERROR_EXCL_START */ @@ -1084,28 +1086,33 @@ LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, int *take = to; #ifdef WITHZLIB if ( ( RetCode = FillInputGZIP(file,position,(UBYTE *)buffer - ,file->POsize,npat) ) < 0 ) { + ,size,npat) ) < 0 ) { /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>PutIn: We have RetCode = %x while reading %x bytes", - RetCode,file->POsize); + RetCode,size); MUNLOCK(ErrorMessageLock); Terminate(-1); /* INTERNAL_ERROR_EXCL_STOP */ } #else + if ( ISGEPOS(*position,AT.SS->fPatchesStop[npat]) ) return(0); + if ( !ISGEPOSINC(AT.SS->fPatchesStop[npat],*position,readsize) ) { + DIFPOS(remaining,AT.SS->fPatchesStop[npat],*position); + readsize = (LONG)BASEPOSITION(remaining); + } #ifdef ALLLOCK LOCK(file->pthreadslock); #endif SeekFile(file->handle,position,SEEK_SET); - if ( ( RetCode = ReadFile(file->handle,(UBYTE *)buffer,file->POsize) ) < 0 ) { + if ( ( RetCode = ReadFile(file->handle,(UBYTE *)buffer,readsize) ) < 0 ) { #ifdef ALLLOCK UNLOCK(file->pthreadslock); #endif /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>PutIn: We have RetCode = %x while reading %x bytes", - RetCode,file->POsize); + RetCode,readsize); MUNLOCK(ErrorMessageLock); Terminate(-1); /* INTERNAL_ERROR_EXCL_STOP */ @@ -3525,21 +3532,21 @@ int MergePatches(WORD par) WORD *p, lpat, mpat, level, l1, l2, r1, r2, r3, c; WORD *m2, *m3, r31, r33, ki, *rr; UWORD *coef; - POSITION position; + POSITION position, position2; FILEHANDLE *fin, *fout; + LONG cachesize = 0; int fhandle; /* UBYTE *s; */ #ifdef WITHZLIB - POSITION position2; int oldgzipCompress = AR.gzipCompress; if ( par == 2 ) { AR.gzipCompress = 0; } #endif fin = &S->file; - fout = &(AR.FoStage4[0]); + fout = &(S->FoStage4); NewMerge: coef = AN.SoScratC; poin = S->poina; poin2 = S->poin2a; @@ -3598,14 +3605,8 @@ int MergePatches(WORD par) else if ( par == 0 && S->stage4 > 0 ) { /* We will have to do our job more than once. - Input is from S->file and output will go to AR.FoStage4. + Input is from S->file and output will go to S->FoStage4. The file corresponding to this last one must be made now. -*/ - AR.Stage4Name ^= 1; -/* - s = (UBYTE *)(fout->name); while ( *s ) s++; - if ( AR.Stage4Name ) s[-1] += 1; - else s[-1] -= 1; */ S->iPatches = S->fPatches; S->fPatches = S->inPatches; @@ -3616,14 +3617,6 @@ int MergePatches(WORD par) m1 = S->fpincompressed; S->fpincompressed = S->fpcompressed; S->fpcompressed = m1; - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->iPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } #endif S->stage4 = 0; goto FileMake; @@ -3642,14 +3635,6 @@ int MergePatches(WORD par) m1 = S->fpincompressed; S->fpincompressed = S->fpcompressed; S->fpcompressed = m1; - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->fPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } #endif } fout = AR.outfile; @@ -3663,24 +3648,37 @@ int MergePatches(WORD par) } else { /* Load the patches */ S->lPatch = (S->inNum); + for ( i = 0; i < S->lPatch; i++ ) { + S->fPatchesStop[i] = S->iPatches[i+1]; +#ifdef GZIPDEBUG + MLOCK(ErrorMessageLock); + MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); + MUNLOCK(ErrorMessageLock); +#endif + } #ifdef WITHMPI - if ( S->lPatch > 1 || ( (PF.exprtodo <0) && (fout == AR.outfile || fout == AR.hidefile ) ) ) { + if ( S->lPatch > 0 && ( S->lPatch > 1 || ( (PF.exprtodo <0) && (fout == AR.outfile || fout == AR.hidefile ) ) ) ) { #else if ( S->lPatch > 1 ) { #endif #ifdef WITHZLIB SetupAllInputGZIP(S); #endif + // We use the combined large+small buffer for caching the patches. + // Divide it by the current number of patches, rather than by MaxFPatches, + // and make sure that we retain the usual per-cache headroom. + cachesize = ((S->LargeSize+S->SmallEsize)/S->lPatch-COMPINC) + *sizeof(WORD)-2*AM.MaxTer; p = S->lBuffer; for ( i = 0; i < S->lPatch; i++ ) { p = (WORD *)(((UBYTE *)p)+2*AM.MaxTer+COMPINC*sizeof(WORD)); S->Patches[i] = p; - p = (WORD *)(((UBYTE *)p) + fin->POsize); + p = (WORD *)(((UBYTE *)p) + cachesize); S->pStop[i] = m2 = p; #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,i); + PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,cachesize,i); #else - ADDPOS(S->iPatches[i],PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,i)); + ADDPOS(S->iPatches[i],PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,cachesize,i)); #endif } } @@ -3827,9 +3825,20 @@ int MergePatches(WORD par) ADDPOS(S->SizeInFile[par],1); #else /* old code */ + POSITION remaining; + LONG readsize; + position2 = S->iPatches[0]; SeekFile(fin->handle,&(S->iPatches[0]),SEEK_SET); /* needed for stage4 */ - while ( ( length = ReadFile(fin->handle, - (UBYTE *)(S->sBuffer),S->SmallEsize*sizeof(WORD)) ) > 0 ) { + length = 0; + while ( !ISGEPOS(position2,S->fPatchesStop[0]) ) { + readsize = S->SmallEsize*sizeof(WORD); + if ( !ISGEPOSINC(S->fPatchesStop[0],position2,readsize) ) { + DIFPOS(remaining,S->fPatchesStop[0],position2); + readsize = (LONG)BASEPOSITION(remaining); + } + length = ReadFile(fin->handle,(UBYTE *)(S->sBuffer),readsize); + if ( length <= 0 ) break; + ADDPOS(position2,length); if ( WriteFile(fout->handle,(UBYTE *)(S->sBuffer),length) != length ) goto PatwCall; ADDPOS(position,length); @@ -4047,10 +4056,10 @@ int MergePatches(WORD par) if ( !par && (poin[ul] + im + COMPINC) >= S->pStop[ki] && im > 0 ) { #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[ul]),ki); + PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[ul]),cachesize,ki); #else ADDPOS(S->iPatches[ki],PutIn(fin,&(S->iPatches[ki]), - S->Patches[ki],&(poin[ul]),ki)); + S->Patches[ki],&(poin[ul]),cachesize,ki)); #endif poin2[ul] = poin[ul] + im; } @@ -4116,10 +4125,10 @@ int MergePatches(WORD par) if ( !par && ( (poin[k] + im + COMPINC) >= S->pStop[ki] ) && im > 0 ) { #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[k]),ki); + PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[k]),cachesize,ki); #else ADDPOS(S->iPatches[ki],PutIn(fin,&(S->iPatches[ki]), - S->Patches[ki],&(poin[k]),ki)); + S->Patches[ki],&(poin[k]),cachesize,ki)); #endif poin2[k] = poin[k] + im; } @@ -4208,27 +4217,9 @@ int MergePatches(WORD par) /* INTERNAL_ERROR_EXCL_STOP */ } *rr = 0; -#ifdef WITHZLIB - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->iPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } -#endif goto ConMer; } else { -/* - if ( fin == &(AR.FoStage4[0]) ) { - s = (UBYTE *)(fin->name); while ( *s ) s++; - if ( AR.Stage4Name == 1 ) s[-1] -= 1; - else s[-1] += 1; - } -*/ -/* TruncateFile(fin->handle); */ UpdateMaxSize(); #ifdef WITHZLIB ClearSortGZIP(fin); @@ -4240,13 +4231,6 @@ int MergePatches(WORD par) MesPrint("%w MergePatches removed in file %s",fin->name); MUNLOCK(ErrorMessageLock); #endif -/* - if ( fin == &(AR.FoStage4[0]) ) { - s = (UBYTE *)(fin->name); while ( *s ) s++; - if ( AR.Stage4Name == 1 ) s[-1] += 1; - else s[-1] -= 1; - } -*/ fin->handle = -1; { FILEHANDLE *ff = fin; fin = fout; fout = ff; } PUTZERO(S->SizeInFile[0]); @@ -4440,35 +4424,26 @@ void StageSort(FILEHANDLE *fout) SORTING *S = AT.SS; if ( S->fPatchN >= S->MaxFpatches ) { POSITION position; - if ( S != AT.S0 ) { -/* - There are no proper provisions for stage 4 or higher sorts - for function arguments and $ variables. The reason: - The current code maps out the patches, based on the size of - the buffers in the FoStage4 structs, while they are used - inside the S->file struct that may have far smaller buffers. - By itself that might still be repairable, but it goes completely - wrong when during the sort polyRatFuns have to be added and they - would go into stage4 (very rare but possible). - The only really correct solution would be to put FoStage4 structs - in all sort levels. Messy. (JV 8-oct-2018). -*/ - MLOCK(ErrorMessageLock); - MesPrint("Currently Stage 4 sorts are not allowed for function arguments or $ variables."); - MesPrint("Please increase correspondingsorting parameters (sub-) in the setup."); - MUNLOCK(ErrorMessageLock); - Terminate(-1); - } PUTZERO(position); - MLOCK(ErrorMessageLock); + if ( S == AT.S0 ) { + MLOCK(ErrorMessageLock); #ifdef WITHPTHREADS - MesPrint("StageSort in thread %d",identity); + MesPrint("StageSort in thread %d",identity); #elif defined(WITHMPI) - MesPrint("StageSort in process %d",PF.me); + MesPrint("StageSort in process %d",PF.me); #else - MesPrint("StageSort"); + MesPrint("StageSort"); #endif - MUNLOCK(ErrorMessageLock); + MUNLOCK(ErrorMessageLock); + } + else { + if ( AT.SubSortStage4Warning == 0 ) { + AT.SubSortStage4Warning = 1; + MLOCK(ErrorMessageLock); + Warning("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes"); + MUNLOCK(ErrorMessageLock); + } + } SeekFile(fout->handle,&position,SEEK_END); /* No extra compression data has to be written. @@ -4492,25 +4467,16 @@ void StageSort(FILEHANDLE *fout) S->fPatches[0] = fout->filesize; S->fPatchN = 0; - if ( AR.FoStage4[0].PObuffer == 0 ) { - AR.FoStage4[0].PObuffer = (WORD *)Malloc1(AR.FoStage4[0].POsize*sizeof(WORD) - ,"Stage 4 buffer"); - AR.FoStage4[0].POfill = AR.FoStage4[0].PObuffer; - AR.FoStage4[0].POstop = AR.FoStage4[0].PObuffer - + AR.FoStage4[0].POsize/sizeof(WORD); -#ifdef WITHPTHREADS - AR.FoStage4[0].pthreadslock = dummylock; -#endif - } - if ( AR.FoStage4[1].PObuffer == 0 ) { - AR.FoStage4[1].PObuffer = (WORD *)Malloc1(AR.FoStage4[1].POsize*sizeof(WORD) - ,"Stage 4 buffer"); - AR.FoStage4[1].POfill = AR.FoStage4[1].PObuffer; - AR.FoStage4[1].POstop = AR.FoStage4[1].PObuffer - + AR.FoStage4[1].POsize/sizeof(WORD); -#ifdef WITHPTHREADS - AR.FoStage4[1].pthreadslock = dummylock; -#endif + { + FILEHANDLE *stage4 = &(S->FoStage4); + if ( stage4->PObuffer == 0 ) { + size_t namesize = strlen(S->file.name) + 4; + stage4->PObuffer = (WORD *)Malloc1(stage4->POsize + namesize, "Stage 4 buffer"); + stage4->POfill = stage4->POfull = stage4->PObuffer; + stage4->POstop = stage4->PObuffer + stage4->POsize/sizeof(WORD); + stage4->name = (char *)((UBYTE *)stage4->PObuffer + stage4->POsize); + snprintf(stage4->name,namesize,"%s.4a",S->file.name); + } } S->stage4 = 1; } @@ -4644,6 +4610,18 @@ void CleanUpSort(int num) for ( i = minnum; i < AN.NumFunSorts; i++ ) { S = AN.FunSorts[i]; if ( S ) { + if ( S->FoStage4.handle >= 0 ) { + UpdateMaxSize(); +#ifdef WITHZLIB + ClearSortGZIP(&(S->FoStage4)); +#endif + CloseFile(S->FoStage4.handle); + remove(S->FoStage4.name); + S->FoStage4.handle = -1; + } + if ( S->FoStage4.PObuffer ) { + M_free(S->FoStage4.PObuffer, "CleanUpSort: Stage 4 buffer"); + } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ UpdateMaxSize(); @@ -4684,6 +4662,15 @@ void CleanUpSort(int num) if ( num == 0 ) { S = AN.FunSorts[0]; if ( S ) { + if ( S->FoStage4.handle >= 0 ) { + UpdateMaxSize(); +#ifdef WITHZLIB + ClearSortGZIP(&(S->FoStage4)); +#endif + CloseFile(S->FoStage4.handle); + remove(S->FoStage4.name); + S->FoStage4.handle = -1; + } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ UpdateMaxSize(); @@ -4702,22 +4689,6 @@ void CleanUpSort(int num) } } } - for ( i = 0; i < 2; i++ ) { - if ( AR.FoStage4[i].handle >= 0 ) { - UpdateMaxSize(); -#ifdef WITHZLIB - ClearSortGZIP(&(AR.FoStage4[i])); -#endif - CloseFile(AR.FoStage4[i].handle); - remove(AR.FoStage4[i].name); - AR.FoStage4[i].handle = -1; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w CleanUpSort removed stage4 file %s",AR.FoStage4[i].name); - MUNLOCK(ErrorMessageLock); -#endif - } - } } /* diff --git a/sources/startup.c b/sources/startup.c index 126b7d3b..96fdc6bc 100644 --- a/sources/startup.c +++ b/sources/startup.c @@ -850,7 +850,7 @@ classic:; *t = 0; /* - Now we should assign a name to the main sort file and the two stage 4 files. + Now we should assign a name to the main sort file. */ AM.S0->file.name = (char *)Malloc1(sizeof(char)*(i+DEFAULTFNAMELENGTH),"name for temporary files"); s = (UBYTE *)AM.S0->file.name; @@ -877,27 +877,13 @@ classic:; } #endif /* - With the stage4 and scratch file names we have to be a bit more careful. + With the scratch file names we have to be a bit more careful. They are to be allocated after the threads are initialized when there are threads of course. */ if ( par == 0 ) { - s = (UBYTE *)((void *)(FG.fname2)); i = 0; - while ( *s ) { s++; i++; } - /* +1 for null terminator */ - s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for stage4 file a"); - AR.FoStage4[1].name = (char *)s; - t = (UBYTE *)FG.fname2; - while ( *t ) *s++ = *t++; - s[-2] = '4'; s[-1] = 'a'; *s = 0; s = (UBYTE *)((void *)(FG.fname)); i = 0; while ( *s ) { s++; i++; } - /* +1 for null terminator */ - s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for stage4 file b"); - AR.FoStage4[0].name = (char *)s; - t = (UBYTE *)FG.fname; - while ( *t ) *s++ = *t++; - s[-2] = '4'; s[-1] = 'b'; *s = 0; for ( j = 0; j < 3; j++ ) { /* +1 for null terminator */ s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for scratch file"); @@ -909,23 +895,8 @@ classic:; } #ifdef WITHPTHREADS else if ( par == 2 ) { - size_t tname; - s = (UBYTE *)((void *)(FG.fname2)); i = 0; - while ( *s ) { s++; i++; } - /* +1 for null terminator, +10 for 32bit int, +1 for "." */ - tname = sizeof(char)*(i+12); - s = (UBYTE *)Malloc1(tname,"name for stage4 file a"); - snprintf((char *)s,tname,"%s.%d",FG.fname2,AT.identity); - s[i-2] = '4'; s[i-1] = 'a'; - AR.FoStage4[1].name = (char *)s; s = (UBYTE *)((void *)(FG.fname)); i = 0; while ( *s ) { s++; i++; } - /* +1 for null terminator, +10 for 32bit int, +1 for "." */ - tname = sizeof(char)*(i+12); - s = (UBYTE *)Malloc1(tname,"name for stage4 file b"); - snprintf((char *)s,tname,"%s.%d",FG.fname,AT.identity); - s[i-2] = '4'; s[i-1] = 'b'; - AR.FoStage4[0].name = (char *)s; if ( AT.identity == 0 ) { for ( j = 0; j < 3; j++ ) { /* +1 for null terminator */ @@ -1496,8 +1467,6 @@ void IniVars(void) AM.gOutNumberType = RATIONALMODE; #ifdef WITHZLIB AR.gzipCompress = GZIPDEFAULT; - AR.FoStage4[0].ziobuffer = 0; - AR.FoStage4[1].ziobuffer = 0; #ifdef WITHZSTD /* Zstd compression is on by default, if we have compiled with it */ ZWRAP_useZSTDcompression(1); diff --git a/sources/structs.h b/sources/structs.h index 707e3e0f..d92fd15c 100644 --- a/sources/structs.h +++ b/sources/structs.h @@ -1113,6 +1113,7 @@ typedef struct PaRtI { typedef struct sOrT { FILEHANDLE file; /* The own sort file */ + FILEHANDLE FoStage4; /* Temporary file for stage 4 and higher */ POSITION SizeInFile[3]; /* Sizes in the various files */ POSITION OldPosIn; /* Sort file fill positions */ POSITION OldPosOut; @@ -1990,7 +1991,6 @@ struct S_const { struct R_const { FILEDATA StoreData; /* (O) */ FILEHANDLE Fscr[3]; /* (R) Dollars etc play with it too */ - FILEHANDLE FoStage4[2]; /* (R) In Sort. Stage 4. */ POSITION DefPosition; /* (R) Deferred position of keep brackets. */ FILEHANDLE *infile; /* (R) Points alternatingly to Fscr[0] or Fscr[1] */ FILEHANDLE *outfile; /* (R) Points alternatingly to Fscr[1] or Fscr[0] */ @@ -2033,7 +2033,6 @@ struct R_const { WORD DeferFlag; /* (R) For deferred brackets */ WORD TePos; /* (R) */ WORD sLevel; /* (R) Sorting level */ - WORD Stage4Name; /* (R) Sorting only */ WORD GetOneFile; /* (R) Getting from hide or regular */ WORD PolyFun; /* (C) Number of the PolyFun function */ WORD PolyFunInv; /* (C) Number of the Inverse of the PolyFun function */ @@ -2152,6 +2151,7 @@ struct T_const { int numpoly; int LeaveNegative; int TrimPower; /* Indicates trimming in polyratfun expansion */ + int SubSortStage4Warning; /* Has the sub-sort stage 4 warning been printed? */ WORD small_power_maxx; /* size of the cache for small powers */ WORD small_power_maxn; /* size of the cache for small powers */ WORD dummysubexp[SUBEXPSIZE+4]; /* () used in normal.c */ diff --git a/sources/threads.c b/sources/threads.c index 9dc5760c..64e21ee8 100644 --- a/sources/threads.c +++ b/sources/threads.c @@ -402,7 +402,7 @@ ALLPRIVATES *InitializeOneThread(int identity) { WORD *t, *ScratchBuf; int i, j, bsize, *bp; - LONG ScratchSize[3], IOsize; + LONG ScratchSize[3]; ALLPRIVATES *B; UBYTE *s; @@ -563,18 +563,6 @@ ALLPRIVATES *InitializeOneThread(int identity) AR.Fscr[0].handle = -1; AR.Fscr[1].handle = -1; AR.Fscr[2].handle = -1; - AR.FoStage4[0].handle = -1; - AR.FoStage4[1].handle = -1; - IOsize = AM.S0->file.POsize; -#ifdef WITHZLIB - AR.FoStage4[0].ziosize = IOsize; - AR.FoStage4[1].ziosize = IOsize; - AR.FoStage4[0].ziobuffer = 0; - AR.FoStage4[1].ziobuffer = 0; -#endif - AR.FoStage4[0].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.FoStage4[1].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.hidefile = &(AR.Fscr[2]); AR.StoreData.Handle = -1; AR.SortType = AC.SortType; @@ -637,7 +625,6 @@ ALLPRIVATES *InitializeOneThread(int identity) /* Still to do: the SS stuff. the Fscr[3] - the FoStage4[2] */ if ( AT.WorkSpace == 0 || AT.Nest == 0 || @@ -761,8 +748,7 @@ ALLPRIVATES *InitializeOneThread(int identity) ,AM.S0->SmallEsize*sizeof(WORD)/numberofworkers ,AM.S0->TermsInSmall ,AM.S0->MaxPatches -/* ,AM.S0->MaxPatches/numberofworkers */ - ,AM.S0->MaxFpatches/numberofworkers + ,MaX(AM.S0->MaxFpatches/numberofworkers, MINFPATCHES) ,AM.S0->file.POsize ,0); } @@ -1076,7 +1062,6 @@ int LoadOneThread(int from, int identity, THREADBUCKET *thr, int par) AR.DeferFlag = AR0.DeferFlag; AR.TePos = 0; AR.sLevel = AR0.sLevel; - AR.Stage4Name = AR0.Stage4Name; AR.GetOneFile = AR0.GetOneFile; AR.PolyFun = AR0.PolyFun; AR.PolyFunInv = AR0.PolyFunInv; @@ -1320,6 +1305,7 @@ void *RunThread(void *dummy) AR.PolyFunExp = AB[0]->R.PolyFunExp; AR.PolyFunVar = AB[0]->R.PolyFunVar; AR.PolyFunPow = AB[0]->R.PolyFunPow; + AT.SubSortStage4Warning = 0; /* Now fire up the sort buffer. */ @@ -1608,6 +1594,7 @@ bucketstolen:; AR.CurExpr = i; AR.SortType = AC.SortType; AR.expchanged = 0; + AT.SubSortStage4Warning = 0; if ( ( e->vflags & ISFACTORIZED ) != 0 ) { AR.BracketOn = 1; AT.BrackBuf = AM.BracketFactors; @@ -4773,4 +4760,4 @@ extern void optimize_expression_given_Horner_threaded(void) { #] optimize_expression_given_Horner_threaded : */ -#endif \ No newline at end of file +#endif diff --git a/sources/tools.c b/sources/tools.c index 65d3df53..dd2eaf0a 100644 --- a/sources/tools.c +++ b/sources/tools.c @@ -966,8 +966,6 @@ void StartFiles(void) AR.Fscr[0].handle = -1; AR.Fscr[1].handle = -1; AR.Fscr[2].handle = -1; - AR.FoStage4[0].handle = -1; - AR.FoStage4[1].handle = -1; AR.infile = &(AR.Fscr[0]); AR.outfile = &(AR.Fscr[1]); AR.hidefile = &(AR.Fscr[2]); @@ -1614,8 +1612,9 @@ int CloseChannel(char *name) void UpdateMaxSize(void) { POSITION position, sumsize; - int i; + int i, k; FILEHANDLE *scr; + SORTING *sort; #ifdef WITHMPI /* Currently, it works only on the master. The sort files on the slaves * are ignored. (TU 11 Oct 2011) */ @@ -1649,29 +1648,30 @@ void UpdateMaxSize(void) ALLPRIVATES *B; for ( j = 0; j < AM.totalnumberofthreads; j++ ) { B = AB[j]; - if ( AT.SS && AT.SS->file.handle >= 0 ) { - position = AT.SS->file.filesize; -/* -MLOCK(ErrorMessageLock); -MesPrint("%d: %10p",j,&(AT.SS->file.filesize)); -MUNLOCK(ErrorMessageLock); -*/ - ADD2POS(sumsize,position); - } - if ( AR.FoStage4[0].handle >= 0 ) { - position = AR.FoStage4[0].filesize; - ADD2POS(sumsize,position); + for ( k = 0; k < AN.NumFunSorts; k++ ) { + sort = AN.FunSorts[k]; + if ( sort && sort->file.handle >= 0 ) { + position = sort->file.filesize; + ADD2POS(sumsize,position); + } + if ( sort && sort->FoStage4.handle >= 0 ) { + position = sort->FoStage4.filesize; + ADD2POS(sumsize,position); + } } } } #else - if ( AT.SS && AT.SS->file.handle >= 0 ) { - position = AT.SS->file.filesize; - ADD2POS(sumsize,position); - } - if ( AR.FoStage4[0].handle >= 0 ) { - position = AR.FoStage4[0].filesize; - ADD2POS(sumsize,position); + for ( k = 0; k < AN.NumFunSorts; k++ ) { + sort = AN.FunSorts[k]; + if ( sort && sort->file.handle >= 0 ) { + position = sort->file.filesize; + ADD2POS(sumsize,position); + } + if ( sort && sort->FoStage4.handle >= 0 ) { + position = sort->FoStage4.filesize; + ADD2POS(sumsize,position); + } } #endif /*