From 9f0cf1b389b6be5b8af413d6c6fa23570e1e6991 Mon Sep 17 00:00:00 2001 From: Josh Davies Date: Tue, 15 Sep 2026 22:39:21 +0100 Subject: [PATCH 1/3] fix: make stage4 files sort-private, enable sub-buffer stage4 sorts For a long time, Stage4 sorting in the sub-buffers (function arguments, dollar variables, term environments) has been forbidden due to crashes. Resolve this by moving the Stage4 file handles into the sorting struct; every sort has its own files. File-to-file based sorting on a per-term basis will be horrible for performance so print a Warning for the user. But FORM will not simply terminate during a potentially long-running program, when the going gets tough. --- check/fixes.frm | 159 +++++++++++++++++++++++++++++++++++++++-- doc/manual/sorting.tex | 8 +-- sources/checkpoint.c | 3 - sources/proces.c | 1 + sources/setfile.c | 27 ++++--- sources/sort.c | 145 ++++++++++++++++--------------------- sources/startup.c | 35 +-------- sources/structs.h | 5 +- sources/threads.c | 20 ++---- sources/tools.c | 48 +++++++------ 10 files changed, 271 insertions(+), 180 deletions(-) diff --git a/check/fixes.frm b/check/fixes.frm index 96783c8b1..f9205bc57 100644 --- a/check/fixes.frm +++ b/check/fixes.frm @@ -2162,13 +2162,160 @@ Print; .end # Only for 64-bit systems. Otherwise "Output term too large". #require wordsize == 4 -# For now it fails because -# "Currently Stage 4 sorts are not allowed for function arguments or $ variables." -assert runtime_error? -#assert succeeded? -#assert result("test1") =~ expr("0") -#assert result("test2") =~ expr("g(0)") +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("test1") =~ expr("0") +assert result("test2") =~ expr("g(0)") *--#] Issue211 : +*--#[ Issue215_1 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol j,x; + +#define BLOWUP "1000" + +Local argument = f(x) + f(2*x); +Argument f; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("argument") =~ expr("3003/2") +*--#] Issue215_1 : +*--#[ Issue215_2 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +Symbol j,x; + +#define BLOWUP "1000" + +#$dol = x; +Local dollar = 1; +Inside $dol; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndInside; +Multiply $dol; +Identify x = 1; +Print; +ModuleOption local $dol; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("dollar") =~ expr("1001/2") +*--#] Issue215_2 : +*--#[ Issue215_3 : +#- + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +Symbol j,x; + +#define BLOWUP "1000" + +Local term = x; +Term; + Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP'); +EndTerm; +Identify x = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("term") =~ expr("1001/2") +*--#] Issue215_3 : +*--#[ Issue215_4 : +#- +#: TermsInSmall 16 +#: LargePatches 4 +#: FilePatches 4 + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol i,j,x; + +#define N "257" +#define BLOWUP "257" + +Local nested = {`N'*(`N'+1)/2} + #do i = 1,`N' + - f(x^`i') + #enddo + ; + +Argument f; + Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x^i? = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("nested") =~ expr("0") +*--#] Issue215_4 : +*--#[ Issue215_4b : +#- +#: TermsInSmall 16 +#: LargePatches 4 +#: FilePatches 4 + +#: SubTermsInSmall 16 +#: SubLargePatches 4 +#: SubFilePatches 4 + +Off statistics; +Off threadstats; + +CFunction f; +Symbol i,j,x; + +* Enough terms to have a stagesort in each thread of a -w5 test +#define N "{5*257}" +#define BLOWUP "257" + +Local nested = {`N'*(`N'+1)/2} + #do i = 1,`N' + - f(x^`i') + #enddo + ; +* Sort to distribute terms around workers +.sort + +Argument f; + Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP'); +EndArgument; +Identify f(x?) = x; +Identify x^i? = 1; +Print; +.end +assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes") +assert result("nested") =~ expr("0") +*--#] Issue215_4b : *--#[ Issue222 : * accessing #factdollar factors causes program termination Symbol x; diff --git a/doc/manual/sorting.tex b/doc/manual/sorting.tex index ed5146b67..adfa26a18 100644 --- a/doc/manual/sorting.tex +++ b/doc/manual/sorting.tex @@ -254,10 +254,10 @@ \chapter{Sorting and statistics} the chapter on the setup \ref{setup}. This chapter gives also all default values. -There is one restriction on the sorting of function arguments and -\$-expressions: They are not allowed to go into the stage4 sorting. Any -such attempt will result in an error message and the suggestion to raise -the size of the buffers for this type of sorting. +Sorting these `sub-buffers' may also run into a stage 4 sort when necessary. +This usually implies that performance will be severely affected, hence the +first time this happens during a program a warning will be printed, with the +suggestion to increase the sizes of the sub-buffers. When \FORM\ is running in parallel mode (either \TFORM\ or \ParFORM) each worker will need its own buffers. In \ParFORM\ in which the processors each control diff --git a/sources/checkpoint.c b/sources/checkpoint.c index 343170d69..8216e77f6 100644 --- a/sources/checkpoint.c +++ b/sources/checkpoint.c @@ -1184,7 +1184,6 @@ static void print_R() MesPrint("%d", AR.DeferFlag); MesPrint("%d", AR.TePos); MesPrint("%d", AR.sLevel); - MesPrint("%d", AR.Stage4Name); MesPrint("%d", AR.GetOneFile); MesPrint("%d", AR.PolyFun); MesPrint("%d", AR.PolyFunInv); @@ -2333,7 +2332,6 @@ int DoRecovery(int *moduletype) R_SET(AR.DeferFlag, WORD); R_SET(AR.TePos, WORD); R_SET(AR.sLevel, WORD); - R_SET(AR.Stage4Name, WORD); R_SET(AR.GetOneFile, WORD); R_SET(AR.PolyFun, WORD); R_SET(AR.PolyFunInv, WORD); @@ -2957,7 +2955,6 @@ static int DoSnapshot(int moduletype) S_WRITE_B(&AR.DeferFlag, sizeof(WORD)); S_WRITE_B(&AR.TePos, sizeof(WORD)); S_WRITE_B(&AR.sLevel, sizeof(WORD)); - S_WRITE_B(&AR.Stage4Name, sizeof(WORD)); S_WRITE_B(&AR.GetOneFile, sizeof(WORD)); S_WRITE_B(&AR.PolyFun, sizeof(WORD)); S_WRITE_B(&AR.PolyFunInv, sizeof(WORD)); diff --git a/sources/proces.c b/sources/proces.c index b0fc8f74c..b7b11edf5 100644 --- a/sources/proces.c +++ b/sources/proces.c @@ -205,6 +205,7 @@ int Processor(void) #endif AS.CollectOverFlag = 0; AR.expchanged = 0; + AT.SubSortStage4Warning = 0; if ( i == last ) LastExpression = 1; else LastExpression = 0; if ( e->inmem ) { diff --git a/sources/setfile.c b/sources/setfile.c index d5ecee9ed..eb98a29de 100644 --- a/sources/setfile.c +++ b/sources/setfile.c @@ -416,11 +416,8 @@ int AllocSetups(void) { SETUPPARAMETERS *sp; LONG LargeSize, SmallSize, SmallEsize, TermsInSmall, IOsize; - int MaxPatches, MaxFpatches, error = 0, i, size; + int MaxPatches, MaxFpatches, error = 0, i, j, size; UBYTE *s; -#ifndef WITHPTHREADS - int j; -#endif sp = GetSetupPar((UBYTE *)"threads"); if ( sp->value > 0 ) AM.totalnumberofthreads = sp->value+1; @@ -588,19 +585,16 @@ int AllocSetups(void) /* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */ #ifdef WITHZLIB AM.S0->file.ziosize = IOsize; + for ( j = 0; j < 2; j++ ) { + AM.S0->FoStage4[j].ziosize = IOsize; + } #ifndef WITHPTHREADS - AR.FoStage4[0].ziosize = IOsize; - AR.FoStage4[1].ziosize = IOsize; AT.S0 = AM.S0; #endif #else #ifndef WITHPTHREADS AT.S0 = AM.S0; #endif -#endif -#ifndef WITHPTHREADS - AR.FoStage4[0].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.FoStage4[1].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); #endif sp = GetSetupPar((UBYTE *)"subsmallsize"); AM.SSmallSize = sp->value; @@ -999,6 +993,19 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i sort->file.ziosize = IObuffersize*sizeof(WORD); sort->file.ziobuffer = 0; #endif + for ( j = 0; j < 2; j++ ) { + FILEHANDLE *stage4 = &(sort->FoStage4[j]); + memset(stage4,0,sizeof(*stage4)); + stage4->POsize = sort->file.POsize; + stage4->handle = -1; +#ifdef WITHPTHREADS + stage4->pthreadslock = dummylock; +#endif +#ifdef WITHZLIB + stage4->ziosize = sort->file.ziosize; + stage4->ziobuffer = 0; +#endif + } if ( AM.S0 != 0 ) { sort->file.name = (char *)(sort->file.PObuffer + IObuffersize); AllocSortFileName(sort); diff --git a/sources/sort.c b/sources/sort.c index 69de71e76..0ab0faa6a 100644 --- a/sources/sort.c +++ b/sources/sort.c @@ -444,6 +444,7 @@ int NewSort(PHEAD0) S->sTerms = 0; PUTZERO(S->file.POposition); S->stage4 = 0; + S->Stage4Name = 0; if ( AR.sLevel > AN.MaxFunSorts ) AN.MaxFunSorts = AR.sLevel; // Zero the SortVerbose counters: @@ -872,7 +873,7 @@ LONG EndSort(PHEAD WORD *buffer, int par) #endif } } - AR.Stage4Name = 0; + S->Stage4Name = 0; #ifdef WITHPTHREADS if ( AS.MasterSort && AC.ThreadSortFileSynch ) { if ( S->file.handle >= 0 ) { @@ -3539,7 +3540,7 @@ int MergePatches(WORD par) } #endif fin = &S->file; - fout = &(AR.FoStage4[0]); + fout = &(S->FoStage4[0]); NewMerge: coef = AN.SoScratC; poin = S->poina; poin2 = S->poin2a; @@ -3598,15 +3599,10 @@ int MergePatches(WORD par) else if ( par == 0 && S->stage4 > 0 ) { /* We will have to do our job more than once. - Input is from S->file and output will go to AR.FoStage4. + Input is from S->file and output will go to S->FoStage4. The file corresponding to this last one must be made now. */ - AR.Stage4Name ^= 1; -/* - s = (UBYTE *)(fout->name); while ( *s ) s++; - if ( AR.Stage4Name ) s[-1] += 1; - else s[-1] -= 1; -*/ + S->Stage4Name ^= 1; S->iPatches = S->fPatches; S->fPatches = S->inPatches; S->inPatches = S->iPatches; @@ -4221,14 +4217,6 @@ int MergePatches(WORD par) goto ConMer; } else { -/* - if ( fin == &(AR.FoStage4[0]) ) { - s = (UBYTE *)(fin->name); while ( *s ) s++; - if ( AR.Stage4Name == 1 ) s[-1] -= 1; - else s[-1] += 1; - } -*/ -/* TruncateFile(fin->handle); */ UpdateMaxSize(); #ifdef WITHZLIB ClearSortGZIP(fin); @@ -4240,13 +4228,6 @@ int MergePatches(WORD par) MesPrint("%w MergePatches removed in file %s",fin->name); MUNLOCK(ErrorMessageLock); #endif -/* - if ( fin == &(AR.FoStage4[0]) ) { - s = (UBYTE *)(fin->name); while ( *s ) s++; - if ( AR.Stage4Name == 1 ) s[-1] += 1; - else s[-1] -= 1; - } -*/ fin->handle = -1; { FILEHANDLE *ff = fin; fin = fout; fout = ff; } PUTZERO(S->SizeInFile[0]); @@ -4440,35 +4421,26 @@ void StageSort(FILEHANDLE *fout) SORTING *S = AT.SS; if ( S->fPatchN >= S->MaxFpatches ) { POSITION position; - if ( S != AT.S0 ) { -/* - There are no proper provisions for stage 4 or higher sorts - for function arguments and $ variables. The reason: - The current code maps out the patches, based on the size of - the buffers in the FoStage4 structs, while they are used - inside the S->file struct that may have far smaller buffers. - By itself that might still be repairable, but it goes completely - wrong when during the sort polyRatFuns have to be added and they - would go into stage4 (very rare but possible). - The only really correct solution would be to put FoStage4 structs - in all sort levels. Messy. (JV 8-oct-2018). -*/ - MLOCK(ErrorMessageLock); - MesPrint("Currently Stage 4 sorts are not allowed for function arguments or $ variables."); - MesPrint("Please increase correspondingsorting parameters (sub-) in the setup."); - MUNLOCK(ErrorMessageLock); - Terminate(-1); - } PUTZERO(position); - MLOCK(ErrorMessageLock); + if ( S == AT.S0 ) { + MLOCK(ErrorMessageLock); #ifdef WITHPTHREADS - MesPrint("StageSort in thread %d",identity); + MesPrint("StageSort in thread %d",identity); #elif defined(WITHMPI) - MesPrint("StageSort in process %d",PF.me); + MesPrint("StageSort in process %d",PF.me); #else - MesPrint("StageSort"); + MesPrint("StageSort"); #endif - MUNLOCK(ErrorMessageLock); + MUNLOCK(ErrorMessageLock); + } + else { + if ( AT.SubSortStage4Warning == 0 ) { + AT.SubSortStage4Warning = 1; + MLOCK(ErrorMessageLock); + Warning("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes"); + MUNLOCK(ErrorMessageLock); + } + } SeekFile(fout->handle,&position,SEEK_END); /* No extra compression data has to be written. @@ -4492,25 +4464,21 @@ void StageSort(FILEHANDLE *fout) S->fPatches[0] = fout->filesize; S->fPatchN = 0; - if ( AR.FoStage4[0].PObuffer == 0 ) { - AR.FoStage4[0].PObuffer = (WORD *)Malloc1(AR.FoStage4[0].POsize*sizeof(WORD) - ,"Stage 4 buffer"); - AR.FoStage4[0].POfill = AR.FoStage4[0].PObuffer; - AR.FoStage4[0].POstop = AR.FoStage4[0].PObuffer - + AR.FoStage4[0].POsize/sizeof(WORD); -#ifdef WITHPTHREADS - AR.FoStage4[0].pthreadslock = dummylock; -#endif - } - if ( AR.FoStage4[1].PObuffer == 0 ) { - AR.FoStage4[1].PObuffer = (WORD *)Malloc1(AR.FoStage4[1].POsize*sizeof(WORD) - ,"Stage 4 buffer"); - AR.FoStage4[1].POfill = AR.FoStage4[1].PObuffer; - AR.FoStage4[1].POstop = AR.FoStage4[1].PObuffer - + AR.FoStage4[1].POsize/sizeof(WORD); -#ifdef WITHPTHREADS - AR.FoStage4[1].pthreadslock = dummylock; -#endif + { + int i; + for ( i = 0; i < 2; i++ ) { + FILEHANDLE *stage4 = &(S->FoStage4[i]); + if ( stage4->PObuffer == 0 ) { + size_t namesize = strlen(S->file.name) + 4; + stage4->PObuffer = (WORD *)Malloc1(stage4->POsize + namesize, + "Stage 4 buffer"); + stage4->POfill = stage4->POfull = stage4->PObuffer; + stage4->POstop = stage4->PObuffer + + stage4->POsize/sizeof(WORD); + stage4->name = (char *)((UBYTE *)stage4->PObuffer + stage4->POsize); + snprintf(stage4->name,namesize,"%s.4%c",S->file.name,'a'+i); + } + } } S->stage4 = 1; } @@ -4632,7 +4600,7 @@ void CleanUpSort(int num) { GETIDENTITY SORTING *S; - int minnum = num, i; + int minnum = num, i, j; if ( AN.FunSorts ) { if ( num == -1 ) { if ( AN.MaxFunSorts > 3 ) { @@ -4644,6 +4612,20 @@ void CleanUpSort(int num) for ( i = minnum; i < AN.NumFunSorts; i++ ) { S = AN.FunSorts[i]; if ( S ) { + for ( j = 0; j < 2; j++ ) { + if ( S->FoStage4[j].handle >= 0 ) { + UpdateMaxSize(); +#ifdef WITHZLIB + ClearSortGZIP(&(S->FoStage4[j])); +#endif + CloseFile(S->FoStage4[j].handle); + remove(S->FoStage4[j].name); + S->FoStage4[j].handle = -1; + } + if ( S->FoStage4[j].PObuffer ) { + M_free(S->FoStage4[j].PObuffer, "CleanUpSort: Stage 4 buffer"); + } + } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ UpdateMaxSize(); @@ -4684,6 +4666,17 @@ void CleanUpSort(int num) if ( num == 0 ) { S = AN.FunSorts[0]; if ( S ) { + for ( j = 0; j < 2; j++ ) { + if ( S->FoStage4[j].handle >= 0 ) { + UpdateMaxSize(); +#ifdef WITHZLIB + ClearSortGZIP(&(S->FoStage4[j])); +#endif + CloseFile(S->FoStage4[j].handle); + remove(S->FoStage4[j].name); + S->FoStage4[j].handle = -1; + } + } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ UpdateMaxSize(); @@ -4702,22 +4695,6 @@ void CleanUpSort(int num) } } } - for ( i = 0; i < 2; i++ ) { - if ( AR.FoStage4[i].handle >= 0 ) { - UpdateMaxSize(); -#ifdef WITHZLIB - ClearSortGZIP(&(AR.FoStage4[i])); -#endif - CloseFile(AR.FoStage4[i].handle); - remove(AR.FoStage4[i].name); - AR.FoStage4[i].handle = -1; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w CleanUpSort removed stage4 file %s",AR.FoStage4[i].name); - MUNLOCK(ErrorMessageLock); -#endif - } - } } /* diff --git a/sources/startup.c b/sources/startup.c index 126b7d3b1..96fdc6bc3 100644 --- a/sources/startup.c +++ b/sources/startup.c @@ -850,7 +850,7 @@ classic:; *t = 0; /* - Now we should assign a name to the main sort file and the two stage 4 files. + Now we should assign a name to the main sort file. */ AM.S0->file.name = (char *)Malloc1(sizeof(char)*(i+DEFAULTFNAMELENGTH),"name for temporary files"); s = (UBYTE *)AM.S0->file.name; @@ -877,27 +877,13 @@ classic:; } #endif /* - With the stage4 and scratch file names we have to be a bit more careful. + With the scratch file names we have to be a bit more careful. They are to be allocated after the threads are initialized when there are threads of course. */ if ( par == 0 ) { - s = (UBYTE *)((void *)(FG.fname2)); i = 0; - while ( *s ) { s++; i++; } - /* +1 for null terminator */ - s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for stage4 file a"); - AR.FoStage4[1].name = (char *)s; - t = (UBYTE *)FG.fname2; - while ( *t ) *s++ = *t++; - s[-2] = '4'; s[-1] = 'a'; *s = 0; s = (UBYTE *)((void *)(FG.fname)); i = 0; while ( *s ) { s++; i++; } - /* +1 for null terminator */ - s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for stage4 file b"); - AR.FoStage4[0].name = (char *)s; - t = (UBYTE *)FG.fname; - while ( *t ) *s++ = *t++; - s[-2] = '4'; s[-1] = 'b'; *s = 0; for ( j = 0; j < 3; j++ ) { /* +1 for null terminator */ s = (UBYTE *)Malloc1(sizeof(char)*(i+1),"name for scratch file"); @@ -909,23 +895,8 @@ classic:; } #ifdef WITHPTHREADS else if ( par == 2 ) { - size_t tname; - s = (UBYTE *)((void *)(FG.fname2)); i = 0; - while ( *s ) { s++; i++; } - /* +1 for null terminator, +10 for 32bit int, +1 for "." */ - tname = sizeof(char)*(i+12); - s = (UBYTE *)Malloc1(tname,"name for stage4 file a"); - snprintf((char *)s,tname,"%s.%d",FG.fname2,AT.identity); - s[i-2] = '4'; s[i-1] = 'a'; - AR.FoStage4[1].name = (char *)s; s = (UBYTE *)((void *)(FG.fname)); i = 0; while ( *s ) { s++; i++; } - /* +1 for null terminator, +10 for 32bit int, +1 for "." */ - tname = sizeof(char)*(i+12); - s = (UBYTE *)Malloc1(tname,"name for stage4 file b"); - snprintf((char *)s,tname,"%s.%d",FG.fname,AT.identity); - s[i-2] = '4'; s[i-1] = 'b'; - AR.FoStage4[0].name = (char *)s; if ( AT.identity == 0 ) { for ( j = 0; j < 3; j++ ) { /* +1 for null terminator */ @@ -1496,8 +1467,6 @@ void IniVars(void) AM.gOutNumberType = RATIONALMODE; #ifdef WITHZLIB AR.gzipCompress = GZIPDEFAULT; - AR.FoStage4[0].ziobuffer = 0; - AR.FoStage4[1].ziobuffer = 0; #ifdef WITHZSTD /* Zstd compression is on by default, if we have compiled with it */ ZWRAP_useZSTDcompression(1); diff --git a/sources/structs.h b/sources/structs.h index 707e3e0f1..40a8e20d9 100644 --- a/sources/structs.h +++ b/sources/structs.h @@ -1113,6 +1113,7 @@ typedef struct PaRtI { typedef struct sOrT { FILEHANDLE file; /* The own sort file */ + FILEHANDLE FoStage4[2]; /* Temporary files for stage 4 and higher */ POSITION SizeInFile[3]; /* Sizes in the various files */ POSITION OldPosIn; /* Sort file fill positions */ POSITION OldPosOut; @@ -1179,6 +1180,7 @@ typedef struct sOrT { WORD fPatchN; /* Number of patches on file (output) */ WORD inNum; /* Number of patches on file (input) */ WORD stage4; /* Are we using stage4? */ + WORD Stage4Name; /* Selects the stage 4 temporary file name */ } SORTING; #ifdef WITHPTHREADS @@ -1990,7 +1992,6 @@ struct S_const { struct R_const { FILEDATA StoreData; /* (O) */ FILEHANDLE Fscr[3]; /* (R) Dollars etc play with it too */ - FILEHANDLE FoStage4[2]; /* (R) In Sort. Stage 4. */ POSITION DefPosition; /* (R) Deferred position of keep brackets. */ FILEHANDLE *infile; /* (R) Points alternatingly to Fscr[0] or Fscr[1] */ FILEHANDLE *outfile; /* (R) Points alternatingly to Fscr[1] or Fscr[0] */ @@ -2033,7 +2034,6 @@ struct R_const { WORD DeferFlag; /* (R) For deferred brackets */ WORD TePos; /* (R) */ WORD sLevel; /* (R) Sorting level */ - WORD Stage4Name; /* (R) Sorting only */ WORD GetOneFile; /* (R) Getting from hide or regular */ WORD PolyFun; /* (C) Number of the PolyFun function */ WORD PolyFunInv; /* (C) Number of the Inverse of the PolyFun function */ @@ -2152,6 +2152,7 @@ struct T_const { int numpoly; int LeaveNegative; int TrimPower; /* Indicates trimming in polyratfun expansion */ + int SubSortStage4Warning; /* Has the sub-sort stage 4 warning been printed? */ WORD small_power_maxx; /* size of the cache for small powers */ WORD small_power_maxn; /* size of the cache for small powers */ WORD dummysubexp[SUBEXPSIZE+4]; /* () used in normal.c */ diff --git a/sources/threads.c b/sources/threads.c index 9dc5760c0..2479b74cf 100644 --- a/sources/threads.c +++ b/sources/threads.c @@ -402,7 +402,7 @@ ALLPRIVATES *InitializeOneThread(int identity) { WORD *t, *ScratchBuf; int i, j, bsize, *bp; - LONG ScratchSize[3], IOsize; + LONG ScratchSize[3]; ALLPRIVATES *B; UBYTE *s; @@ -563,18 +563,6 @@ ALLPRIVATES *InitializeOneThread(int identity) AR.Fscr[0].handle = -1; AR.Fscr[1].handle = -1; AR.Fscr[2].handle = -1; - AR.FoStage4[0].handle = -1; - AR.FoStage4[1].handle = -1; - IOsize = AM.S0->file.POsize; -#ifdef WITHZLIB - AR.FoStage4[0].ziosize = IOsize; - AR.FoStage4[1].ziosize = IOsize; - AR.FoStage4[0].ziobuffer = 0; - AR.FoStage4[1].ziobuffer = 0; -#endif - AR.FoStage4[0].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.FoStage4[1].POsize = ((IOsize+sizeof(WORD)-1)/sizeof(WORD))*sizeof(WORD); - AR.hidefile = &(AR.Fscr[2]); AR.StoreData.Handle = -1; AR.SortType = AC.SortType; @@ -637,7 +625,6 @@ ALLPRIVATES *InitializeOneThread(int identity) /* Still to do: the SS stuff. the Fscr[3] - the FoStage4[2] */ if ( AT.WorkSpace == 0 || AT.Nest == 0 || @@ -1076,7 +1063,6 @@ int LoadOneThread(int from, int identity, THREADBUCKET *thr, int par) AR.DeferFlag = AR0.DeferFlag; AR.TePos = 0; AR.sLevel = AR0.sLevel; - AR.Stage4Name = AR0.Stage4Name; AR.GetOneFile = AR0.GetOneFile; AR.PolyFun = AR0.PolyFun; AR.PolyFunInv = AR0.PolyFunInv; @@ -1320,6 +1306,7 @@ void *RunThread(void *dummy) AR.PolyFunExp = AB[0]->R.PolyFunExp; AR.PolyFunVar = AB[0]->R.PolyFunVar; AR.PolyFunPow = AB[0]->R.PolyFunPow; + AT.SubSortStage4Warning = 0; /* Now fire up the sort buffer. */ @@ -1608,6 +1595,7 @@ bucketstolen:; AR.CurExpr = i; AR.SortType = AC.SortType; AR.expchanged = 0; + AT.SubSortStage4Warning = 0; if ( ( e->vflags & ISFACTORIZED ) != 0 ) { AR.BracketOn = 1; AT.BrackBuf = AM.BracketFactors; @@ -4773,4 +4761,4 @@ extern void optimize_expression_given_Horner_threaded(void) { #] optimize_expression_given_Horner_threaded : */ -#endif \ No newline at end of file +#endif diff --git a/sources/tools.c b/sources/tools.c index 65d3df532..3ee4798ce 100644 --- a/sources/tools.c +++ b/sources/tools.c @@ -966,8 +966,6 @@ void StartFiles(void) AR.Fscr[0].handle = -1; AR.Fscr[1].handle = -1; AR.Fscr[2].handle = -1; - AR.FoStage4[0].handle = -1; - AR.FoStage4[1].handle = -1; AR.infile = &(AR.Fscr[0]); AR.outfile = &(AR.Fscr[1]); AR.hidefile = &(AR.Fscr[2]); @@ -1614,8 +1612,9 @@ int CloseChannel(char *name) void UpdateMaxSize(void) { POSITION position, sumsize; - int i; + int i, k; FILEHANDLE *scr; + SORTING *sort; #ifdef WITHMPI /* Currently, it works only on the master. The sort files on the slaves * are ignored. (TU 11 Oct 2011) */ @@ -1649,29 +1648,34 @@ void UpdateMaxSize(void) ALLPRIVATES *B; for ( j = 0; j < AM.totalnumberofthreads; j++ ) { B = AB[j]; - if ( AT.SS && AT.SS->file.handle >= 0 ) { - position = AT.SS->file.filesize; -/* -MLOCK(ErrorMessageLock); -MesPrint("%d: %10p",j,&(AT.SS->file.filesize)); -MUNLOCK(ErrorMessageLock); -*/ - ADD2POS(sumsize,position); - } - if ( AR.FoStage4[0].handle >= 0 ) { - position = AR.FoStage4[0].filesize; - ADD2POS(sumsize,position); + for ( k = 0; k < AN.NumFunSorts; k++ ) { + sort = AN.FunSorts[k]; + if ( sort && sort->file.handle >= 0 ) { + position = sort->file.filesize; + ADD2POS(sumsize,position); + } + for ( i = 0; i < 2; i++ ) { + if ( sort && sort->FoStage4[i].handle >= 0 ) { + position = sort->FoStage4[i].filesize; + ADD2POS(sumsize,position); + } + } } } } #else - if ( AT.SS && AT.SS->file.handle >= 0 ) { - position = AT.SS->file.filesize; - ADD2POS(sumsize,position); - } - if ( AR.FoStage4[0].handle >= 0 ) { - position = AR.FoStage4[0].filesize; - ADD2POS(sumsize,position); + for ( k = 0; k < AN.NumFunSorts; k++ ) { + sort = AN.FunSorts[k]; + if ( sort && sort->file.handle >= 0 ) { + position = sort->file.filesize; + ADD2POS(sumsize,position); + } + for ( i = 0; i < 2; i++ ) { + if ( sort && sort->FoStage4[i].handle >= 0 ) { + position = sort->FoStage4[i].filesize; + ADD2POS(sumsize,position); + } + } } #endif /* From a893177ab5af1052c86b62d9969de73df305e40a Mon Sep 17 00:00:00 2001 From: Josh Davies Date: Tue, 15 Sep 2026 23:02:08 +0100 Subject: [PATCH 2/3] refactor: remove unused stage4 sort file handle --- sources/setfile.c | 25 +++++++++---------- sources/sort.c | 62 +++++++++++++++++++---------------------------- sources/structs.h | 3 +-- sources/tools.c | 16 +++++------- 4 files changed, 44 insertions(+), 62 deletions(-) diff --git a/sources/setfile.c b/sources/setfile.c index eb98a29de..1ef3b69e7 100644 --- a/sources/setfile.c +++ b/sources/setfile.c @@ -416,7 +416,10 @@ int AllocSetups(void) { SETUPPARAMETERS *sp; LONG LargeSize, SmallSize, SmallEsize, TermsInSmall, IOsize; - int MaxPatches, MaxFpatches, error = 0, i, j, size; + int MaxPatches, MaxFpatches, error = 0, i, size; +#ifndef WITHPTHREADS + int j; +#endif UBYTE *s; sp = GetSetupPar((UBYTE *)"threads"); if ( sp->value > 0 ) AM.totalnumberofthreads = sp->value+1; @@ -585,9 +588,7 @@ int AllocSetups(void) /* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */ #ifdef WITHZLIB AM.S0->file.ziosize = IOsize; - for ( j = 0; j < 2; j++ ) { - AM.S0->FoStage4[j].ziosize = IOsize; - } + AM.S0->FoStage4.ziosize = IOsize; #ifndef WITHPTHREADS AT.S0 = AM.S0; #endif @@ -993,19 +994,17 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i sort->file.ziosize = IObuffersize*sizeof(WORD); sort->file.ziobuffer = 0; #endif - for ( j = 0; j < 2; j++ ) { - FILEHANDLE *stage4 = &(sort->FoStage4[j]); - memset(stage4,0,sizeof(*stage4)); - stage4->POsize = sort->file.POsize; - stage4->handle = -1; + FILEHANDLE *stage4 = &(sort->FoStage4); + memset(stage4,0,sizeof(*stage4)); + stage4->POsize = sort->file.POsize; + stage4->handle = -1; #ifdef WITHPTHREADS - stage4->pthreadslock = dummylock; + stage4->pthreadslock = dummylock; #endif #ifdef WITHZLIB - stage4->ziosize = sort->file.ziosize; - stage4->ziobuffer = 0; + stage4->ziosize = sort->file.ziosize; + stage4->ziobuffer = 0; #endif - } if ( AM.S0 != 0 ) { sort->file.name = (char *)(sort->file.PObuffer + IObuffersize); AllocSortFileName(sort); diff --git a/sources/sort.c b/sources/sort.c index 0ab0faa6a..520008d76 100644 --- a/sources/sort.c +++ b/sources/sort.c @@ -444,7 +444,6 @@ int NewSort(PHEAD0) S->sTerms = 0; PUTZERO(S->file.POposition); S->stage4 = 0; - S->Stage4Name = 0; if ( AR.sLevel > AN.MaxFunSorts ) AN.MaxFunSorts = AR.sLevel; // Zero the SortVerbose counters: @@ -873,7 +872,6 @@ LONG EndSort(PHEAD WORD *buffer, int par) #endif } } - S->Stage4Name = 0; #ifdef WITHPTHREADS if ( AS.MasterSort && AC.ThreadSortFileSynch ) { if ( S->file.handle >= 0 ) { @@ -3540,7 +3538,7 @@ int MergePatches(WORD par) } #endif fin = &S->file; - fout = &(S->FoStage4[0]); + fout = &(S->FoStage4); NewMerge: coef = AN.SoScratC; poin = S->poina; poin2 = S->poin2a; @@ -3602,7 +3600,6 @@ int MergePatches(WORD par) Input is from S->file and output will go to S->FoStage4. The file corresponding to this last one must be made now. */ - S->Stage4Name ^= 1; S->iPatches = S->fPatches; S->fPatches = S->inPatches; S->inPatches = S->iPatches; @@ -4465,19 +4462,14 @@ void StageSort(FILEHANDLE *fout) S->fPatchN = 0; { - int i; - for ( i = 0; i < 2; i++ ) { - FILEHANDLE *stage4 = &(S->FoStage4[i]); - if ( stage4->PObuffer == 0 ) { - size_t namesize = strlen(S->file.name) + 4; - stage4->PObuffer = (WORD *)Malloc1(stage4->POsize + namesize, - "Stage 4 buffer"); - stage4->POfill = stage4->POfull = stage4->PObuffer; - stage4->POstop = stage4->PObuffer - + stage4->POsize/sizeof(WORD); - stage4->name = (char *)((UBYTE *)stage4->PObuffer + stage4->POsize); - snprintf(stage4->name,namesize,"%s.4%c",S->file.name,'a'+i); - } + FILEHANDLE *stage4 = &(S->FoStage4); + if ( stage4->PObuffer == 0 ) { + size_t namesize = strlen(S->file.name) + 4; + stage4->PObuffer = (WORD *)Malloc1(stage4->POsize + namesize, "Stage 4 buffer"); + stage4->POfill = stage4->POfull = stage4->PObuffer; + stage4->POstop = stage4->PObuffer + stage4->POsize/sizeof(WORD); + stage4->name = (char *)((UBYTE *)stage4->PObuffer + stage4->POsize); + snprintf(stage4->name,namesize,"%s.4a",S->file.name); } } S->stage4 = 1; @@ -4600,7 +4592,7 @@ void CleanUpSort(int num) { GETIDENTITY SORTING *S; - int minnum = num, i, j; + int minnum = num, i; if ( AN.FunSorts ) { if ( num == -1 ) { if ( AN.MaxFunSorts > 3 ) { @@ -4612,19 +4604,17 @@ void CleanUpSort(int num) for ( i = minnum; i < AN.NumFunSorts; i++ ) { S = AN.FunSorts[i]; if ( S ) { - for ( j = 0; j < 2; j++ ) { - if ( S->FoStage4[j].handle >= 0 ) { - UpdateMaxSize(); + if ( S->FoStage4.handle >= 0 ) { + UpdateMaxSize(); #ifdef WITHZLIB - ClearSortGZIP(&(S->FoStage4[j])); + ClearSortGZIP(&(S->FoStage4)); #endif - CloseFile(S->FoStage4[j].handle); - remove(S->FoStage4[j].name); - S->FoStage4[j].handle = -1; - } - if ( S->FoStage4[j].PObuffer ) { - M_free(S->FoStage4[j].PObuffer, "CleanUpSort: Stage 4 buffer"); - } + CloseFile(S->FoStage4.handle); + remove(S->FoStage4.name); + S->FoStage4.handle = -1; + } + if ( S->FoStage4.PObuffer ) { + M_free(S->FoStage4.PObuffer, "CleanUpSort: Stage 4 buffer"); } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ @@ -4666,16 +4656,14 @@ void CleanUpSort(int num) if ( num == 0 ) { S = AN.FunSorts[0]; if ( S ) { - for ( j = 0; j < 2; j++ ) { - if ( S->FoStage4[j].handle >= 0 ) { - UpdateMaxSize(); + if ( S->FoStage4.handle >= 0 ) { + UpdateMaxSize(); #ifdef WITHZLIB - ClearSortGZIP(&(S->FoStage4[j])); + ClearSortGZIP(&(S->FoStage4)); #endif - CloseFile(S->FoStage4[j].handle); - remove(S->FoStage4[j].name); - S->FoStage4[j].handle = -1; - } + CloseFile(S->FoStage4.handle); + remove(S->FoStage4.name); + S->FoStage4.handle = -1; } if ( S->file.handle >= 0 ) { /* TruncateFile(S->file.handle); */ diff --git a/sources/structs.h b/sources/structs.h index 40a8e20d9..d92fd15c0 100644 --- a/sources/structs.h +++ b/sources/structs.h @@ -1113,7 +1113,7 @@ typedef struct PaRtI { typedef struct sOrT { FILEHANDLE file; /* The own sort file */ - FILEHANDLE FoStage4[2]; /* Temporary files for stage 4 and higher */ + FILEHANDLE FoStage4; /* Temporary file for stage 4 and higher */ POSITION SizeInFile[3]; /* Sizes in the various files */ POSITION OldPosIn; /* Sort file fill positions */ POSITION OldPosOut; @@ -1180,7 +1180,6 @@ typedef struct sOrT { WORD fPatchN; /* Number of patches on file (output) */ WORD inNum; /* Number of patches on file (input) */ WORD stage4; /* Are we using stage4? */ - WORD Stage4Name; /* Selects the stage 4 temporary file name */ } SORTING; #ifdef WITHPTHREADS diff --git a/sources/tools.c b/sources/tools.c index 3ee4798ce..dd2eaf0a4 100644 --- a/sources/tools.c +++ b/sources/tools.c @@ -1654,11 +1654,9 @@ void UpdateMaxSize(void) position = sort->file.filesize; ADD2POS(sumsize,position); } - for ( i = 0; i < 2; i++ ) { - if ( sort && sort->FoStage4[i].handle >= 0 ) { - position = sort->FoStage4[i].filesize; - ADD2POS(sumsize,position); - } + if ( sort && sort->FoStage4.handle >= 0 ) { + position = sort->FoStage4.filesize; + ADD2POS(sumsize,position); } } } @@ -1670,11 +1668,9 @@ void UpdateMaxSize(void) position = sort->file.filesize; ADD2POS(sumsize,position); } - for ( i = 0; i < 2; i++ ) { - if ( sort && sort->FoStage4[i].handle >= 0 ) { - position = sort->FoStage4[i].filesize; - ADD2POS(sumsize,position); - } + if ( sort && sort->FoStage4.handle >= 0 ) { + position = sort->FoStage4.filesize; + ADD2POS(sumsize,position); } } #endif From 105f3a881f1729dbaf4565cc77220b94903955e9 Mon Sep 17 00:00:00 2001 From: Josh Davies Date: Tue, 17 Nov 2026 15:32:40 +0000 Subject: [PATCH 3/3] refactor: decouple file patch cache from SortIOSize To maximise use of large+small buffers for caching file patches during sorting, FORM used to update SortIOSize relect the maximum possible cache size. This had the effect of also making the output "PObuffer" rather large. Confusingly, also setting FilePatches smaller could sometimes cause an increased LargeSize alloc for TFORM workers. Decouple these sizes: now the PObuffer is sized according to the user's SortIOSize (and at least MaxTermSize), and when reading file patches we share the large+small buffer space between the current number of active patches (which might be less than file patches: in this case the per-patch cache size is larger than before). --- doc/manual/setup.tex | 23 ++++------- sources/compress.c | 13 ++++-- sources/declare.h | 2 +- sources/fsizes.h | 1 + sources/setfile.c | 41 ++++++------------ sources/sort.c | 98 +++++++++++++++++++++++--------------------- sources/threads.c | 3 +- 7 files changed, 86 insertions(+), 95 deletions(-) diff --git a/doc/manual/setup.tex b/doc/manual/setup.tex index 12168922f..5bf936701 100644 --- a/doc/manual/setup.tex +++ b/doc/manual/setup.tex @@ -370,12 +370,12 @@ \chapter{The setup} file operations are notoriously slow the combination of the small buffer, the small extension and the large buffer is used for caching\index{cache} purposes. Hence this space can be split in `FilePatches' caches. The -limitation is that each cache should be capable to contain at least two -terms of maximal size. This means that the sum of SmallExtension and -LargeSize must be at least FilePatches times 2*MaxTermSize*(bytes in short -integer). It is possible to set the size of these caches directly with the -variable {\sl SortIOsize\index{sortiosize}}. If the variable is too large, -the variable FilePatches may be adjusted by \FORM\@. If there are more than +limitation is that each cache should be capable of containing at least +three terms of maximal size. This means that the sum of SmallExtension +and LargeSize must be at least FilePatches times 3*MaxTermSize*(bytes in +short integer). FilePatches may be adjusted by \FORM\ to accomodate this, +and LargeSize if FilePatches is already minimal. +If there are more than FilePatches patches in the sort file, a second sort file is needed for the output of each `superpatch'\index{superpatch}. When the first sort file has been treated, the second sort file can be treated in exactly the same way @@ -409,14 +409,9 @@ \chapter{The setup} \rightvitem{13cm}{The size of the large buffer.} \leftvitem{3cm}{SortIOsize\index{setup!sortiosize}\index{sortiosize}} -\rightvitem{13cm}{The size of the buffer that is used to write to the -intermediate sorting file and to read from it. It should be noted that if -this buffer is not very large, the sorting of large files may become rather -slow, depending on the operating system. Hence we recommend a potential -fourth stage in the sorting over having this number too small to fit more -filepatches in the combined small and large buffer. Setting the small and -large buffers to a decent size may avoid all problems by a: making more -space for the caching, b: creating fewer file patches to start with.} +\rightvitem{13cm}{The size of the output buffer for the intermediate sorting +file, and of the buffers used for compressed file I/O. \FORM\ will ensure +that this is at least MaxTermSize.} There is a second set of the above setup parameters for sorts of subexpressions\index{subexpressions} as in function arguments or in the diff --git a/sources/compress.c b/sources/compress.c index 8fc82ef9a..f8e02e0d1 100644 --- a/sources/compress.c +++ b/sources/compress.c @@ -712,16 +712,23 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer /* INTERNAL_ERROR_EXCL_STOP */ } else { + // The caches may be larger than the remaining patch data. Limit the read size: + if ( ISGEPOS(*position,S->fPatchesStop[numstream]) ) return(0); + toread = buffersize; + if ( !ISGEPOSINC(S->fPatchesStop[numstream],*position,toread) ) { + DIFPOS(pos,S->fPatchesStop[numstream],*position); + toread = (LONG)BASEPOSITION(pos); + } #ifdef GZIPDEBUG MLOCK(ErrorMessageLock); - MesPrint("%w++Reading %l bytes at position %10p",buffersize,position); + MesPrint("%w++Reading %l bytes at position %10p",toread,position); MUNLOCK(ErrorMessageLock); #endif #ifdef ALLLOCK LOCK(f->pthreadslock); #endif SeekFile(f->handle,position,SEEK_SET); - readsize = ReadFile(f->handle,buffer,buffersize); + readsize = ReadFile(f->handle,buffer,toread); SeekFile(f->handle,position,SEEK_CUR); #ifdef ALLLOCK UNLOCK(f->pthreadslock); @@ -730,7 +737,7 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>%wFillInputGZIP: Read error during uncompressed sort."); - MesPrint("%w++Reading %l bytes at position %10p",buffersize,position); + MesPrint("%w++Reading %l bytes at position %10p",toread,position); MUNLOCK(ErrorMessageLock); /* INTERNAL_ERROR_EXCL_STOP */ } diff --git a/sources/declare.h b/sources/declare.h index cd4fb2e40..43a676dde 100644 --- a/sources/declare.h +++ b/sources/declare.h @@ -688,7 +688,7 @@ extern void PrintFeatureList(void); extern void PrintRunningTime(void); extern LONG GetRunningTime(void); extern int PutBracket(PHEAD WORD *); -extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,int); +extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,LONG,int); extern int PutInStore(INDEXENTRY *,WORD); extern WORD PutOut(PHEAD WORD *,POSITION *,FILEHANDLE *,WORD); extern UWORD Quotient(UWORD *,WORD *,WORD); diff --git a/sources/fsizes.h b/sources/fsizes.h index bcd877a01..6c8f87c2d 100644 --- a/sources/fsizes.h +++ b/sources/fsizes.h @@ -129,6 +129,7 @@ #endif #define MAXPATCHES 256 +#define MINFPATCHES 4 #define MAXFPATCHES 256 #define SORTIOSIZE 200000L diff --git a/sources/setfile.c b/sources/setfile.c index 1ef3b69e7..847c9f68e 100644 --- a/sources/setfile.c +++ b/sources/setfile.c @@ -585,17 +585,8 @@ int AllocSetups(void) AM.S0 = 0; AM.S0 = AllocSort(LargeSize,SmallSize,SmallEsize,TermsInSmall ,MaxPatches,MaxFpatches,IOsize,0); - /* AM.S0->file.ziosize was already set to a (larger) value by AllocSort, here it is re-set. */ -#ifdef WITHZLIB - AM.S0->file.ziosize = IOsize; - AM.S0->FoStage4.ziosize = IOsize; -#ifndef WITHPTHREADS - AT.S0 = AM.S0; -#endif -#else #ifndef WITHPTHREADS AT.S0 = AM.S0; -#endif #endif sp = GetSetupPar((UBYTE *)"subsmallsize"); AM.SSmallSize = sp->value; @@ -872,11 +863,9 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i LONG TermsInSmall = inTermsInSmall; int MaxPatches = inMaxPatches; int MaxFpatches = inMaxFpatches; - LONG IOsize = inIOsize; LONG longer,terms2insmall,sortsize,longerp; - LONG IObuffersize = IOsize; - LONG IOtry; + LONG IObuffersize = inIOsize; SORTING *sort; int fname2Size = 0, j = 0; char *s; @@ -885,7 +874,7 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i while ( *s ) { s++; fname2Size++; } fname2Size += 16; } - if ( MaxFpatches < 4 ) MaxFpatches = 4; + if ( MaxFpatches < MINFPATCHES ) MaxFpatches = MINFPATCHES; longer = MaxPatches > MaxFpatches ? MaxPatches : MaxFpatches; longerp = longer; while ( (1 << j) < longerp ) j++; @@ -902,28 +891,24 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i if ( LargeSize < 0 ) LargeSize = 0; sortsize = sizeof(SORTING); sortsize = (sortsize+15)&(-16L); + if ( IObuffersize < AM.MaxTer ) IObuffersize = AM.MaxTer; IObuffersize = (IObuffersize+sizeof(WORD)-1)/sizeof(WORD); /* - The next statement fixes a bug. In the rare case that we have a - problem here, we expand the size of the large buffer or the - small extension + File patches are cached in the combined large+small buffer when reading. + Each cache needs to fit needs to fit a maximally sized term, plus headroom + of two maximally sized terms and COMPINC. COMPINC guarantees that the + two-WORD header of the following compressed term fits in the cache. These + cache spaces are independent of the file's own IO buffer (PObuffer) which + is controlled by "SortIOSize". */ - if ( (ULONG)( LargeSize+SmallEsize ) < MaxFpatches*((IObuffersize - +COMPINC)*sizeof(WORD)+2*AM.MaxTer) ) { + if ( (ULONG)( LargeSize+SmallEsize ) < MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)) ) { if ( LargeSize == 0 ) - SmallEsize = MaxFpatches*((IObuffersize+COMPINC)*sizeof(WORD)+2*AM.MaxTer); + SmallEsize = MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)); else - LargeSize = MaxFpatches*((IObuffersize+COMPINC)*sizeof(WORD)+2*AM.MaxTer) + LargeSize = MaxFpatches*(3*AM.MaxTer+COMPINC*sizeof(WORD)) - SmallEsize; } - IOtry = ((LargeSize+SmallEsize)/MaxFpatches-2*AM.MaxTer)/sizeof(WORD)-COMPINC; - - /* Here both IObuffersize and IOtry are in units of sizeof(WORD). */ - if ( IObuffersize < IOtry ) { - IObuffersize = IOtry; - } - #if DEBUGGING char *prefix; if ( level == 0 ) { prefix = ""; } @@ -934,8 +919,6 @@ SORTING *AllocSort(LONG inLargeSize, LONG inSmallSize, LONG inSmallEsize, LONG i if ( TermsInSmall != inTermsInSmall ) { MesPrint("Warning: %sTermsInSmall adjusted: %l -> %l", prefix, inTermsInSmall, TermsInSmall); } if ( MaxPatches != inMaxPatches ) { MesPrint("Warning: MaxPatches adjusted: %d -> %d", inMaxPatches, MaxPatches); } if ( MaxFpatches != inMaxFpatches ) {MesPrint("Warning: MaxFPatches adjusted: %d -> %d", inMaxFpatches, MaxFpatches); } - /* This one is always changed if the LargeSize has not been... */ - /* if ( IObuffersize != inIOsize/(LONG)sizeof(WORD) ) { MesPrint("Warning: IOsize adjusted: %l -> %l", inIOsize/sizeof(WORD), IObuffersize); } */ #endif /* Allocate separate buffers for most struct members, for better testing and debugging with valgrind. */ diff --git a/sources/sort.c b/sources/sort.c index 520008d76..591be52f2 100644 --- a/sources/sort.c +++ b/sources/sort.c @@ -1044,7 +1044,7 @@ LONG EndSort(PHEAD WORD *buffer, int par) /* #] EndSort : - #[ PutIn : LONG PutIn(handle,position,buffer,take,npat) + #[ PutIn : LONG PutIn(handle,position,buffer,take,size,npat) */ /** * Reads a new patch from position in file handle. @@ -1056,19 +1056,22 @@ LONG EndSort(PHEAD WORD *buffer, int par) * @param position The position from which to read * @param buffer The buffer into which to read * @param take The unused tail should be moved before the buffer + * @param size The size of the input patch cache in bytes * @param npat The number of the patch. Is needed if the information * was compressed with gzip, because each patch has its * own independent gzip encoding. */ -LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, int npat) +LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, LONG size, int npat) { LONG i, RetCode; WORD *from, *to; #ifndef WITHZLIB - DUMMYUSE(npat); + GETIDENTITY + POSITION remaining; + LONG readsize = size; #endif - from = buffer + ( file->POsize * sizeof(UBYTE) )/sizeof(WORD); + from = buffer + size/sizeof(WORD); i = from - *take; if ( i*((LONG)(sizeof(WORD))) > AM.MaxTer ) { /* INTERNAL_ERROR_EXCL_START */ @@ -1083,28 +1086,33 @@ LONG PutIn(FILEHANDLE *file, POSITION *position, WORD *buffer, WORD **take, int *take = to; #ifdef WITHZLIB if ( ( RetCode = FillInputGZIP(file,position,(UBYTE *)buffer - ,file->POsize,npat) ) < 0 ) { + ,size,npat) ) < 0 ) { /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>PutIn: We have RetCode = %x while reading %x bytes", - RetCode,file->POsize); + RetCode,size); MUNLOCK(ErrorMessageLock); Terminate(-1); /* INTERNAL_ERROR_EXCL_STOP */ } #else + if ( ISGEPOS(*position,AT.SS->fPatchesStop[npat]) ) return(0); + if ( !ISGEPOSINC(AT.SS->fPatchesStop[npat],*position,readsize) ) { + DIFPOS(remaining,AT.SS->fPatchesStop[npat],*position); + readsize = (LONG)BASEPOSITION(remaining); + } #ifdef ALLLOCK LOCK(file->pthreadslock); #endif SeekFile(file->handle,position,SEEK_SET); - if ( ( RetCode = ReadFile(file->handle,(UBYTE *)buffer,file->POsize) ) < 0 ) { + if ( ( RetCode = ReadFile(file->handle,(UBYTE *)buffer,readsize) ) < 0 ) { #ifdef ALLLOCK UNLOCK(file->pthreadslock); #endif /* INTERNAL_ERROR_EXCL_START */ MLOCK(ErrorMessageLock); MesPrint("!>PutIn: We have RetCode = %x while reading %x bytes", - RetCode,file->POsize); + RetCode,readsize); MUNLOCK(ErrorMessageLock); Terminate(-1); /* INTERNAL_ERROR_EXCL_STOP */ @@ -3524,14 +3532,14 @@ int MergePatches(WORD par) WORD *p, lpat, mpat, level, l1, l2, r1, r2, r3, c; WORD *m2, *m3, r31, r33, ki, *rr; UWORD *coef; - POSITION position; + POSITION position, position2; FILEHANDLE *fin, *fout; + LONG cachesize = 0; int fhandle; /* UBYTE *s; */ #ifdef WITHZLIB - POSITION position2; int oldgzipCompress = AR.gzipCompress; if ( par == 2 ) { AR.gzipCompress = 0; @@ -3609,14 +3617,6 @@ int MergePatches(WORD par) m1 = S->fpincompressed; S->fpincompressed = S->fpcompressed; S->fpcompressed = m1; - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->iPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } #endif S->stage4 = 0; goto FileMake; @@ -3635,14 +3635,6 @@ int MergePatches(WORD par) m1 = S->fpincompressed; S->fpincompressed = S->fpcompressed; S->fpcompressed = m1; - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->fPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } #endif } fout = AR.outfile; @@ -3656,24 +3648,37 @@ int MergePatches(WORD par) } else { /* Load the patches */ S->lPatch = (S->inNum); + for ( i = 0; i < S->lPatch; i++ ) { + S->fPatchesStop[i] = S->iPatches[i+1]; +#ifdef GZIPDEBUG + MLOCK(ErrorMessageLock); + MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); + MUNLOCK(ErrorMessageLock); +#endif + } #ifdef WITHMPI - if ( S->lPatch > 1 || ( (PF.exprtodo <0) && (fout == AR.outfile || fout == AR.hidefile ) ) ) { + if ( S->lPatch > 0 && ( S->lPatch > 1 || ( (PF.exprtodo <0) && (fout == AR.outfile || fout == AR.hidefile ) ) ) ) { #else if ( S->lPatch > 1 ) { #endif #ifdef WITHZLIB SetupAllInputGZIP(S); #endif + // We use the combined large+small buffer for caching the patches. + // Divide it by the current number of patches, rather than by MaxFPatches, + // and make sure that we retain the usual per-cache headroom. + cachesize = ((S->LargeSize+S->SmallEsize)/S->lPatch-COMPINC) + *sizeof(WORD)-2*AM.MaxTer; p = S->lBuffer; for ( i = 0; i < S->lPatch; i++ ) { p = (WORD *)(((UBYTE *)p)+2*AM.MaxTer+COMPINC*sizeof(WORD)); S->Patches[i] = p; - p = (WORD *)(((UBYTE *)p) + fin->POsize); + p = (WORD *)(((UBYTE *)p) + cachesize); S->pStop[i] = m2 = p; #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,i); + PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,cachesize,i); #else - ADDPOS(S->iPatches[i],PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,i)); + ADDPOS(S->iPatches[i],PutIn(fin,&(S->iPatches[i]),S->Patches[i],&m2,cachesize,i)); #endif } } @@ -3820,9 +3825,20 @@ int MergePatches(WORD par) ADDPOS(S->SizeInFile[par],1); #else /* old code */ + POSITION remaining; + LONG readsize; + position2 = S->iPatches[0]; SeekFile(fin->handle,&(S->iPatches[0]),SEEK_SET); /* needed for stage4 */ - while ( ( length = ReadFile(fin->handle, - (UBYTE *)(S->sBuffer),S->SmallEsize*sizeof(WORD)) ) > 0 ) { + length = 0; + while ( !ISGEPOS(position2,S->fPatchesStop[0]) ) { + readsize = S->SmallEsize*sizeof(WORD); + if ( !ISGEPOSINC(S->fPatchesStop[0],position2,readsize) ) { + DIFPOS(remaining,S->fPatchesStop[0],position2); + readsize = (LONG)BASEPOSITION(remaining); + } + length = ReadFile(fin->handle,(UBYTE *)(S->sBuffer),readsize); + if ( length <= 0 ) break; + ADDPOS(position2,length); if ( WriteFile(fout->handle,(UBYTE *)(S->sBuffer),length) != length ) goto PatwCall; ADDPOS(position,length); @@ -4040,10 +4056,10 @@ int MergePatches(WORD par) if ( !par && (poin[ul] + im + COMPINC) >= S->pStop[ki] && im > 0 ) { #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[ul]),ki); + PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[ul]),cachesize,ki); #else ADDPOS(S->iPatches[ki],PutIn(fin,&(S->iPatches[ki]), - S->Patches[ki],&(poin[ul]),ki)); + S->Patches[ki],&(poin[ul]),cachesize,ki)); #endif poin2[ul] = poin[ul] + im; } @@ -4109,10 +4125,10 @@ int MergePatches(WORD par) if ( !par && ( (poin[k] + im + COMPINC) >= S->pStop[ki] ) && im > 0 ) { #ifdef WITHZLIB - PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[k]),ki); + PutIn(fin,&(S->iPatches[ki]),S->Patches[ki],&(poin[k]),cachesize,ki); #else ADDPOS(S->iPatches[ki],PutIn(fin,&(S->iPatches[ki]), - S->Patches[ki],&(poin[k]),ki)); + S->Patches[ki],&(poin[k]),cachesize,ki)); #endif poin2[k] = poin[k] + im; } @@ -4201,16 +4217,6 @@ int MergePatches(WORD par) /* INTERNAL_ERROR_EXCL_STOP */ } *rr = 0; -#ifdef WITHZLIB - for ( i = 0; i < S->inNum; i++ ) { - S->fPatchesStop[i] = S->iPatches[i+1]; -#ifdef GZIPDEBUG - MLOCK(ErrorMessageLock); - MesPrint("%w fPatchesStop[%d] = %10p",i,&(S->fPatchesStop[i])); - MUNLOCK(ErrorMessageLock); -#endif - } -#endif goto ConMer; } else { diff --git a/sources/threads.c b/sources/threads.c index 2479b74cf..64e21ee80 100644 --- a/sources/threads.c +++ b/sources/threads.c @@ -748,8 +748,7 @@ ALLPRIVATES *InitializeOneThread(int identity) ,AM.S0->SmallEsize*sizeof(WORD)/numberofworkers ,AM.S0->TermsInSmall ,AM.S0->MaxPatches -/* ,AM.S0->MaxPatches/numberofworkers */ - ,AM.S0->MaxFpatches/numberofworkers + ,MaX(AM.S0->MaxFpatches/numberofworkers, MINFPATCHES) ,AM.S0->file.POsize ,0); }