Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 153 additions & 6 deletions check/fixes.frm
Original file line number Diff line number Diff line change
Expand Up @@ -2162,13 +2162,160 @@ Print;
.end
# Only for 64-bit systems. Otherwise "Output term too large".
#require wordsize == 4
# For now it fails because
# "Currently Stage 4 sorts are not allowed for function arguments or $ variables."
assert runtime_error?
#assert succeeded?
#assert result("test1") =~ expr("0")
#assert result("test2") =~ expr("g(0)")
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("test1") =~ expr("0")
assert result("test2") =~ expr("g(0)")
*--#] Issue211 :
*--#[ Issue215_1 :
#-

#: SubTermsInSmall 16
#: SubLargePatches 4
#: SubFilePatches 4

Off statistics;
Off threadstats;

CFunction f;
Symbol j,x;

#define BLOWUP "1000"

Local argument = f(x) + f(2*x);
Argument f;
Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
EndArgument;
Identify f(x?) = x;
Identify x = 1;
Print;
.end
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("argument") =~ expr("3003/2")
*--#] Issue215_1 :
*--#[ Issue215_2 :
#-

#: SubTermsInSmall 16
#: SubLargePatches 4
#: SubFilePatches 4

Off statistics;
Off threadstats;

Symbol j,x;

#define BLOWUP "1000"

#$dol = x;
Local dollar = 1;
Inside $dol;
Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
EndInside;
Multiply $dol;
Identify x = 1;
Print;
ModuleOption local $dol;
.end
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("dollar") =~ expr("1001/2")
*--#] Issue215_2 :
*--#[ Issue215_3 :
#-

#: SubTermsInSmall 16
#: SubLargePatches 4
#: SubFilePatches 4

Off statistics;
Off threadstats;

Symbol j,x;

#define BLOWUP "1000"

Local term = x;
Term;
Identify x = sum_(j,1,`BLOWUP',x*j/`BLOWUP');
EndTerm;
Identify x = 1;
Print;
.end
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("term") =~ expr("1001/2")
*--#] Issue215_3 :
*--#[ Issue215_4 :
#-
#: TermsInSmall 16
#: LargePatches 4
#: FilePatches 4

#: SubTermsInSmall 16
#: SubLargePatches 4
#: SubFilePatches 4

Off statistics;
Off threadstats;

CFunction f;
Symbol i,j,x;

#define N "257"
#define BLOWUP "257"

Local nested = {`N'*(`N'+1)/2}
#do i = 1,`N'
- f(x^`i')
#enddo
;

Argument f;
Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP');
EndArgument;
Identify f(x?) = x;
Identify x^i? = 1;
Print;
.end
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("nested") =~ expr("0")
*--#] Issue215_4 :
*--#[ Issue215_4b :
#-
#: TermsInSmall 16
#: LargePatches 4
#: FilePatches 4

#: SubTermsInSmall 16
#: SubLargePatches 4
#: SubFilePatches 4

Off statistics;
Off threadstats;

CFunction f;
Symbol i,j,x;

* Enough terms to have a stagesort in each thread of a -w5 test
#define N "{5*257}"
#define BLOWUP "257"

Local nested = {`N'*(`N'+1)/2}
#do i = 1,`N'
- f(x^`i')
#enddo
;
* Sort to distribute terms around workers
.sort

Argument f;
Identify x^i? = sum_(j,1,`BLOWUP',x^i*i/`BLOWUP');
EndArgument;
Identify f(x?) = x;
Identify x^i? = 1;
Print;
.end
assert warning?("StageSort in sub-buffer: recommend increasing setup sub-buffer sizes")
assert result("nested") =~ expr("0")
*--#] Issue215_4b :
*--#[ Issue222 :
* accessing #factdollar factors causes program termination
Symbol x;
Expand Down
23 changes: 9 additions & 14 deletions doc/manual/setup.tex
Original file line number Diff line number Diff line change
Expand Up @@ -370,12 +370,12 @@ \chapter{The setup}
file operations are notoriously slow the combination of the small buffer,
the small extension and the large buffer is used for caching\index{cache}
purposes. Hence this space can be split in `FilePatches' caches. The
limitation is that each cache should be capable to contain at least two
terms of maximal size. This means that the sum of SmallExtension and
LargeSize must be at least FilePatches times 2*MaxTermSize*(bytes in short
integer). It is possible to set the size of these caches directly with the
variable {\sl SortIOsize\index{sortiosize}}. If the variable is too large,
the variable FilePatches may be adjusted by \FORM\@. If there are more than
limitation is that each cache should be capable of containing at least
three terms of maximal size. This means that the sum of SmallExtension
and LargeSize must be at least FilePatches times 3*MaxTermSize*(bytes in
short integer). FilePatches may be adjusted by \FORM\ to accomodate this,
and LargeSize if FilePatches is already minimal.
If there are more than
FilePatches patches in the sort file, a second sort file is needed for the
output of each `superpatch'\index{superpatch}. When the first sort file has
been treated, the second sort file can be treated in exactly the same way
Expand Down Expand Up @@ -409,14 +409,9 @@ \chapter{The setup}
\rightvitem{13cm}{The size of the large buffer.}

\leftvitem{3cm}{SortIOsize\index{setup!sortiosize}\index{sortiosize}}
\rightvitem{13cm}{The size of the buffer that is used to write to the
intermediate sorting file and to read from it. It should be noted that if
this buffer is not very large, the sorting of large files may become rather
slow, depending on the operating system. Hence we recommend a potential
fourth stage in the sorting over having this number too small to fit more
filepatches in the combined small and large buffer. Setting the small and
large buffers to a decent size may avoid all problems by a: making more
space for the caching, b: creating fewer file patches to start with.}
\rightvitem{13cm}{The size of the output buffer for the intermediate sorting
file, and of the buffers used for compressed file I/O. \FORM\ will ensure
that this is at least MaxTermSize.}

There is a second set of the above setup parameters for sorts of
subexpressions\index{subexpressions} as in function arguments or in the
Expand Down
8 changes: 4 additions & 4 deletions doc/manual/sorting.tex
Original file line number Diff line number Diff line change
Expand Up @@ -254,10 +254,10 @@ \chapter{Sorting and statistics}
the chapter on the setup \ref{setup}. This chapter gives also all default
values.

There is one restriction on the sorting of function arguments and
\$-expressions: They are not allowed to go into the stage4 sorting. Any
such attempt will result in an error message and the suggestion to raise
the size of the buffers for this type of sorting.
Sorting these `sub-buffers' may also run into a stage 4 sort when necessary.
This usually implies that performance will be severely affected, hence the
first time this happens during a program a warning will be printed, with the
suggestion to increase the sizes of the sub-buffers.

When \FORM\ is running in parallel mode (either \TFORM\ or \ParFORM) each worker
will need its own buffers. In \ParFORM\ in which the processors each control
Expand Down
3 changes: 0 additions & 3 deletions sources/checkpoint.c
Original file line number Diff line number Diff line change
Expand Up @@ -1184,7 +1184,6 @@ static void print_R()
MesPrint("%d", AR.DeferFlag);
MesPrint("%d", AR.TePos);
MesPrint("%d", AR.sLevel);
MesPrint("%d", AR.Stage4Name);
MesPrint("%d", AR.GetOneFile);
MesPrint("%d", AR.PolyFun);
MesPrint("%d", AR.PolyFunInv);
Expand Down Expand Up @@ -2333,7 +2332,6 @@ int DoRecovery(int *moduletype)
R_SET(AR.DeferFlag, WORD);
R_SET(AR.TePos, WORD);
R_SET(AR.sLevel, WORD);
R_SET(AR.Stage4Name, WORD);
R_SET(AR.GetOneFile, WORD);
R_SET(AR.PolyFun, WORD);
R_SET(AR.PolyFunInv, WORD);
Expand Down Expand Up @@ -2957,7 +2955,6 @@ static int DoSnapshot(int moduletype)
S_WRITE_B(&AR.DeferFlag, sizeof(WORD));
S_WRITE_B(&AR.TePos, sizeof(WORD));
S_WRITE_B(&AR.sLevel, sizeof(WORD));
S_WRITE_B(&AR.Stage4Name, sizeof(WORD));
S_WRITE_B(&AR.GetOneFile, sizeof(WORD));
S_WRITE_B(&AR.PolyFun, sizeof(WORD));
S_WRITE_B(&AR.PolyFunInv, sizeof(WORD));
Expand Down
13 changes: 10 additions & 3 deletions sources/compress.c
Original file line number Diff line number Diff line change
Expand Up @@ -712,16 +712,23 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer
/* INTERNAL_ERROR_EXCL_STOP */
}
else {
// The caches may be larger than the remaining patch data. Limit the read size:
if ( ISGEPOS(*position,S->fPatchesStop[numstream]) ) return(0);
toread = buffersize;
if ( !ISGEPOSINC(S->fPatchesStop[numstream],*position,toread) ) {
DIFPOS(pos,S->fPatchesStop[numstream],*position);
toread = (LONG)BASEPOSITION(pos);
}
#ifdef GZIPDEBUG
MLOCK(ErrorMessageLock);
MesPrint("%w++Reading %l bytes at position %10p",buffersize,position);
MesPrint("%w++Reading %l bytes at position %10p",toread,position);
MUNLOCK(ErrorMessageLock);
#endif
#ifdef ALLLOCK
LOCK(f->pthreadslock);
#endif
SeekFile(f->handle,position,SEEK_SET);
readsize = ReadFile(f->handle,buffer,buffersize);
readsize = ReadFile(f->handle,buffer,toread);
SeekFile(f->handle,position,SEEK_CUR);
#ifdef ALLLOCK
UNLOCK(f->pthreadslock);
Expand All @@ -730,7 +737,7 @@ LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffer
/* INTERNAL_ERROR_EXCL_START */
MLOCK(ErrorMessageLock);
MesPrint("!>%wFillInputGZIP: Read error during uncompressed sort.");
MesPrint("%w++Reading %l bytes at position %10p",buffersize,position);
MesPrint("%w++Reading %l bytes at position %10p",toread,position);
MUNLOCK(ErrorMessageLock);
/* INTERNAL_ERROR_EXCL_STOP */
}
Expand Down
2 changes: 1 addition & 1 deletion sources/declare.h
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ extern void PrintFeatureList(void);
extern void PrintRunningTime(void);
extern LONG GetRunningTime(void);
extern int PutBracket(PHEAD WORD *);
extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,int);
extern LONG PutIn(FILEHANDLE *,POSITION *,WORD *,WORD **,LONG,int);
extern int PutInStore(INDEXENTRY *,WORD);
extern WORD PutOut(PHEAD WORD *,POSITION *,FILEHANDLE *,WORD);
extern UWORD Quotient(UWORD *,WORD *,WORD);
Expand Down
1 change: 1 addition & 0 deletions sources/fsizes.h
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@
#endif

#define MAXPATCHES 256
#define MINFPATCHES 4
#define MAXFPATCHES 256
#define SORTIOSIZE 200000L

Expand Down
1 change: 1 addition & 0 deletions sources/proces.c
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ int Processor(void)
#endif
AS.CollectOverFlag = 0;
AR.expchanged = 0;
AT.SubSortStage4Warning = 0;
if ( i == last ) LastExpression = 1;
else LastExpression = 0;
if ( e->inmem ) {
Expand Down
Loading
Loading