From a10b8c5975ef4128610f9c0402047c9a740f0249 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Wed, 7 Dec 2022 11:46:31 -0800 Subject: [PATCH] Implement support for --MaxSolveTimeoutInSeconds. --- include/PerfectHash.h | 1 + include/PerfectHashErrors.h | 32 +++++++- src/PerfectHash/Chm01.c | 26 ++++++- src/PerfectHash/ExtractArg.c | 2 + src/PerfectHash/PerfectHashContext.c | 72 +++++++++++++++++- src/PerfectHash/PerfectHashContext.h | 9 ++- src/PerfectHash/PerfectHashErrors.dbg | 2 + src/PerfectHash/PerfectHashErrors.mc | 30 +++++++- src/PerfectHash/PerfectHashErrors_English.bin | Bin 89836 -> 90348 bytes src/PerfectHash/PerfectHashTable.h | 8 +- src/PerfectHash/PerfectHashTableCreate.c | 9 +++ src/PerfectHash/Rtl.h | 6 ++ src/PerfectHash/RtlOutput.h | 2 + 13 files changed, 189 insertions(+), 10 deletions(-) diff --git a/include/PerfectHash.h b/include/PerfectHash.h index d290a938..bbeaafcf 100644 --- a/include/PerfectHash.h +++ b/include/PerfectHash.h @@ -3172,6 +3172,7 @@ typedef RNG_VTBL *PRNG_VTBL; ENTRY(RngOffset) \ ENTRY(Seed3Byte1MaskCounts) \ ENTRY(Seed3Byte2MaskCounts) \ + ENTRY(MaxSolveTimeInSeconds) \ LAST_ENTRY(Remark) #define TABLE_CREATE_PARAMETER_TABLE_ENTRY(ENTRY) \ diff --git a/include/PerfectHashErrors.h b/include/PerfectHashErrors.h index 41a053f7..cd732441 100644 --- a/include/PerfectHashErrors.h +++ b/include/PerfectHashErrors.h @@ -230,6 +230,15 @@ Module Name: // #define PH_I_TABLE_CREATED_BUT_VALUES_ARRAY_ALLOC_FAILED ((HRESULT)0x60040087L) +// +// MessageId: PH_I_SOLVE_TIMEOUT_EXPIRED +// +// MessageText: +// +// Solve timeout expired. +// +#define PH_I_SOLVE_TIMEOUT_EXPIRED ((HRESULT)0x60040088L) + //////////////////////////////////////////////////////////////////////////////// // PH_SEVERITY_INFORMATIONAL -- Usage Messages @@ -899,6 +908,11 @@ Module Name: // be returned if the provided string contains commas (as this will // break the .csv output). // +// --MaxSolveTimeInSeconds= +// +// Supplies the maximum number of seconds to try and solve an individual +// graph. +// // // Console Output Character Legend // @@ -913,7 +927,7 @@ Module Name: // were possible (due to the maximum resize limit also being hit). // // F Failed to create a table due to a target not being reached by a specific -// number of attempts or time duration. Not yet implemented. +// number of attempts. // // * None of the worker threads were able to allocate sufficient memory to // attempt solving the graph. @@ -928,13 +942,16 @@ Module Name: // // V The graph was created successfully, however, we weren't able to allocate // enough memory for the table values array in order for the array to be -// used after creation. This can be avoided by omitting --TestAfterCreate. +// used after creation. This can be avoided by supplying the command line +// parameter --SkipTestAfterCreate. // -// T The requested number of table elements was too large (exceeded 32 bits). +// T The requested number of table elements was too large. // // S A shutdown event was received. This shouldn't be seen unless externally // signaling the named shutdown event associated with a context. // +// t The solve timeout was reached before a solution was found. +// // #define PH_MSG_PERFECT_HASH_USAGE ((HRESULT)0x60040101L) @@ -4057,3 +4074,12 @@ Module Name: // #define PH_E_INVALID_REMARK ((HRESULT)0xE00403D3L) +// +// MessageId: PH_E_INVALID_MAX_SOLVE_TIME_IN_SECONDS +// +// MessageText: +// +// Invalid --MaxSolveTimeInSeconds. +// +#define PH_E_INVALID_MAX_SOLVE_TIME_IN_SECONDS ((HRESULT)0xE00403D4L) + diff --git a/src/PerfectHash/Chm01.c b/src/PerfectHash/Chm01.c index 138da344..69d94ff2 100644 --- a/src/PerfectHash/Chm01.c +++ b/src/PerfectHash/Chm01.c @@ -652,6 +652,18 @@ Return Value: } } + // + // If we've been asked to cap the maximum solve time, submit a threadpool + // timer now to achieve that. + // + + if (Table->MaxSolveTimeInSeconds > 0) { + SetThreadpoolTimer(Context->SolveTimeout, + &Table->RelativeMaxSolveTimeInFiletime.AsFileTime, + 0, + 0); + } + // // Submit all of the file preparation work items. // @@ -840,6 +852,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); WaitForThreadpoolWorkCallbacks(Context->FinishedWork, FALSE); + WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); if (!NoFileIo(Table)) { @@ -975,6 +988,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); WaitForThreadpoolWorkCallbacks(Context->FinishedWork, FALSE); + WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); Success = (Context->FinishedCount > 0); @@ -1017,8 +1031,10 @@ Return Value: } else if (FailedEventSet) { - if (Context->State.AllGraphsFailedMemoryAllocation == TRUE) { + if (Context->State.AllGraphsFailedMemoryAllocation != FALSE) { Result = PH_I_FAILED_TO_ALLOCATE_MEMORY_FOR_ALL_GRAPHS; + } else if (Context->State.SolveTimeoutExpired != FALSE) { + Result = PH_I_SOLVE_TIMEOUT_EXPIRED; } else { Result = PH_I_CREATE_TABLE_ROUTINE_FAILED_TO_FIND_SOLUTION; } @@ -1040,6 +1056,12 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, CancelPending); + // + // Wait for the solve timeout, if applicable. + // + + WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); + // // Perform the same operation for the file work threadpool. Note that // the only work item type we've dispatched to this pool at this point @@ -1074,6 +1096,7 @@ Return Value: WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); WaitForThreadpoolWorkCallbacks(Context->FinishedWork, FALSE); + WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); if (CtrlCPressed) { Result = PH_E_CTRL_C_PRESSED; @@ -1383,6 +1406,7 @@ Return Value: SetEvent(Context->ShutdownEvent); WaitForThreadpoolWorkCallbacks(Context->MainWork, TRUE); + WaitForThreadpoolTimerCallbacks(Context->SolveTimeout, TRUE); if (!NoFileIo(Table)) { WaitForThreadpoolWorkCallbacks(Context->FileWork, TRUE); } diff --git a/src/PerfectHash/ExtractArg.c b/src/PerfectHash/ExtractArg.c index d38b420e..13612781 100644 --- a/src/PerfectHash/ExtractArg.c +++ b/src/PerfectHash/ExtractArg.c @@ -958,6 +958,8 @@ Return Value: ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MinNumberOfKeysForFindBestGraph); + ADD_PARAM_IF_EQUAL_AND_VALUE_IS_INTEGER(MaxSolveTimeInSeconds); + #define IS_VALUE_EQUAL(ValueName) \ Rtl->RtlEqualUnicodeString(ValueString, &ValueName, TRUE) diff --git a/src/PerfectHash/PerfectHashContext.c b/src/PerfectHash/PerfectHashContext.c index 6df451c4..394a9f21 100644 --- a/src/PerfectHash/PerfectHashContext.c +++ b/src/PerfectHash/PerfectHashContext.c @@ -40,6 +40,13 @@ VOID _Inout_ PTP_WORK Work ); +typedef +VOID +(NTAPI TP_TIMER_CALLBACK)( + _Inout_ PTP_CALLBACK_INSTANCE Instance, + _Inout_opt_ PVOID Context, + _Inout_ PTP_TIMER Work + ); typedef VOID @@ -56,6 +63,7 @@ TP_WORK_CALLBACK MainWorkCallback; TP_WORK_CALLBACK FileWorkCallback; TP_WORK_CALLBACK ErrorWorkCallback; TP_WORK_CALLBACK FinishedWorkCallback; +TP_TIMER_CALLBACK SolveTimeoutCallback; TP_CLEANUP_GROUP_CANCEL_CALLBACK CleanupCallback; // @@ -465,6 +473,19 @@ Return Value: goto Error; } + // + // Create the timer object for the solve timeout. + // + + Context->SolveTimeout = CreateThreadpoolTimer(SolveTimeoutCallback, + Context, + &Context->MainCallbackEnv); + if (!Context->SolveTimeout) { + SYS_ERROR(CreateThreadpoolTimer); + Result = PH_E_SYSTEM_CALL_FAILED; + goto Error; + } + // // Create the File threadpool structures. Automatically clamp the min/max // threads for this threadpool to the number of system processors. @@ -761,17 +782,19 @@ Return Value: TRUE, NULL); Context->MainWork = NULL; + Context->SolveTimeout = NULL; CloseThreadpoolCleanupGroup(Context->MainCleanupGroup); Context->MainCleanupGroup = NULL; } else { // - // Invariant check: Context->MainWork should never be set if + // Invariant check: MainWork and SolveTimeout should never be set if // MainCleanupGroup is not set. // ASSERT(!Context->MainWork); + ASSERT(!Context->SolveTimeout); } if (Context->MainThreadpool) { @@ -910,6 +933,7 @@ Return Value: Context->GraphMemoryFailures = 0; Context->LowMemoryObserved = 0; Context->State.AllGraphsFailedMemoryAllocation = FALSE; + Context->State.SolveTimeoutExpired = FALSE; Context->VertexCollisionFailures = 0; Context->CyclicGraphFailures = 0; @@ -1017,6 +1041,52 @@ Return Value: return; } +_Use_decl_annotations_ +VOID +SolveTimeoutCallback( + PTP_CALLBACK_INSTANCE Instance, + PVOID Ctx, + PTP_TIMER Timer + ) +/*++ + +Routine Description: + + This is the callback routine for the Main threadpool "solve timeout" timer. + +Arguments: + + Instance - Supplies a pointer to the callback instance responsible for this + threadpool callback invocation. + + Ctx - Supplies a pointer to the owning PERFECT_HASH_CONTEXT. + + Timer - Supplies a pointer to the TP_TIMER object for this routine. + +Return Value: + + None. + +--*/ +{ + PPERFECT_HASH_CONTEXT Context; + + UNREFERENCED_PARAMETER(Instance); + UNREFERENCED_PARAMETER(Timer); + + if (!ARGUMENT_PRESENT(Ctx)) { + PH_RAISE(PH_E_INVARIANT_CHECK_FAILED); + } + + Context = (PPERFECT_HASH_CONTEXT)Ctx; + + SetStopSolving(Context); + Context->State.SolveTimeoutExpired = TRUE; + SubmitThreadpoolWork(Context->FinishedWork); + + return; +} + _Use_decl_annotations_ VOID FileWorkCallback( diff --git a/src/PerfectHash/PerfectHashContext.h b/src/PerfectHash/PerfectHashContext.h index eb483535..dcd85d1a 100644 --- a/src/PerfectHash/PerfectHashContext.h +++ b/src/PerfectHash/PerfectHashContext.h @@ -119,11 +119,17 @@ typedef union _PERFECT_HASH_CONTEXT_STATE { ULONG IsTableCreate:1; + // + // When set, indicates a solve timeout expired. + // + + ULONG SolveTimeoutExpired:1; + // // Unused bits. // - ULONG Unused:24; + ULONG Unused:23; }; LONG AsLong; ULONG AsULong; @@ -785,6 +791,7 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_CONTEXT { PTP_CLEANUP_GROUP MainCleanupGroup; PTP_POOL MainThreadpool; PTP_WORK MainWork; + PTP_TIMER SolveTimeout; ULONG MinimumConcurrency; ULONG MaximumConcurrency; diff --git a/src/PerfectHash/PerfectHashErrors.dbg b/src/PerfectHash/PerfectHashErrors.dbg index 88630234..ca129c02 100644 --- a/src/PerfectHash/PerfectHashErrors.dbg +++ b/src/PerfectHash/PerfectHashErrors.dbg @@ -26,6 +26,7 @@ struct { (HRESULT) PH_I_LOW_MEMORY, "PH_I_LOW_MEMORY", (HRESULT) PH_I_OUT_OF_MEMORY, "PH_I_OUT_OF_MEMORY", (HRESULT) PH_I_TABLE_CREATED_BUT_VALUES_ARRAY_ALLOC_FAILED, "PH_I_TABLE_CREATED_BUT_VALUES_ARRAY_ALLOC_FAILED", + (HRESULT) PH_I_SOLVE_TIMEOUT_EXPIRED, "PH_I_SOLVE_TIMEOUT_EXPIRED", (HRESULT) PH_MSG_PERFECT_HASH_ALGO_HASH_MASK_NAMES, "PH_MSG_PERFECT_HASH_ALGO_HASH_MASK_NAMES", (HRESULT) PH_MSG_PERFECT_HASH_USAGE, "PH_MSG_PERFECT_HASH_USAGE", (HRESULT) PH_MSG_PERFECT_HASH_SELF_TEST_EXE_USAGE, "PH_MSG_PERFECT_HASH_SELF_TEST_EXE_USAGE", @@ -362,5 +363,6 @@ struct { (HRESULT) PH_E_TRY_USE_AVX2_HASH_FUNC_FLAG_REQUIRE_HASH_ALL_KEYS_FIRST, "PH_E_TRY_USE_AVX2_HASH_FUNC_FLAG_REQUIRE_HASH_ALL_KEYS_FIRST", (HRESULT) PH_E_TRY_USE_AVX512_HASH_FUNC_FLAG_REQUIRE_HASH_ALL_KEYS_FIRST, "PH_E_TRY_USE_AVX512_HASH_FUNC_FLAG_REQUIRE_HASH_ALL_KEYS_FIRST", (HRESULT) PH_E_INVALID_REMARK, "PH_E_INVALID_REMARK", + (HRESULT) PH_E_INVALID_MAX_SOLVE_TIME_IN_SECONDS, "PH_E_INVALID_MAX_SOLVE_TIME_IN_SECONDS", (HRESULT) 0xFFFFFFFF, NULL }; diff --git a/src/PerfectHash/PerfectHashErrors.mc b/src/PerfectHash/PerfectHashErrors.mc index 6ac6be90..3bac445b 100644 --- a/src/PerfectHash/PerfectHashErrors.mc +++ b/src/PerfectHash/PerfectHashErrors.mc @@ -175,6 +175,14 @@ Language=English The table was created successfully, however, the values array could not be allocated. The table cannot be used. . +MessageId=0x088 +Severity=Informational +Facility=ITF +SymbolicName=PH_I_SOLVE_TIMEOUT_EXPIRED +Language=English +Solve timeout expired. +. + ; ;//////////////////////////////////////////////////////////////////////////////// ;// PH_SEVERITY_INFORMATIONAL -- Usage Messages @@ -844,6 +852,11 @@ Table Create Parameters: be returned if the provided string contains commas (as this will break the .csv output). + --MaxSolveTimeInSeconds= + + Supplies the maximum number of seconds to try and solve an individual + graph. + Console Output Character Legend @@ -858,7 +871,7 @@ Console Output Character Legend were possible (due to the maximum resize limit also being hit). F Failed to create a table due to a target not being reached by a specific - number of attempts or time duration. Not yet implemented. + number of attempts. * None of the worker threads were able to allocate sufficient memory to attempt solving the graph. @@ -873,13 +886,16 @@ Console Output Character Legend V The graph was created successfully, however, we weren't able to allocate enough memory for the table values array in order for the array to be - used after creation. This can be avoided by omitting --TestAfterCreate. + used after creation. This can be avoided by supplying the command line + parameter --SkipTestAfterCreate. - T The requested number of table elements was too large (exceeded 32 bits). + T The requested number of table elements was too large. S A shutdown event was received. This shouldn't be seen unless externally signaling the named shutdown event associated with a context. + t The solve timeout was reached before a solution was found. + . MessageId=0x102 @@ -3682,3 +3698,11 @@ Language=English --Remark must not contain commas. . +MessageId=0x3d4 +Severity=Fail +Facility=ITF +SymbolicName=PH_E_INVALID_MAX_SOLVE_TIME_IN_SECONDS +Language=English +Invalid --MaxSolveTimeInSeconds. +. + diff --git a/src/PerfectHash/PerfectHashErrors_English.bin b/src/PerfectHash/PerfectHashErrors_English.bin index 9e322854b4e22349da4f329f00cdbc549f528514..95860b6ad25f4bdf3d44d12431442c82aecbd692 100644 GIT binary patch delta 656 zcmZ{iQAkr!7{~u-ySp39VA+Lf!n}pV%r<-pBrAG}DDbHf*3-mB4sJk;E7{(rY5(*!JV2dcDA%+AJKJ-5$m_Eee$N#$@=lh@UeCO7de6_c} z`zNPnU?huP0Ln;u$gUsYXXyax{>D61$wQJGj66o=5PA9oI81FPg)QJHHA=dF16|a6 z#NGiOP#=?8tD>n`Yx{9263qjEsruZHCGyyX-wE8{|= zK=+facQuAP)PcH@O{+%7I%o4peMFkKNK+>;Ah%wJC9)cn&F`k1cMNxJec9{GUstBg zzHD+gKRxudj&tKE*D%es!Oi+nLr6K(y{b(Twk2y<4cX|hq+%VHHO5?~#5Z)^L8DW8p)6i1pRwAd`AucnuCbf0{=n3w#v)e}PueY{_8&ujdD Lka7J8`X9qz*3Prg delta 369 zcmW-ZJxD@f6o%j9y=Ig{C6;96E&5SRhgupUXsW5PpUp}Mp%nUoh6_PUW6`&@WYFkf z-KC`{kfx?uf)=k_YPm!ZL_tLFb$Gt#{a(&F?w#Mg_8*Pz9YD{VYXC&g_((MX@LOpm zro$dm@saW?lTWX;lkYbmM90X(JJ3(ZiTMd6=|i&q1sv0-WFjIws@Ec9vRBYeM@eZ| zrqIVQKxRh-gY+5shzl-gkLwHd(&`r&+_JD7zzS{k65+ zTRqP=TEajPn=r7AJT{PH%3_Dja-%7tfHXS;bISfns`*Gc+kVZIL7#{H1@WTP}7|ut5i>mz*=%<%zpwDW{m&< diff --git a/src/PerfectHash/PerfectHashTable.h b/src/PerfectHash/PerfectHashTable.h index 8b8cbd5e..66677da3 100644 --- a/src/PerfectHash/PerfectHashTable.h +++ b/src/PerfectHash/PerfectHashTable.h @@ -416,7 +416,9 @@ typedef struct _Struct_size_bytes_(SizeOfStruct) _PERFECT_HASH_TABLE { ULONG BenchmarkWarmups; ULONG BenchmarkAttempts; ULONG BenchmarkIterations; - ULONG Padding4; + + ULONG MaxSolveTimeInSeconds; + FILETIME64 RelativeMaxSolveTimeInFiletime; TIMESTAMP SlowIndexTimestamp; TIMESTAMP SeededHashTimestamp; @@ -514,6 +516,10 @@ typedef PERFECT_HASH_TABLE *PPERFECT_HASH_TABLE; BIGS(); \ break; \ \ + case PH_I_SOLVE_TIMEOUT_EXPIRED: \ + LITTLET(); \ + break; \ + \ default: \ QUESTION(); \ UnknownTableCreateResult = TRUE; \ diff --git a/src/PerfectHash/PerfectHashTableCreate.c b/src/PerfectHash/PerfectHashTableCreate.c index ff53534a..38f58945 100644 --- a/src/PerfectHash/PerfectHashTableCreate.c +++ b/src/PerfectHash/PerfectHashTableCreate.c @@ -741,6 +741,15 @@ Return Value: Table->Remark = &Param->AsUnicodeString; break; + case TableCreateParameterMaxSolveTimeInSecondsId: + Table->MaxSolveTimeInSeconds = Param->AsULong; + Table->RelativeMaxSolveTimeInFiletime.AsULongLong = (ULONGLONG)( + -1LL * + (LONGLONG)Table->MaxSolveTimeInSeconds * + 10000000LL + ); + break; + case TableCreateParameterNullId: case TableCreateParameterInvalidId: default: diff --git a/src/PerfectHash/Rtl.h b/src/PerfectHash/Rtl.h index 52669214..a57986c6 100644 --- a/src/PerfectHash/Rtl.h +++ b/src/PerfectHash/Rtl.h @@ -104,6 +104,12 @@ typedef const CHAR *PCCHAR; typedef _Null_terminated_ const WCHAR *PCWSZ; typedef const WCHAR *PCWCHAR; +typedef union _FILETIME64 { + FILETIME AsFileTime; + LONGLONG AsLongLong; + ULONGLONG AsULongLong; +} FILETIME64, *PFILETIME64; + #ifndef PAGE_SHIFT #define PAGE_SHIFT 12 #endif diff --git a/src/PerfectHash/RtlOutput.h b/src/PerfectHash/RtlOutput.h index 843dd0ab..ffa07058 100644 --- a/src/PerfectHash/RtlOutput.h +++ b/src/PerfectHash/RtlOutput.h @@ -477,6 +477,7 @@ static PCSZ Dash = "-"; static PCSZ Plus = "+"; static PCSZ Caret = "^"; static PCSZ Cross = "x"; +static PCSZ LittleT = "t"; static PCSZ Percent = "%"; static PCSZ Newline = "\n"; static PCSZ Question = "?"; @@ -510,6 +511,7 @@ static PCSZ Exclamation = "!"; #define DASH() DO_OUTPUT(Dash, 1) #define CARET() DO_OUTPUT(Caret, 1) #define CROSS() DO_OUTPUT(Cross, 1) +#define LITTLET() DO_OUTPUT(LittleT, 1) #define PERCENT() DO_OUTPUT(Percent, 1) #define NEWLINE() DO_OUTPUT(Newline, 1) #define QUESTION() DO_OUTPUT(Question, 1)