//+--------------------------------------------------------------------------- // // Microsoft Windows // Copyright (C) Microsoft Corporation, 1991 - 2000. // // File: RESMAN.HXX // // Contents: Resource Manager // // Classes: CResManager // // History: 4-Apr-91 BartoszM Created. // 4-Jan-95 BartoszM Separated Filter Manager // //---------------------------------------------------------------------------- #pragma once //+--------------------------------------------------------------------------- // // Class: CResManager // // Purpose: Manages resources (threads, memory, disk) // // Interface: CResManager // // History: 4-Apr-91 BartoszM Created // //---------------------------------------------------------------------------- #include #include #include #include #include #include #include "cibackup.hxx" #include "flushnot.hxx" #include "afwwork.hxx" #include "abortwid.hxx" #include "fresh.hxx" #include "partlst.hxx" #include "index.hxx" #include "merge.hxx" #include "wordlist.hxx" #include "keylist.hxx" #include "pqueue.hxx" #include "rmstate.hxx" class CContentIndex; class CPartition; class CDocList; class CEntryBuffer; class CPersIndex; class CMasterMergeIndex; class CGetFilterDocsState; class CFilterAgent; class PStorage; class CPidMapper; class CSimplePidRemapper; class CResManager; class PSaveProgressTracker; BOOL LokCheckIfDiskLow( CResManager & resman, PStorage & storage, BOOL fIsCurrentFull, ICiCAdviseStatus & adviseStatus ); typedef CDynStackInPlace CPartIdStack; const LONGLONG eSigResman = 0x20204e414d534552i64; // "RESMAN" //+--------------------------------------------------------------------------- // // Class: CMergeThreadKiller // // Purpose: An object to kill the merge thread should the constructor of // CResManager fail. // // History: 3-06-95 srikants Created // // Notes: // //---------------------------------------------------------------------------- class CMergeThreadKiller { public: CMergeThreadKiller( CResManager & resman ); ~CMergeThreadKiller(); void Defuse() { _fKill = FALSE; } private: CResManager & _resman; BOOL _fKill; // If set to TRUE, the merge thread must be // killed by this object. }; class CDeletedIIDTrans; //+--------------------------------------------------------------------------- // // Class: CWorkIdToDocName // // Purpose: Converts a objectId (workId) to its corresponding docname // // History: 20-Oct-94 DwightKr Created // 22-Jan-97 SrikantS Changed to generating un-interpreted // BYTES for framework support. // //---------------------------------------------------------------------------- class CWorkIdToDocName { public: CWorkIdToDocName( ICiCDocNameToWorkidTranslator * pITranslator, ICiCDocNameToWorkidTranslatorEx * pITranslator2 ) { Win4Assert( 0 != pITranslator ); ICiCDocName * pIDocName = 0; SCODE sc = pITranslator->QueryDocName( &pIDocName ); if ( S_OK != sc ) THROW( CException( sc ) ); _docName.Set( pIDocName ); pITranslator->AddRef(); _translator.Set( pITranslator ); if ( 0 != pITranslator2 ) { pITranslator2->AddRef(); _translator2.Set( pITranslator2 ); } } const BYTE * GetDocName( WORKID objectId, ULONG & cbDocName, BOOL fAccurate ) { _docName->Clear(); BYTE const * pbBuffer = 0; cbDocName = 0; SCODE sc; if ( fAccurate && !_translator2.IsNull() ) { sc = _translator2->WorkIdToAccurateDocName( objectId, _docName.GetPointer() ); } else sc = _translator->WorkIdToDocName( objectId, _docName.GetPointer() ); if ( S_OK == sc ) { sc = _docName->GetNameBuffer( &pbBuffer, &cbDocName ); Win4Assert( S_OK == sc ); } return pbBuffer; } private: XInterface _translator; XInterface _translator2; XInterface _docName; }; //+--------------------------------------------------------------------------- // // Class: CResManager // // History: 08-Apr-91 BartoszM Created // //---------------------------------------------------------------------------- class CResManager { friend class CFilterManager; friend class CFilterAgent; friend class CMerge; friend class CMasterMerge; friend class CIndexSnapshot; friend class CReleaseMMergeIndex; friend class CIndexTrans; friend class CMasterMergeTrans; friend class CMergeThreadKiller; friend class CDeletedIIDTrans; friend class CNotificationTransaction; friend class CBackupCiPersData; public: CResManager( PStorage& storage, CCiFrameworkParams & params, ICiCDocStore * pICiCDocStore, CI_STARTUP_INFO const & startupInfo, IPropertyMapper * pIPropertyMapper, CTransaction &xact, XInterface & xIndexNotifTable ); ~CResManager(); void RegisterFilterAgent ( CFilterAgent* pFilterAgent ) { _pFilterAgent = pFilterAgent; } static DWORD WINAPI MergeThread( void* self ); // // State Changes // NTSTATUS Dismount(); void Empty(); NTSTATUS ForceMerge ( PARTITIONID partid, CI_MERGE_TYPE mt ); NTSTATUS StopCurrentMerge(); void DisableUpdates( PARTITIONID partid ); void EnableUpdates( PARTITIONID partid ); // // Global Resources // BOOL IsLowOnDiskSpace() const { return _isLowOnDiskSpace; } BOOL VerifyIfLowOnDiskSpace() { CPriLock lock(_mutex); return LokCheckLowOnDiskSpace(); } BOOL LokCheckLowOnDiskSpace () { _isLowOnDiskSpace = LokCheckIfDiskLow(*this, _storage, _isLowOnDiskSpace, _adviseStatus.GetReference() ); return _isLowOnDiskSpace; } BOOL LokCheckWordlistQuotas(); NTSTATUS MarkCorruptIndex(); void NoFailFreeResources(); BOOL IsMemoryLow(); BOOL IsIoHigh(); BOOL IsBatteryLow(); BOOL IsOnBatteryPower(); BOOL IsUserActive( BOOL fCheckLongTermActivity ); void SampleUserActivity(); void IndexSize( ULONG & mbIndex ); // // Statistics and State // BOOL IsEmpty() const { return _isBeingEmptied; } unsigned CountPendingUpdates(unsigned& secCount); inline unsigned CountPendingUpdates() { unsigned temp; return CountPendingUpdates( temp ); } void LokUpdateCounters(); // performance meter void ReferenceQuery(); void DereferenceQuery(); LONG GetQueryCount() const { return _cQuery; } // // Updates and queries // unsigned ReserveUpdate( WORKID wid ); SCODE UpdateDocument( unsigned iHint, WORKID wid, PARTITIONID partid, USN usn, VOLUMEID volumeId, ULONG action); void FlushUpdates(); void MarkUnReachable( WORKID wid ) { _docStore->MarkDocUnReachable( wid ); } CIndex** QueryIndexes ( unsigned cPartitions, PARTITIONID aPartID[], CFreshTest** freshTest, unsigned& cInd, ULONG cPendingUpdates, CCurStack * pcurPending, ULONG* pFlags ); void ReleaseIndexes ( unsigned cInd, CIndex** apIndex, CFreshTest* fresh ); SCODE ClearNonStoragePropertiesForWid( WORKID wid ); // // Backup of Content Index Data. // SCODE BackupCIData( PStorage & storage, BOOL & fFull, XInterface & xEnumWorkids, PSaveProgressTracker & progressTracker ); // // Feedback from downlevel scan // inline void MarkOutOfDate(); inline void MarkUpToDate(); // // Scanning // void SetUpdatesLost( PARTITIONID partId ); // // Notification of ChangeLog flushes support. // void NotifyFlush( FILETIME const & ft, ULONG cEntries, USN_FLUSH_INFO ** aInfo ); void ProcessFlushNotifies(); // KeyIndex specific Methods ULONG KeyToId( CKey const * pkey ); void IdToKey( ULONG ulKid, CKey & rkey ); // Relevant Words CRWStore * ComputeRelevantWords(ULONG cRows,ULONG cRW, WORKID *pwid,PARTITIONID partid); CRWStore * RetrieveRelevantWords(BOOL fAcquire,PARTITIONID partid); NTSTATUS CiState(CIF_STATE & state); void IncrementFilteredDocumentCount() { InterlockedIncrement(&_cFilteredDocuments); } ULONG GetFilteredDocumentCount() const { return _cFilteredDocuments; } CCiFrameworkParams & GetRegParams() { return _frmwrkParams; } SCODE FPSToPROPID( CFullPropSpec const & fps, PROPID & pid ); BOOL FPushFiltering() { Win4Assert( _fPushFiltering == !_xIndexNotifTable.IsNull() ); return _fPushFiltering; } void NoFailAbortWidsInDocList(); void NoFailAbortWid( WORKID wid, USN usn ); BOOL LokIsWidAborted( WORKID wid, USN usn ) { return _aAbortedWids.LokIsWidAborted( wid, usn ); } void LokRemoveAbortedWid( WORKID wid, USN usn ) { _aAbortedWids.LokRemoveWid( wid, usn ); } BOOL IsCiCorrupt() { return _isCorrupt; } # ifdef CIEXTMODE void CiExtDump(void *ciExtSelf); # endif private: // Allow Fiter Manager to lock ResManager CMutexSem& GetMutex () { return _mutex; } void CompactResources(); unsigned LokGetFreshCount() { return _fresh.LokCount(); } inline void LokCommitFresh ( CFreshTest* freshTest ); CPartition* LokGetPartition ( PARTITIONID partid ) { return _partList.LokGetPartition(partid); } // Merges void DoMerges(); void StopMerges(); void CheckAndDoMerge( MergeType mt, CPartIdStack & partitionIds ); BOOL CheckDeleteMerge( BOOL fForce ); void Merge ( MergeType mt, PARTITIONID partid ); void MasterMerge ( PARTITIONID partid ); void LokClearForceMerge() { _partidToMerge = partidInvalid; } void LokReleaseIndex ( CIndex* pIndex ); void LokReplaceMasterIndex( CMasterMergeIndex * pMMergeIndex ); #ifdef KEYLIST_ENABLED CKeyList * _AddRefKeyList(); void _ReleaseKeyList(); #endif // KEYLIST_ENABLED void RollBackDeletedIIDChange( CDeletedIIDTrans & xact ); // Filter related methods CDocList & GetDocList() { return _docList; } BOOL GetFilterDocs ( ULONG cMaxDocs, ULONG & cDocs, CGetFilterDocsState & state ); BOOL LokGetFilterDocs( ULONG cMaxDocs, ULONG & cDocs, CGetFilterDocsState & state ); void FillDocBuffer ( BYTE * docBuffer, ULONG & cb, ULONG & cDocs, CGetFilterDocsState & state ); BOOL NoFailReFile ( CDocList& docList ); BOOL ReFileCommit ( CDocList& docList ); void NoFailReFileChanges( CDocList& docList ); void LokNoFailReFileChanges( CDocList& docList ); void LokReFileDocument ( PARTITIONID partid, WORKID wid, USN usn, VOLUMEID volumeId, ULONG retries, ULONG cSecQRetries ); void LokAddToSecQueue( PARTITIONID partid, WORKID wid, VOLUMEID volumeId, ULONG cSecQRetries ); void LokRefileSecQueueDocs(); BOOL LokTransferWordlist ( PWordList& pWordlist ); void LokNoFailRemoveWordlist( CPartition * pPart ); INDEXID LokMakeWordlistId (PARTITIONID partid); void ReportFilterFailure (WORKID wid); SCODE StoreValue( WORKID wid, CFullPropSpec const & ps, CStorageVariant const & var, BOOL & fInSchema ); BOOL StoreSecurity( WORKID wid, PSECURITY_DESCRIPTOR pSD, ULONG cbSD); void ReportFilteringState( DWORD dwState ) { CPriLock lock( _mutex ); _dwFilteringState = dwState; } // // Scan and Recovery support // void LokCleanupForFullCiScan(); // // Misc utility methods. // static BOOL IsDiskFull( NTSTATUS status ) { return STATUS_DISK_FULL == status || ERROR_DISK_FULL == status || HRESULT_FROM_WIN32(ERROR_DISK_FULL) == status || CI_E_CONFIG_DISK_FULL == status; } static BOOL IsCorrupt( NTSTATUS status ) { return CI_CORRUPT_DATABASE == status || STATUS_INTEGER_DIVIDE_BY_ZERO == status || STATUS_ACCESS_VIOLATION == status || STATUS_DATATYPE_MISALIGNMENT == status || STATUS_IN_PAGE_ERROR == status; } static void LogMMergeStartFailure( BOOL fReStart, PStorage & storage, DWORD dwError, ICiCAdviseStatus & adviseStatus ); static void LogMMergePaused( PStorage & storage, DWORD dwError, ICiCAdviseStatus & adviseStatus); // // Interfacing with the framework client // BOOL WorkidToDocName ( WORKID wid, BYTE * pbBuf, unsigned & cb, BOOL fAccurate ); void GetClientState( ULONG & cDocuments ); void LokNotifyCorruptionToClient(); void LokNotifyDisableUpdatesToClient(); void LokNotifyEnableUpdatesToClient(); BOOL LokIsScanNeeded() const { return _isCorrupt || _state.LokGetState() == eUpdatesToBeDisabled || _state.LokGetState() == eUpdatesDisabled || _state.LokGetState() == eUpdatesToBeEnabled; } void LokWakeupMergeThread() { _eventMerge.Set(); } void LokDeleteFlushWorkItems(); BOOL IsIndexMigrationEnabled() const { return _configFlags & CI_CONFIG_ENABLE_INDEX_MIGRATION; } BOOL IsIndexingEnabled() const { return _configFlags & CI_CONFIG_ENABLE_INDEXING; } BOOL IsQueryingEnabled() const { return _configFlags & CI_CONFIG_ENABLE_QUERYING; } CFresh & GetFresh() { return _fresh; } PIndexTable & GetIndexTable() { return *_idxTab; } BOOL LokIsMasterMergeInProgress( PARTITIONID partId = partidDefault ) { CPartition *pPartition = _partList.LokGetPartition ( partId ); return pPartition->InMasterMerge(); } BOOL LokIsMasterIndexPresent( PARTITIONID partId = partidDefault ) { CPartition *pPartition = _partList.LokGetPartition ( partId ); return 0 != pPartition->GetCurrentMasterIndex(); } void BackupContentIndexData(); void LokUpdateBackupMergeProgress(); ULONG GetStorageVersion() { return _storage.GetStorageVersion(); } //----------------------------------------------- // This MUST be the first variable in this class. //----------------------------------------------- const LONGLONG _sigResman; PStorage& _storage; CMutexSem _mutex; ULONGLONG _mergeTime; // Time to next master merge XInterface _xLowRes; // Monitors resource conditions XInterface _xLowResDefault; // Monitors resource conditions //ULONG _ulPagesPerMeg; // System pages per Meg (varies with system page size) //ULONG _ulPageSize; // System page size // // Performance Object // Note : Must be initialized before CFilterDaemon // and destroyed after the partition list. // CCiFrameworkParams & _frmwrkParams; XPtr _sKeyList; // Safe pointer to the keylist object SIndexTable _idxTab; // index table CPartList _partList; // list of partitions CFresh _fresh; // table of fresh indexes // MUST BE after _partList. CPartIter _partIter; // partition iterator CEventSem _eventMerge; PARTITIONID _partidToMerge; // force merge of this partition CI_MERGE_TYPE _mtForceMerge; // type of merge CMerge * _pMerge; BOOL _fStopMerge; long _cQuery; long _cFilteredDocuments; // # filtered documents since boot CFilterAgent* _pFilterAgent; // // Queue used to properly order notifications coming from Cleanup. // CPendingQueue _pendQueue; // // Documents that are currently being filtered by the daemon. // CDocList _docList; // // In-progress backup operation. // CBackupCiWorkItem * _pBackupWorkItem; BOOL _isBeingEmptied; BOOL _isLowOnDiskSpace; BOOL _isDismounted; BOOL _isOutOfDate; BOOL _isCorrupt; BOOL _fFirstTimeUpdatesAreEnabled; // Is this the first time that // updates have been enabled after // startup ? This is used in push // filtering to initialize in-memory // part of changelog. BOOL _fPushFiltering; // Push model of filtering ? const CI_STARTUP_FLAGS _configFlags; // Content Index flags CResManState _state; DWORD _dwFilteringState; // State of the Filter Manager (for ::CiState) // // Framework interfacing. // XInterface _docStore; XInterface _translator; XInterface _translator2; XInterface _adviseStatus; XInterface _propStore; XInterface _mapper; XInterface _xIndexNotifTable; // Table of notifications CAbortedWids _aAbortedWids; // List of aborted wids // // Mutex to guard the flush notify list and the flush notify list. // BOOL _fFlushWorkerActive; CMutexSem _mtxChangeFlush; CChangesNotifyFlushList _flushList; // // WorkQueue manager for the async work items. // CWorkManager _workMan; // Work Queue Manager // // This mutex is for the workIdToDocName only. It is too expensive // to hold resman lock while doing the wid->docname conversion. // CMutexSem _workidToDocNameMutex; XPtr _xWorkidToDocName; CCiFrmPerfCounter _prfCounter; // // Be sure the threads are created *after* any resources they use. // CThread _thrMerge; CMergeThreadKiller _mergeKiller; CDeletedIndex _activeDeletedIndex; }; inline void CResManager::NoFailReFileChanges ( CDocList& docList ) { CPriLock lock(_mutex); LokNoFailReFileChanges(docList); } inline BOOL CResManager::GetFilterDocs ( ULONG cMaxDocs, ULONG & cDocs, CGetFilterDocsState & state ) { CPriLock lock (_mutex); return LokGetFilterDocs ( cMaxDocs, cDocs, state ); } inline void CResManager::MarkOutOfDate() { CPriLock lock (_mutex); _isOutOfDate = TRUE; } inline void CResManager::MarkUpToDate() { CPriLock lock (_mutex); _isOutOfDate = FALSE; } inline void CResManager::ReferenceQuery() { InterlockedIncrement(&_cQuery); } inline void CResManager::DereferenceQuery() { Win4Assert( _cQuery > 0 ); InterlockedDecrement(&_cQuery); } //+--------------------------------------------------------------------------- // // Function: ForceMerge, public // // Synopsis: Forces a master merge on the partition specified // // Arguments: [partid] -- partition ID to merge // [mt] -- merge type (master, shadow, etc.) // // History: 16-Nov-94 DwightKr Added this header // //---------------------------------------------------------------------------- inline NTSTATUS CResManager::ForceMerge ( PARTITIONID partid, CI_MERGE_TYPE mt ) { _partidToMerge = partid; _mtForceMerge = mt; _eventMerge.Set(); // wake up return STATUS_SUCCESS; } //+--------------------------------------------------------------------------- // // Member: CResManager::LokCommitFresh, public // // Arguments: [pFreshTest] -- fresh test to be committed // // History: 08-Oct-91 BartoszM Created // // Notes: Commits a new fresh test, All transactions are under resman lock // //---------------------------------------------------------------------------- inline void CResManager::LokCommitFresh( CFreshTest* pFreshTest ) { _fresh.LokCommitMaster(pFreshTest); } //+--------------------------------------------------------------------------- // // Class: CMasterMergePolicy // // Purpose: Encapsulates all rules used to determine if it is time to // automatically trigger a master merge. // // History: 03-Aug-94 DwightKr Created // //---------------------------------------------------------------------------- class CMasterMergePolicy { public : CMasterMergePolicy( PStorage & storage, __int64 & shadowIndexSize, unsigned freshCount, ULONGLONG & mergeTime, CCiFrameworkParams & regParams, ICiCAdviseStatus & adviseStatus) : _frmwrkParams( regParams ), _storage(storage), _shadowIndexSize(shadowIndexSize), _freshCount(freshCount), _mergeTime(mergeTime), _adviseStatus(adviseStatus) {} BOOL IsTimeToMasterMerge() const; BOOL IsMidNightMergeTime(); static ULONGLONG ComputeMidNightMergeTime( LONG mergeTime ); private: BOOL IsMinDiskFreeForceMerge() const; BOOL IsMaxShadowIndexSize() const; BOOL IsMaxFreshListCount() const; PStorage & _storage; __int64 & _shadowIndexSize; unsigned _freshCount; ULONGLONG & _mergeTime; CCiFrameworkParams & _frmwrkParams; ICiCAdviseStatus & _adviseStatus; };