CAcIFFileSystem Class Reference

#include <CAcIFFileSystem.h>

Inheritance diagram for CAcIFFileSystem:

List of all members.

Public Member Functions
bool	operator() () const
	CAcIFFileSystem (const CXMLElement &inCollectionElement)
bool	init (bool)
	~CAcIFFileSystem ()
string	IDToURL (TID inID) const
virtual pair< bool, TID >	URLToID (const string &inURL) const
void	getAllIDs (list< TID > &) const
void	getAllAccessorElements (list< CAccessorElement > &) const
void	getRandomIDs (list< TID > &, list< TID >::size_type) const
void	getRandomAccessorElements (list< CAccessorElement > &outResult, list< CAccessorElement >::size_type inSize) const
int	size () const
TID	getMaximumFeatureID () const
list< TID > *	getAllFeatureIDs () const
virtual pair< bool, CAccessorElement >	IDToAccessorElement (TID inID) const
	operator bool () const
The proper inverted file access
CDocumentFrequencyList *	FeatureToList (TFeatureID) const
CDocumentFrequencyList *	URLToFeatureList (string inURL) const
CDocumentFrequencyList *	DIDToFeatureList (TID inDID) const
Accessing information about features
double	FeatureToCollectionFrequency (TFeatureID) const
unsigned int	getFeatureDescription (TID inFeatureID) const
Accessing additional document information
double	DIDToMaxDocumentFrequency (TID) const
double	DIDToDFSquareSum (TID) const
	returns the Document frequency squaresum for a given document ID
double	DIDToSquareDFLogICFSum (TID) const
bool	generateInvertedFile ()
bool	newGenerateInvertedFile ()
bool	checkConsistency ()
bool	findWithinStream (TID inFeatureID, TID inDocumentID, double inDocumentFrequency) const
	needed for checking the consistency
Protected Types
typedef HASH_MAP< TID, streampos >	CIDToOffset
Protected Member Functions
void	writeOffsetFileElement (TID inFeatureID, streampos inPosition, ostream &inOpenOffsetFile)
CDocumentFrequencyList *	getFeatureFile (string inFileName) const
Protected Attributes
CMutex	mMutex
CSelfDestroyPointer< CAcURL2FTS >	mURL2FTS
TID	mMaximumFeatureID
string	mInvertedFileBuffer
string	mTemporaryIndexingFileBase
CSelfDestroyPointer< istream >	mInvertedFile
ifstream	mOffsetFile
ifstream	mFeatureDescriptionFile
string	mInvertedFileName
string	mOffsetFileName
string	mFeatureDescriptionFileName
CIDToOffset	mIDToOffset
HASH_MAP< TID, double >	mFeatureToCollectionFrequency
for fast access...
HASH_MAP< TID, unsigned int >	mFeatureDescription
CADIHash	mDocumentInformation

---

Detailed Description

An accessor to an inverted file. This access is done "by hand".

For a long time we wanted to move to memory mapped files (like SWISH++) but currently I think this is not the best idea.

Definition at line 93 of file CAcIFFileSystem.h.

---

Member Typedef Documentation

typedef HASH_MAP<TID,streampos> CAcIFFileSystem::CIDToOffset [protected]

map from feature id to the offset for this feature

Reimplemented from CAcInvertedFile.

Definition at line 138 of file CAcIFFileSystem.h.

---

Constructor & Destructor Documentation

CAcIFFileSystem::CAcIFFileSystem

(

const CXMLElement &

inCollectionElement

)

This opens an exsisting inverted file, and then inits this structure. After that it is fully usable

As a paramter it takes an XMLElement which contains a "collection" element and its content.

If the attribute cui-generate-inverted-file is true, then a new inverted file will be generated using the parameters given in inCollectionElement. you will NOT be able to use *this afterwards.

Like every accessor, this accessor takes a <collection> MRML element as input (

See also:

CXMLElement for how to access the attributes of this element). Currently this accessor understands the following attributes

cui-base-dir: the directory containing the following files cui-inverted-file-location: the location of the inverted file cui-offset-file-location: a file containing offsets into the inverted file cui-feature-file-location: the location of the "url2fts" file which translates urls to feature file names.

Definition at line 769 of file CAcIFFileSystem.cc.

References CXMLElement::boolReadAttribute(), mrml_const::cui_base_dir, mrml_const::cui_feature_file_location, mrml_const::cui_generate_inverted_file, mrml_const::cui_in_memory, mrml_const::cui_inverted_file_location, mrml_const::cui_offset_file_location, init(), CMutex::lock(), mFeatureDescriptionFileName, mInvertedFileName, mOffsetFileName, mURL2FTS, CAccessorImplementation::mURLToID, newGenerateInvertedFile(), CXMLElement::stringReadAttribute(), and CMutex::unlock().

                                                                      :
  mURL2FTS(new CAcURL2FTS(inCollectionElement)),
  mOffsetFileName(inCollectionElement.stringReadAttribute(mrml_const::cui_base_dir).second
      +inCollectionElement.stringReadAttribute(mrml_const::cui_offset_file_location).second),
  mInvertedFileName(inCollectionElement.stringReadAttribute(mrml_const::cui_base_dir).second
        +inCollectionElement.stringReadAttribute(mrml_const::cui_inverted_file_location).second),
  mFeatureDescriptionFileName(inCollectionElement.stringReadAttribute(mrml_const::cui_base_dir).second
            +inCollectionElement.stringReadAttribute(mrml_const::cui_feature_description_location).second),
  mTemporaryIndexingFileBase(inCollectionElement.stringReadAttribute(mrml_const::cui_base_dir).second),
  mInvertedFile(0),
#ifdef V295
  mInvertedFileBuffer(0),
#else
  mInvertedFileBuffer(""),
#endif
  mMaximumFeatureID(0){
  gMutex->lock();

  if(inCollectionElement.stringReadAttribute(mrml_const::cui_generate_inverted_file).first
     && inCollectionElement.boolReadAttribute(mrml_const::cui_generate_inverted_file).second){
    newGenerateInvertedFile();
  }else{

    
  bool lSuccessfulStart(inCollectionElement.stringReadAttribute(mrml_const::cui_base_dir).first
      && inCollectionElement.stringReadAttribute(mrml_const::cui_feature_file_location).first
      && inCollectionElement.stringReadAttribute(mrml_const::cui_offset_file_location).first
      && inCollectionElement.stringReadAttribute(mrml_const::cui_inverted_file_location).first);
  
  if(lSuccessfulStart 
     && init(inCollectionElement.boolReadAttribute(mrml_const::cui_in_memory).first
       && inCollectionElement.boolReadAttribute(mrml_const::cui_in_memory).second)){
    cout << "CInvertedFile succuessfully initialised. Parameters:"
   << mInvertedFileName << endl
   << mOffsetFileName << endl
   << mURL2FTS->getURLToFeatureFileName() << endl
   << mFeatureDescriptionFileName << endl
   << endl;

#ifndef _NO_PRINT_INIT
    {
      cout << "DIAGNOSE" << flush << endl;
      cout << "The current size of mURLToID is " << mURLToID.size() << endl;
      cout << "All elements:" 
     << endl;
      int lCount=1;
      for(string_TID_map::const_iterator i=mURLToID.begin();
    i!=mURLToID.end();
    i++){
  cout << ","
       << lCount++
       << flush;
      }
      lCount=1;
      for(string_TID_map::const_iterator i=mURLToID.begin();
    i!=mURLToID.end();
    i++){
  cout << lCount++
       << ":"
       << flush
       << i->first
       << "->"
       << i->second
       << flush
       << endl 
       << flush;
      }
    }
#endif


  };
#ifndef _NO_PRINT_INIT
  checkNPrint();
  cout << "Constructor left"
       << endl;
#endif
}
  gMutex->unlock();
}

CAcIFFileSystem::~CAcIFFileSystem

(

)

Destructor

Definition at line 1748 of file CAcIFFileSystem.cc.

References CMutex::lock(), and CMutex::unlock().

                                 {
  cout << "CAcIFFileSystem::~CAcIFFileSystem() called "
       << endl
       << flush;
  gMutex->lock();
  gMutex->unlock();
};

---

Member Function Documentation

void CAcIFFileSystem::writeOffsetFileElement	(	TID	inFeatureID,
		streampos	inPosition,
		ostream &	inOpenOffsetFile
	)			[protected]

add a pair of FeatureID,Offset to the open offset file (helper function for inverted file construction)

add a pair of FeatureID,Offset to the open offset file (helper function for inverted file construction)

used for reading the offset file

Definition at line 415 of file CAcIFFileSystem.cc.

References CMutex::lock(), SUIntStreampos::mStreampos, SUIntStreampos::mUInt1, and CMutex::unlock().

Referenced by generateInvertedFile(), and newGenerateInvertedFile().

                                         {
  gMutex->lock();
  SUIntStreampos lWriter;

  lWriter.mUInt1=inFeatureID;
  lWriter.mStreampos=inPosition;
#ifndef _NO_PRINT_OFFSET_CHECK
  lWriter.mUInt2=int(inPosition);
#endif

//   inOpenOffsetFile.write((char*)&inFeatureID,
//       sizeof(TID));
//   inOpenOffsetFile.write((char*)&inPosition,
//       sizeof(streampos));
  inOpenOffsetFile.write((char*)&lWriter,
       sizeof(lWriter));
  std::cout << "[inFeatureID:" << sizeof(TID) << "/" << inFeatureID
      << ";inPosition:" << sizeof(streampos) << "/" << inPosition << "==" << int(inPosition)
      << "]" << sizeof(lWriter) << endl;
  gMutex->unlock();
  
};

CDocumentFrequencyList * CAcIFFileSystem::getFeatureFile

(

string

inFileName

)

const [protected]

loads a *.fts file. and returns the feature list

Reimplemented from CAcInvertedFile.

Definition at line 1592 of file CAcIFFileSystem.cc.

References CMutex::lock(), CDocumentFrequencyList::readBinary(), and CMutex::unlock().

Referenced by DIDToFeatureList(), and URLToFeatureList().

                                                                             {
  gMutex->lock();
  CDocumentFrequencyList* lRetVal(0);
  /* if the filename has a size bigger than one, meaning it is defined */
  if(inFileName.size())
    {
      /* a file with the given file name */
      ifstream lFile(inFileName.c_str());
      unsigned int lNumberOfFeatures(0);
      if(lFile){
  lFile.read((char*)&lNumberOfFeatures,
       sizeof(lNumberOfFeatures));
      
  if(lFile && (lRetVal=new CDocumentFrequencyList(lNumberOfFeatures))){
  
    lRetVal->readBinary(lFile);
    
  }
      }
    } /* end of if the URL was proper */
  gMutex->unlock();
  return lRetVal;
}

bool CAcIFFileSystem::operator()

(

)

const [virtual]

for testing if the inverted file is correctly constructed

Implements CAcInvertedFile.

Definition at line 130 of file CAcIFFileSystem.cc.

References CMutex::lock(), mInvertedFile, mOffsetFile, mURL2FTS, and CMutex::unlock().

                                     {
  gMutex->lock();
  bool lReturnValue(mURL2FTS
        && mInvertedFile
        && *mInvertedFile
        && mOffsetFile
        && mURL2FTS->operator bool());
  gMutex->unlock();
  return lReturnValue;
};

bool CAcIFFileSystem::init

(

bool

inMemory

)

called by constructors

Reimplemented from CAcInvertedFile.

Definition at line 862 of file CAcIFFileSystem.cc.

References CIFListStart::getCollectionFrequency(), CIFListStart::getFeatureID(), CMutex::lock(), mDocumentInformation, mFeatureDescription, mFeatureDescriptionFile, mFeatureDescriptionFileName, mFeatureToCollectionFrequency, mIDToOffset, mInvertedFile, mInvertedFileBuffer, mInvertedFileName, mMaximumFeatureID, mOffsetFile, mOffsetFileName, mURL2FTS, and CMutex::unlock().

Referenced by CAcIFFileSystem(), and checkConsistency().

{
  gMutex->lock();
  try{
    mMaximumFeatureID=0;
    cout << "Opening _"
   << mInvertedFileName        
   << "_";
    if(inMemory){
      ifstream lInvertedFile(mInvertedFileName.c_str());
      if(lInvertedFile){
  
  cout << endl 
       << "(TRYING TO READ THE WHOLE FILE INTO MEMORY"
       << endl;
  
  lInvertedFile.seekg(0,ios::end);
  size_t lFileSize=lInvertedFile.tellg();
  
#ifdef V295
  mInvertedFileBuffer=new char[lFileSize];
#else
  mInvertedFileBuffer="x";
  mInvertedFileBuffer.resize(lFileSize);
#endif
  lInvertedFile.seekg(0,ios::beg);
#ifdef V295
  lInvertedFile.read(mInvertedFileBuffer,
         lFileSize);
#else
  // this is a kludge but speed does not matter here
  for(int i=0;i<lFileSize && lInvertedFile;i++){
    char lChar;
    lInvertedFile.get(lChar);
    mInvertedFileBuffer[i]=lChar;
  }
#endif
  
  mInvertedFile=new istringstream(mInvertedFileBuffer);
  // lFileSize does not need to be given
  cout << "DONE)"
       << endl;
      }else{
  mInvertedFile=0;
      }
    }

  mInvertedFile=new ifstream(mInvertedFileName.c_str());
  if(!(*mInvertedFile)){
    cout << " ...FAILED:" << strerror(errno) << endl;
  }else{
    cout << " ...success. " << endl;
  }

  cout << "Opening _"
       << mOffsetFileName 
       << "_";
  mOffsetFile.close();
  mOffsetFile.clear();
  mOffsetFile.open(mOffsetFileName.c_str());
  if(!mOffsetFile){
    cout << " FAILED!" << strerror(errno) << endl;
  }else{
    cout << " ...success. " << endl;
  }


  cout << "Opening _"
       << mFeatureDescriptionFileName
       << "_";
  mFeatureDescriptionFile.close();
  mFeatureDescriptionFile.clear();
  mFeatureDescriptionFile.open(mFeatureDescriptionFileName.c_str());
  if(!mFeatureDescriptionFile){
    cout << " ...FAILED!" << strerror(errno) << endl;
  }else{
    cout << " ...success. " << endl;
  }



  bool lRetVal=(mURL2FTS->operator bool()
    //was the superclass well constructed?
    &&
    mFeatureDescriptionFile
    &&
    mOffsetFile
    &&
    *mInvertedFile);

  assert(mURL2FTS->size());

  cout << endl
       << "Current success status"
       << lRetVal
       << endl;

  {
    list<CAccessorElement> lAllAccessorElements;
    
    mURL2FTS->getAllAccessorElements(lAllAccessorElements);

    //for each element in the database
    for(list<CAccessorElement>::const_iterator i=lAllAccessorElements.begin();
  i!=lAllAccessorElements.end();
  i++){
      TID    lID =i->getID();
      string lURL=i->getURL();

      pair<bool,string> lFeatureFileName=mURL2FTS->URLToFFN(lURL);

      assert(lFeatureFileName.first);
      
      mDocumentInformation
  .insert(make_pair(lID,
        CAdditionalDocumentInformation(lFeatureFileName.second)));
      
      lRetVal = 
  lRetVal && mDocumentInformation[lID].input();
      
    }
  }

  cout << "URLFile " 
       << mURL2FTS->getURLToFeatureFileName() 
       << " processed. Current success status"
       << lRetVal
       << endl;

  /* erase the offset file */
  mIDToOffset.erase(mIDToOffset.begin(),
        mIDToOffset.end());

  mOffsetFile.seekg(0);


  /* while there is no end of file for the offsets */
  while(mOffsetFile){
    SUIntStreampos lVal;

    /* read a value into the offset file */
    mOffsetFile.read((char*)&lVal,sizeof(lVal));
    

    if(mOffsetFile){


      // actually useful code here! ;-I
      unsigned int lFeatureID=lVal.mUInt1;
      streampos lOffset(lVal.mStreampos);//FIXME streampos
      mMaximumFeatureID=(mMaximumFeatureID < lFeatureID)?lFeatureID:mMaximumFeatureID;
      mIDToOffset[lFeatureID]=lOffset;

#ifndef _NO_PRINT_OFFSET_CHECK
      // hexdump of the characters read
      for(int i=0;i<sizeof(lVal);i++){
  cout << hex << (unsigned int)(((unsigned char*)&lVal)[i]) << "/" << dec << flush;
      }
      

      // value different from check value?
      if((unsigned int)(lVal.mStreampos) != lVal.mUInt2){
  cout << "Size:" << sizeof(lVal) << ":"  << lVal.mUInt1 << ":" << lVal.mStreampos << "!=" << lVal.mUInt2 << "!!!" << long(mOffsetFile.tellg()) << endl;
      }
      assert((unsigned int)(lVal.mStreampos) == lVal.mUInt2);

      // the actual values
      cout << "[read:" << sizeof(lVal) << ":inFeatureID:" << flush
     << hex
     << lFeatureID << "==" << dec << lFeatureID
     << ";inPosition:"
     << dec
     << lOffset
     << "/"
     << lVal.mStreampos
     << "]" << long(mOffsetFile.tellg()) << endl;
#endif
      //Reading the offsetfile
      { //move to the right position
  mInvertedFile->seekg(lOffset);
  
  //read the list start chunk 
  //(by constructing an instance of the list start cunk)
  CIFListStart lListStart(*mInvertedFile);
  
  //checking it
  if(lListStart.getFeatureID()!=lFeatureID){
    cout << "[ERROR" << flush
         << hex
         << lFeatureID
         << ","
         << hex
         << lListStart.getFeatureID()
         << dec
         << "]" << flush;
    assert(0);
  }
#ifndef _NO_PRINT_OFFSET_CHECK
  else{
    cout << "-" << flush;
  }
#endif

  //And setting up the translation table from feature 
  //ID to collection frequency
  mFeatureToCollectionFrequency[lListStart.getFeatureID()]=
    lListStart.getCollectionFrequency();
  
  /*      assert(mFeatureToCollectionFrequency[lListStart.getFeatureID()]);*/
  
      }
#ifndef _NO_CHECK_OFFSET_FILE
#endif
    } // if the file is valid --> ID is valid!
  }

  cout << "OffsetFile " 
       << mOffsetFileName
       << " processed. Current success status: "
       << lRetVal
       << endl;

  {
    /* as long as there is no end of file */
    while(mFeatureDescriptionFile){
      TID lFeatureID;
      unsigned int lType;

      
      mFeatureDescriptionFile >> lFeatureID 
            >> lType;

      mFeatureDescription[lFeatureID]=lType;
    } /* end of while */

  }

  cout << "FeatureDescriptionFile " 
       << mFeatureDescriptionFileName
       << " processed: "
       << mFeatureDescription.size()
       << " elements in hash."
       << endl
       << "Initialisation successful? Returning "
       << lRetVal
       << endl;

#ifndef _NO_PRINT_INIT
  checkNPrint();
#endif
  gMutex->unlock();
  return lRetVal;
  }catch(...){
    cerr << "caught here " << endl;
  }

};

string CAcIFFileSystem::IDToURL

(

TID

inID

)

const [virtual]

Translate a DocumentID to a URL (for output)

Implements CAcInvertedFile.

Definition at line 1682 of file CAcIFFileSystem.cc.

References mrml_const::error, CMutex::lock(), mURL2FTS, and CMutex::unlock().

{
  gMutex->lock();

  pair<bool,CAccessorElement> lElement=mURL2FTS->IDToAccessorElement(inID);
  
  if(lElement.first){
    gMutex->unlock();
    return lElement.second.getURL();
  } else {
    cerr << "Error in Conversion from ID "
   << inID 
   << " to URL."
   << endl;
    gMutex->unlock();
    return mrml_const::error;
  }
}

CDocumentFrequencyList * CAcIFFileSystem::FeatureToList

(

TFeatureID

inFeatureID

)

const [virtual]

List of documents containing the feature

Implements CAcInvertedFile.

Definition at line 1252 of file CAcIFFileSystem.cc.

References CIFListStart::getFeatureID(), CIFListStart::getNumberOfElements(), CMutex::lock(), mIDToOffset, mInvertedFile, CDocumentFrequencyList::readBinary(), and CMutex::unlock().

{
  gMutex->lock();
    CDocumentFrequencyList* lRetVal=0;

    
    {
      mInvertedFile->clear();


      //Find the list of URL-IDs for the feature
      if(mIDToOffset.find(inFeatureID)!=mIDToOffset.end()){
  mInvertedFile->seekg((*mIDToOffset.find(inFeatureID)).second);
        assert(*mInvertedFile);
  /* if the inverted file has been able to be opened */
  if(*mInvertedFile){

    //read the beginning chunk of the list;
    CIFListStart lListStart(*mInvertedFile);

    //
    lRetVal=new CDocumentFrequencyList(lListStart.getNumberOfElements());

    if(lListStart.getFeatureID()!=inFeatureID){
      cerr << "Feature "
     << hex
     << inFeatureID
     << " not found.";
      gMutex->unlock();
      return 0;
    }
    
#ifndef _NO_DIDPRINT
    cout << endl;
#endif


    lRetVal->readBinary(*mInvertedFile);


#ifndef _NO_DIDPRINT
    cout << endl;
#endif
  }
      }else{
  cerr << "II:Feature "
       << hex
       << inFeatureID
       << " not found."
       << mIDToOffset.size()
       << endl;
  gMutex->unlock();
  return 0;
      }
    }
    gMutex->unlock();
    return lRetVal;
};

CDocumentFrequencyList * CAcIFFileSystem::URLToFeatureList

(

string

inURL

)

const [virtual]

List of features contained by a document

Implements CAcInvertedFile.

Definition at line 1511 of file CAcIFFileSystem.cc.

References DIDToFeatureList(), getFeatureFile(), CMutex::lock(), mURL2FTS, my_assert, CMutex::unlock(), and URLToID().

Referenced by checkConsistency().

{
  gMutex->lock();

#ifdef PRINT_ADI
  cout <<inURL
       << "(ADI:" ;
#endif

  pair<bool,TID> lID=URLToID(inURL);

  if(!lID.first){//i.e. the URL is not part of the collection

    CDocumentFrequencyList* lReturnValue(0);
#ifdef IGNORE_UNKNOWN_URLS
#else
    pid_t lPID= getpid();
    
    char lFeatureFileName[30];
    char lThumbnailName[30];

    int lTime = int(time(0));
    int lRandomId = random();// in order to make sure that no two images have the same ID

    sprintf(lFeatureFileName,"/tmp/testFTS-%d-%d-%d.fts",int(lPID),lTime,lRandomId);
    sprintf(lThumbnailName,"/tmp/testThumbnail-%d-%d-%d.jpg",int(lPID),lTime,lRandomId);

    //FIXME this is a potential security leak.
    system(string(string(__PERL_LOCATION__)+ " " + string(__EXECBINDIR__)+ "/gift-url-to-fts.pl "+ inURL +" "+ lFeatureFileName + " " + lThumbnailName).c_str());

    // lThumbnailName exists, but is probably less accessible than the initial URL
    mURL2FTS->addImage(inURL,inURL,lFeatureFileName);
    
    lReturnValue=this->getFeatureFile(lFeatureFileName);
 
#endif
    if(!lReturnValue){
      lReturnValue=new CDocumentFrequencyList();
    }
    gMutex->unlock();
    return lReturnValue;
    // {
    //       cout << endl << "this= " << this << endl;
    //       checkNPrint();
    //       cout << "could not find ID for URL "
    //     << inURL << endl;
    //       cout << "The current size of mURLToID is " << mURLToID.size() << endl;
    //       cout << "All elements:" 
    //     << endl;
    //       int lCount=1;
    //       for(string_TID_map::const_iterator i=mURLToID.begin();
    //    i!=mURLToID.end();
    //    i++){
    //  cout << ","
    //       << lCount++
    //       << flush;
    //       }
    //       lCount=1;
    //       for(string_TID_map::const_iterator i=mURLToID.begin();
    //    i!=mURLToID.end();
    //    i++){
    // cout << lCount++
    //       << ":"
    //       << flush
    //       << i->first
    //       << "->"
    //       << i->second
    //       << flush
    //       << endl 
    //       << flush;
    //       }
    //     }
  }
  my_assert(lID.first,inURL.c_str());   

  gMutex->unlock();
  return DIDToFeatureList(lID.second);
};

CDocumentFrequencyList * CAcIFFileSystem::DIDToFeatureList

(

TID

inDID

)

const [virtual]

List of features contained by a document with ID inDID

Implements CAcInvertedFile.

Definition at line 1628 of file CAcIFFileSystem.cc.

References getFeatureFile(), CMutex::lock(), mDocumentInformation, mURL2FTS, CADIHash::output(), and CMutex::unlock().

Referenced by URLToFeatureList().

                                               {
  gMutex->lock();

#ifdef PRINT_ADI
  cout <<inURL
       << "(ADI:" ;

  //FIXME this is temporal. The following two lines should be put back in
  CADIHash::const_iterator iADI=mDocumentInformation.find(inDID);
  assert(iADI!=mDocumentInformation.end());
  //FIXME I am not quite clear if, when and where the ADI is generated.
  //FIXME fix on demand.
#endif

#ifdef PRINT_ADI
  iADI->second.output(cout);

  cout << ":ADI)"
       << endl;
#endif


  CDocumentFrequencyList* lRetVal=0;
  
  /* tests if URL is found */
  pair<bool,string> lFeatureFileName=mURL2FTS->IDToFFN(inDID);
  if(lFeatureFileName.first){
      
    lRetVal=this->getFeatureFile(lFeatureFileName.second);
    

  } /* end of the if statement */

  if(!lRetVal){
    lRetVal=new CDocumentFrequencyList(0);
  }
  gMutex->unlock();
  return lRetVal;
};

double CAcIFFileSystem::FeatureToCollectionFrequency

(

TFeatureID

inFeatureID

)

const [virtual]

Collection frequency for a given feature

Implements CAcInvertedFile.

Definition at line 1134 of file CAcIFFileSystem.cc.

References CMutex::lock(), mFeatureToCollectionFrequency, and CMutex::unlock().

                                                         {
  gMutex->lock();

  if((mFeatureToCollectionFrequency.find(inFeatureID))!=
     mFeatureToCollectionFrequency.end())
    {
      assert(0<(*(mFeatureToCollectionFrequency.find(inFeatureID))).second);
      
      double lReturnValue((*(mFeatureToCollectionFrequency.find(inFeatureID))).second);
      gMutex->unlock();
      return lReturnValue;
    }
  else
    {
      gMutex->unlock();
      return 1;
    }
}

unsigned int CAcIFFileSystem::getFeatureDescription

(

TID

inFeatureID

)

const [virtual]

What kind of feature is the feature with ID inFeatureID?

debugging code

/debugging code

Implements CAcInvertedFile.

Definition at line 1713 of file CAcIFFileSystem.cc.

References CMutex::lock(), mFeatureDescription, and CMutex::unlock().

                                                                {
  gMutex->lock();
    
  if(mFeatureDescription.find(inID)!=mFeatureDescription.end()){
    unsigned int lReturnValue((*mFeatureDescription.find(inID)).second);
    gMutex->unlock();  
    return lReturnValue;
  }else{
    cout << "[UnknownFeature: "
   << inID
   << "]"
   << flush;

    cout << mFeatureDescription.size() << flush;
    gMutex->unlock();
    return 0;
  }
}

double CAcIFFileSystem::DIDToMaxDocumentFrequency

(

TID

inID

)

const [virtual]

returns the maximum document frequency for one document ID

Implements CAcInvertedFile.

Definition at line 1166 of file CAcIFFileSystem.cc.

References CMutex::lock(), mDocumentInformation, and CMutex::unlock().

                                                              {
  gMutex->lock();

  if(mDocumentInformation.find(inID)!=mDocumentInformation.end())
    {
      double lReturnValue((*mDocumentInformation.find(inID)).second.getMaximumDF());
      gMutex->unlock();
      return lReturnValue;
    }
  else
    {
      assert(1==0);
      gMutex->unlock();
      return 1;
    }
};

double CAcIFFileSystem::DIDToDFSquareSum

(

TID

inID

)

const [virtual]

returns the Document frequency squaresum for a given document ID

Returns the document-frequency square sum for a given document ID

Implements CAcInvertedFile.

Definition at line 1197 of file CAcIFFileSystem.cc.

References CMutex::lock(), mDocumentInformation, and CMutex::unlock().

                                                     {
  gMutex->lock();
  if(mDocumentInformation.find(inID)!=mDocumentInformation.end()){
    double lReturnValue(mDocumentInformation.find(inID)->second.getDFSquareSum());
      
    gMutex->unlock();
    return lReturnValue;
  }else{
    assert(1+1==0);
    gMutex->unlock();
    return 1;
  }
}

double CAcIFFileSystem::DIDToSquareDFLogICFSum

(

TID

inID

)

const [virtual]

Returns this function for a given document ID

Implements CAcInvertedFile.

Definition at line 1224 of file CAcIFFileSystem.cc.

References CMutex::lock(), mDocumentInformation, and CMutex::unlock().

                                                           {
  if(mDocumentInformation.find(inID)!=mDocumentInformation.end()){
    gMutex->lock();
    double lReturnValue((*mDocumentInformation.find(inID))
      .second.getSquareDFLogICFSum());
    gMutex->unlock();
    return lReturnValue;
  }else{
    assert(1+1+1==0);
    gMutex->unlock();
    return 1;
  }
};

bool CAcIFFileSystem::generateInvertedFile

(

)

[virtual]

Generating an inverted File, if there is none. Fast but stupid in-memory method. This method is very fast, if all the inverted file (and a bit more) can be kept in memory at runtime. If this is not the case, extensive swapping is the result, virtually halting the inverted file creation.

Meaning additional document information

Implements CAcInvertedFile.

Definition at line 153 of file CAcIFFileSystem.cc.

References checkConsistency(), CDocumentFrequencyElement::getDocumentFrequency(), CDocumentFrequencyElement::getID(), CMutex::lock(), mFeatureDescriptionFile, mFeatureDescriptionFileName, mInvertedFileName, mOffsetFileName, mURL2FTS, CADIHash::output(), CMutex::unlock(), and writeOffsetFileElement().

00153                                           {
00154   gMutex->lock();
00155   // open the feature description file
00156   //
00157   cout << "I want to use/generate the files:"
00158        << mInvertedFileName << endl
00159        << mOffsetFileName << endl
00160        << mURL2FTS->getURLToFeatureFileName() << endl
00161        << mFeatureDescriptionFileName << endl;
00162 
00163   ofstream lNewInvertedFile(mInvertedFileName.c_str());
00164   ofstream lNewOffsetFile(mOffsetFileName.c_str());
00165 
00166   /* if one of the files does not open correctly */
00167   if(!(mURL2FTS->operator bool()
00168        && 
00169        mFeatureDescriptionFile
00170        &&
00171        lNewInvertedFile
00172        &&
00173        lNewOffsetFile)){
00174       cerr << "I could not open the necessary files for" 
00175      << "generating an inverted file";
00176       gMutex->unlock();
00177       return false;
00178   }
00179   cout << "files successfully opened" 
00180        << flush
00181        << endl;
00182   
00183   
00184   bool lError=false;
00185   //Local: A hash of Inverted File Chunks by the feature ID
00186   map<TID,CInvertedFileChunk> lInvertedFileHash;
00187   
00188   
00190   CADIHash lADI;
00191   
00192   list<CAccessorElement> lAllAccessorElements;
00193 
00194   mURL2FTS->getAllAccessorElements(lAllAccessorElements);
00195 
00196   /* process this for all the images in the file URL to Feature file name */
00197   for(list<CAccessorElement>::const_iterator i=lAllAccessorElements.begin();
00198       i!=lAllAccessorElements.end();
00199       i++){
00200     
00201     //This variable is used in the for loop to translate positions in the map into IDs
00202     int lDocumentID=i->getID();
00203 
00204 
00205     cout << endl
00206    << "Processing File: "
00207    << lDocumentID
00208    << flush;
00209     cout << " "
00210    << *i
00211    << "..."
00212    << flush
00213    << endl;
00214     cout << "h" 
00215    << flush
00216    << endl;
00217 
00218     /* use the second part of the map as the 
00219        file name for the next feature file */
00220     ifstream lFeatureFile(i->getFeatureFileName().c_str());
00221     lADI[lDocumentID]=CAdditionalDocumentInformation(i->getFeatureFileName().c_str());
00222     
00223     unsigned int lNumberOfFeatures=0;
00224     /* reads the number of features out of the file */
00225     lFeatureFile.read((char*)&lNumberOfFeatures,
00226           sizeof(lNumberOfFeatures));
00227     
00228     double lMaxDocumentFrequency=0;
00229     double lDocumentFrequencySquareSum=0;
00230     
00231     //Read features for one image
00232     if(lFeatureFile)
00233       {
00234   /* for each feature in the file */
00235   for(unsigned int j=0;
00236       j<lNumberOfFeatures && lFeatureFile;
00237       j++)
00238     {
00239       /* does this already read in the element from the file ? yes*/
00240       struct{
00241         TID first;
00242         float second;
00243       } lInElement;
00244 
00245       lFeatureFile.read((char*)&lInElement,
00246             sizeof(lInElement));
00247 
00248 
00249       CDocumentFrequencyElement lElement(lInElement.first,
00250                  lInElement.second);
00251       
00252       //      cout << "RSIZE" << sizeof(lInElement) 
00253       //     << " " << lElement.getID()
00254       //     << " " << lInElement.first
00255       //     << " " << lElement.getDocumentFrequency()
00256       //     << "== 0x" << hex << lInElement.second
00257       //     << endl;
00258       
00259       //if this assertion fails, there has been a misunderstanding
00260       //about the data format of the feature file 
00261       //(padding: where and how?) WM
00262       assert(lElement.getDocumentFrequency()>1/1000000000);
00263       assert(lElement.getDocumentFrequency()<=1);
00264 
00265       //Adjust values which depend on the document Frequency
00266       //in the Additional Document Information
00267       lADI[lDocumentID].
00268         adjustDF(lElement.getDocumentFrequency());
00269   
00270       /* this adds a feature into the inverted file list of features */
00271       lInvertedFileHash[lElement.getID()].
00272         addElement(lDocumentID,
00273        lElement.getDocumentFrequency());
00274     }
00275       }
00276 
00277     cout << "...finished" 
00278    << flush
00279    << endl;
00280 
00281 
00282     //one or more errors...
00283     if(!lFeatureFile){
00284       lError=true;
00285       cout << "Error reading file "
00286      << i->getFeatureFileName()
00287      << "!"
00288      << endl;
00289     }
00290   }
00291 
00292   // For each Inverted file chunk: this means for each feature 
00293   // including the documents containing this feature
00294   // Write it and its offset
00295   for(map<TID,CInvertedFileChunk>::const_iterator i=lInvertedFileHash.begin();
00296       i!=lInvertedFileHash.end() && lNewInvertedFile;
00297       i++)
00298     {
00299 
00300       assert((*i).second.size());
00301       
00302       //find out the ID of the feature
00303       TID lFeatureID=(*i).first;
00304       //writing the offset
00305       {
00306   // this is the position of the next feature in the inverted file
00307   streampos lPos(lNewInvertedFile.tellp());
00308   
00309   /* updating the offset file */
00310 
00311   writeOffsetFileElement(lFeatureID,
00312              lPos,
00313              lNewOffsetFile);
00314              
00315 
00316   cout << endl
00317        << "Writing Chunk for Feature ID "
00318        << hex
00319        << lFeatureID
00320        << ". The Offset is 0x"
00321        << hex
00322        << lPos
00323        << dec
00324        << "="
00325        << lPos
00326        << endl;
00327       }
00328       // Writing the next piece of the inverted file
00329       {
00330   /* this writes the actual data for the feature, 
00331      meaning the list of all the documents containg 
00332      this feature */
00333   bool lSuccessfullyWritten(((*i).second).writeBinary(lNewInvertedFile,
00334                   lFeatureID,
00335                   mURL2FTS->size()));
00336   if(!lSuccessfullyWritten)
<