part.h

Go to the documentation of this file.

// File: $Id$
// Author: John Wu <John.Wu at ACM.org>
// Copyright 2000-2012 the Regents of the University of California
#ifndef IBIS_PART_H
#define IBIS_PART_H






#include "column.h"
#include "resource.h"
#include "utilidor.h"

#include <string>
#include <vector>

class FASTBIT_CXX_DLLSPEC ibis::part {
public:
    enum TABLE_STATE {
        UNKNOWN_STATE=0, STABLE_STATE, RECEIVING_STATE,
        PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE
    };
    struct info; // To hold some basic information about a table.
    class readLock;

    /******************************************************************/
    // public functions
    /******************************************************************/

    virtual ~part();
    explicit part(const char* name=0, bool ro=false);
    part(const char* adir, const char* bdir, bool ro=false);
    part(const std::vector<const char*> &mtags, bool ro=false);
    part(const ibis::resource::vList &mtags, bool ro=false);

    inline info* getInfo() const;
    TABLE_STATE getState() const;
    TABLE_STATE getStateNoLocking() const {return state;}

    virtual int buildIndexes(const char* iopt=0, int nthr=1);
    void buildSorted(const char* colname) const;
    void loadIndexes(const char* iopt=0, int ropt=0) const;
    void unloadIndexes() const;
    void purgeIndexFiles() const;

    const char* name()          const {return (m_name?m_name:"?");}
    const char* description()   const {return m_desc.c_str();}
    const char* indexSpec()     const {return idxstr;}
    void indexSpec(const char*);
    time_t timestamp()          const {return switchTime;}
    std::string metaTags()      const;
    ibis::table::stringList columnNames() const;
    ibis::table::typeList   columnTypes() const;
    column* getColumn(const char* name) const;
    column* getColumn(uint32_t ind) const;

    const char* currentDataDir() const {return activeDir;}
    uint32_t nColumns() const {return columns.size();}
    uint32_t nRows() const {return nEvents;}

    int updateData();

    void print(std::ostream &out) const;

    bool matchNameValuePair(const char* name, const char* value) const;
    bool matchMetaTags(const std::vector<const char*> &mtags) const;
    bool matchMetaTags(const ibis::resource::vList &mtags) const;
    inline const char* getMetaTag(const char*) const;

    virtual long selfTest(int nth=1, const char* pref=0) const;
    void computeMinMax();

    /******************************************************************/
    // The following functions need to hold a read lock on the partition,
    // however in order for them to produce consistent results, they must
    // share the same read lock.  These functions do not contain the lock
    // themselves, but rely on the caller to maintain a consistent lock.
    /******************************************************************/
    uint32_t getRowNumber(const rid_t &rid) const;
    long evaluateRIDSet(const ibis::RIDSet&, ibis::bitvector&) const; 
    array_t<rid_t>* getRIDs() const {return rids;} // all RIDs
    array_t<rid_t>* getRIDs(const ibis::bitvector &mask) const;// some RIDs
    bool hasRIDs() const {return (rids!=0) ? (rids->size()==nEvents) : false;}

    virtual double estimateCost(const ibis::qContinuousRange &cmp) const;
    virtual double estimateCost(const ibis::qDiscreteRange &cmp) const;
    virtual double estimateCost(const ibis::qIntHod &cmp) const;
    virtual double estimateCost(const ibis::qUIntHod &cmp) const;
    virtual double estimateCost(const ibis::qString &cmp) const;
    virtual double estimateCost(const ibis::qMultiString &cmp) const;

    virtual long estimateRange(const ibis::qContinuousRange &cmp) const;

    virtual long estimateRange(const ibis::qDiscreteRange &cmp) const;
    virtual long estimateRange(const ibis::qIntHod &cmp) const;
    virtual long estimateRange(const ibis::qUIntHod &cmp) const;

    virtual long evaluateRange(const ibis::qContinuousRange &cmp,
                               const ibis::bitvector &mask,
                               ibis::bitvector &res) const;
    virtual long evaluateRange(const ibis::qDiscreteRange &cmp,
                               const ibis::bitvector &mask,
                               ibis::bitvector &res) const;
    virtual long evaluateRange(const ibis::qIntHod &cmp,
                               const ibis::bitvector &mask,
                               ibis::bitvector &res) const;
    virtual long evaluateRange(const ibis::qUIntHod &cmp,
                               const ibis::bitvector &mask,
                               ibis::bitvector &res) const;

    virtual long estimateRange(const ibis::qContinuousRange &cmp,
                               ibis::bitvector &low,
                               ibis::bitvector &high) const;

    virtual float getUndecidable(const ibis::qContinuousRange &cmp,
                                 ibis::bitvector &iffy) const;

    virtual long estimateRange(const ibis::qDiscreteRange &cmp,
                               ibis::bitvector &low,
                               ibis::bitvector &high) const;
    virtual float getUndecidable(const ibis::qDiscreteRange &cmp,
                                 ibis::bitvector &iffy) const;

    virtual long estimateRange(const ibis::qIntHod &cmp,
                               ibis::bitvector &low,
                               ibis::bitvector &high) const;
    virtual float getUndecidable(const ibis::qIntHod &cmp,
                                 ibis::bitvector &iffy) const;

    virtual long estimateRange(const ibis::qUIntHod &cmp,
                               ibis::bitvector &low,
                               ibis::bitvector &high) const;
    virtual float getUndecidable(const ibis::qUIntHod &cmp,
                                 ibis::bitvector &iffy) const;

    virtual long doScan(const ibis::qRange &cmp,
                        ibis::bitvector &hits) const;
    virtual long doScan(const ibis::qRange &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits) const;
    virtual long doScan(const ibis::qRange &cmp,
                        const ibis::bitvector &mask,
                        void *res) const;
    virtual long doScan(const ibis::qRange &cmp,
                        const ibis::bitvector &mask,
                        void *res, ibis::bitvector &hits) const;

    virtual long negativeScan(const ibis::qRange &cmp,
                              const ibis::bitvector &mask,
                              ibis::bitvector &hits) const;

    virtual long doScan(const ibis::math::term&, const ibis::bitvector&,
                        ibis::bitvector&) const;
    virtual long doScan(const ibis::compRange &cmp,
                        ibis::bitvector &hits) const;
    virtual long doScan(const ibis::compRange &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits) const;

    template <typename E> static long
        doScan(const array_t<E> &varr,
               const ibis::qRange &cmp,
               const ibis::bitvector &mask,
               ibis::bitvector &hits);
    template <typename E> static long
        doScan(const array_t<E> &varr,
               const ibis::qContinuousRange &cmp,
               const ibis::bitvector &mask,
               ibis::bitvector &hits);
    template <typename E> static long
        doScan(const array_t<E> &varr,
               const ibis::qContinuousRange &cmp,
               const ibis::bitvector &mask,
               array_t<E> &res);
    template <typename E> static long
        doScan(const array_t<E> &varr,
               const ibis::qContinuousRange &cmp,
               const ibis::bitvector &mask,
               array_t<E> &res, ibis::bitvector &hits);

    long countHits(const ibis::qRange &cmp) const;

    virtual long estimateMatchAny(const ibis::qAnyAny &cmp,
                                  ibis::bitvector &low,
                                  ibis::bitvector &high) const;
    virtual long matchAny(const ibis::qAnyAny &cmp,
                          ibis::bitvector &hits) const;
    virtual long matchAny(const ibis::qAnyAny &cmp,
                          const ibis::bitvector &mask,
                          ibis::bitvector &hits) const;

    long patternSearch(const ibis::qLike &cmp,
                    ibis::bitvector &low) const;
    long lookforString(const ibis::qString &cmp,
                       ibis::bitvector &low) const;
    long lookforString(const ibis::qMultiString &cmp,
                       ibis::bitvector &low) const;
    long patternSearch(const ibis::qLike &cmp) const;
    long lookforString(const ibis::qString &cmp) const;
    long lookforString(const ibis::qMultiString &cmp) const;

    int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector &mask,
                         ibis::bitvector64 &pairs) const;
    int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector &mask,
                         const char* pairfile) const;
    int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector &mask) const;
    int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
                         const ibis::bitvector &mask,
                         ibis::bitvector64 &pairs) const;
    int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
                         const ibis::bitvector &mask) const;
    int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector64 &trial,
                         ibis::bitvector64 &result) const;
    int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
                         const ibis::bitvector64 &trial,
                         ibis::bitvector64 &result) const;
    /******************************************************************/

    array_t<signed char>*
        selectBytes(const char* name, const ibis::bitvector &mask) const;
    array_t<unsigned char>*
        selectUBytes(const char* name, const ibis::bitvector &mask) const;
    array_t<int16_t>*
        selectShorts(const char* name, const ibis::bitvector &mask) const;
    array_t<uint16_t>*
        selectUShorts(const char* name, const ibis::bitvector &mask) const;
    array_t<int32_t>*
        selectInts(const char* name, const ibis::bitvector &mask) const;
    array_t<uint32_t>*
        selectUInts(const char* name, const ibis::bitvector &mask) const;
    array_t<int64_t>*
        selectLongs(const char* name, const ibis::bitvector &mask) const;
    array_t<uint64_t>*
        selectULongs(const char* name, const ibis::bitvector &mask) const;
    array_t<float>*
        selectFloats(const char* name, const ibis::bitvector &mask) const;
    array_t<double>*
        selectDoubles(const char* name, const ibis::bitvector &mask) const;
    std::vector<std::string>*
        selectStrings(const char* name, const ibis::bitvector &mask) const;

    long selectValues(const char* cname, const ibis::bitvector &mask,
                      void* vals) const;
    long selectValues(const ibis::qContinuousRange& cond, void* vals) const;
    long calculate(const ibis::math::term&, const ibis::bitvector&,
                   array_t<double>&) const;

    /******************************************************************/
    // A group of functions added August, 2005 to provide summary
    // information about the data.  Logically, they are an extension of
    // getInfo.

    double getActualMin(const char *name) const;
    double getActualMax(const char *name) const;
    double getColumnSum(const char *name) const;

    long get1DDistribution(const char *constraints, const char *cname,
                           double begin, double end, double stride,
                           std::vector<uint32_t> &counts) const;
    long get2DDistribution(const char *constraints, const char *cname1,
                           double begin1, double end1, double stride1,
                           const char *cname2,
                           double begin2, double end2, double stride2,
                           std::vector<uint32_t> &counts) const;
    long get3DDistribution(const char *constraints, const char *cname1,
                           double begin1, double end1, double stride1,
                           const char *cname2,
                           double begin2, double end2, double stride2,
                           const char *cname3,
                           double begin3, double end3, double stride3,
                           std::vector<uint32_t> &counts) const;
    long get1DDistribution(const char *constraints, const char *cname,
                           double begin, double end, double stride,
                           const char *wtname,
                           std::vector<double> &weights) const;
    long get2DDistribution(const char *constraints, const char *cname1,
                           double begin1, double end1, double stride1,
                           const char *cname2,
                           double begin2, double end2, double stride2,
                           const char *wtname,
                           std::vector<double> &weights) const;
    long get3DDistribution(const char *constraints, const char *cname1,
                           double begin1, double end1, double stride1,
                           const char *cname2,
                           double begin2, double end2, double stride2,
                           const char *cname3,
                           double begin3, double end3, double stride3,
                           const char *wtname,
                           std::vector<double> &weights) const;
    long get1DDistribution(const char *cname, uint32_t nbin,
                           std::vector<double> &bounds,
                           std::vector<uint32_t> &counts) const;
    long get1DDistribution(const char *constraints,
                           const char *cname, uint32_t nbin,
                           std::vector<double> &bounds,
                           std::vector<uint32_t> &counts) const;
    long get2DDistribution(const char *cname1, const char *cname2,
                           uint32_t nb1, uint32_t nb2,
                           std::vector<double> &bounds1,
                           std::vector<double> &bounds2,
                           std::vector<uint32_t> &counts,
                           const char* const option=0) const;
    long get2DDistribution(const char *constraints,
                           const char *name1, const char *name2,
                           uint32_t nb1, uint32_t nb2,
                           std::vector<double> &bounds1,
                           std::vector<double> &bounds2,
                           std::vector<uint32_t> &counts) const;
    long get3DDistribution(const char *cname1, const char *cname2,
                           const char *cname3,
                           uint32_t nb1, uint32_t nb2, uint32_t nb3,
                           std::vector<double> &bounds1,
                           std::vector<double> &bounds2,
                           std::vector<double> &bounds3,
                           std::vector<uint32_t> &counts,
                           const char* const option=0) const;
    long get3DDistribution(const char *constraints,
                           const char *cname1, const char *cname2,
                           const char *cname3,
                           uint32_t nb1, uint32_t nb2, uint32_t nb3,
                           std::vector<double> &bounds1,
                           std::vector<double> &bounds2,
                           std::vector<double> &bounds3,
                           std::vector<uint32_t> &counts) const;
    long get1DBins(const char *constraints, const char *cname,
                   double begin, double end, double stride,
                   std::vector<ibis::bitvector> &bins) const;
    long get1DBins(const char *constraints, const char *cname,
                   double begin, double end, double stride,
                   std::vector<ibis::bitvector*> &bins) const;
    long get1DBins(const char *constraints, const char *cname,
                   double begin, double end, double stride,
                   const char *wtname,
                   std::vector<double> &weights,
                   std::vector<ibis::bitvector*> &bins) const;
    long get2DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   std::vector<ibis::bitvector> &bins) const;
    long get2DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   std::vector<ibis::bitvector*> &bins) const;
    long get2DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   const char *wtname,
                   std::vector<double> &weights,
                   std::vector<ibis::bitvector*> &bins) const;
    long get3DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   const char *cname3,
                   double begin3, double end3, double stride3,
                   std::vector<ibis::bitvector> &bins) const;
    long get3DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   const char *cname3,
                   double begin3, double end3, double stride3,
                   std::vector<ibis::bitvector*> &bins) const;
    long get3DBins(const char *constraints, const char *cname1,
                   double begin1, double end1, double stride1,
                   const char *cname2,
                   double begin2, double end2, double stride2,
                   const char *cname3,
                   double begin3, double end3, double stride3,
                   const char *wtname,
                   std::vector<double> &weights,
                   std::vector<ibis::bitvector*> &bins) const;
    long get1DBins(const char *constraints, const char *cname1, uint32_t nb1,
                   std::vector<double> &bounds1,
                   std::vector<ibis::bitvector> &bins) const;
    long get2DBins(const char *constraints,
                   const char *cname1, const char *cname2,
                   uint32_t nb1, uint32_t nb2,
                   std::vector<double> &bounds1,
                   std::vector<double> &bounds2,
                   std::vector<ibis::bitvector> &bins) const;
    long get3DBins(const char *constraints,
                   const char *cname1, const char *cname2, const char *cname3,
                   uint32_t nb1, uint32_t nb2, uint32_t nb3,
                   std::vector<double> &bounds1,
                   std::vector<double> &bounds2,
                   std::vector<double> &bounds3,
                   std::vector<ibis::bitvector> &bins) const;

    long getDistribution(const char *name,
                         std::vector<double> &bounds,
                         std::vector<uint32_t> &counts) const;
    long getDistribution(const char *constraints,
                         const char *name,
                         std::vector<double> &bounds,
                         std::vector<uint32_t> &counts) const;
    long getDistribution(const char *name, uint32_t nbc,
                         double *bounds, uint32_t *counts) const;
    long getDistribution(const char *name, const char *constraints,
                         uint32_t nbc, double *bounds,
                         uint32_t *counts) const;

    long getJointDistribution(const char *constraints,
                              const char *name1, const char *name2,
                              std::vector<double> &bounds1,
                              std::vector<double> &bounds2,
                              std::vector<uint32_t> &counts) const;

    long getCumulativeDistribution(const char *name,
                                   std::vector<double> &bounds,
                                   std::vector<uint32_t> &counts) const;
    long getCumulativeDistribution(const char *constraints,
                                   const char *name,
                                   std::vector<double> &bounds,
                                   std::vector<uint32_t> &counts) const;
    long getCumulativeDistribution(const char *name, uint32_t nbc,
                                   double *bounds, uint32_t *counts) const;
    long getCumulativeDistribution(const char *constraints, const char *name,
                                   uint32_t nbc, double *bounds,
                                   uint32_t *counts) const;

    /******************************************************************/
    const std::vector<uint32_t> &getMeshShape() const {return shapeSize;}
    const std::vector<std::string> &getMeshDimensions() const {
        return shapeName;}
    void setMeshShape(const char *shape);
    void updateMetaData() const;
    void combineNames(ibis::table::namesTypes &metalist) const;

    /******************************************************************/
    long append(const char* dir);
    long commit(const char* dir);
    long rollback();
    long addColumn(const char* aexpr, const char* cname,
                   ibis::TYPE_T ctype=ibis::DOUBLE);
    long addColumn(const ibis::math::term* xpr, ibis::bitvector& mask,
                   const char* cname, ibis::TYPE_T ctype=ibis::DOUBLE);

    /******************************************************************/
    virtual long reorder();
    virtual long reorder(const ibis::table::stringList &names);
    virtual long reorder(const ibis::table::stringList &names,
                         const std::vector<bool> &directions);

    long deactivate(const std::vector<uint32_t> &rows);
    long deactivate(const char* conds);
    long reactivate(const std::vector<uint32_t> &rows);
    long reactivate(const char* conds);
    long purgeInactive();
    void emptyCache() const;
    const ibis::bitvector &getNullMask() const {return amask;}

    static const char* skipPrefix(const char*);
    static char* readMetaTags(const char* const dir);
    static void  genName(const std::vector<const char*> &mtags,
                         std::string &name);
    static void  genName(const ibis::resource::vList &mtags,
                         std::string &name);
    void rename(const ibis::partAssoc& known);
    void rename(const char*);

    static uint32_t countPages(const ibis::bitvector &mask,
                               unsigned elemsize=4);
    ibis::fileManager::ACCESS_PREFERENCE
    accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const;

    struct thrArg {
        const part* et;
        const char* pref;
        long* nerrors;  
        ibis::util::counter cnt;
        std::vector<std::string> conds; 
        std::vector<unsigned> super;
        std::vector<unsigned> hits; 
    };
    struct indexBuilderPool {
        ibis::util::counter cnt;
        const char* opt;
        const part &tbl;
        indexBuilderPool(const part &t, const char* spec)
            : cnt(), opt(spec), tbl(t) {}
    };

    void queryTest(const char* pref, long* nerrors) const;
    void quickTest(const char* pref, long* nerrors) const;
    void testRangeOperators(const ibis::column* col,
                            long* nerrors) const;

    // These functions are used in classes barrel and vault
    void logWarning(const char* event, const char* fmt, ...) const;
    void logMessage(const char* event, const char* fmt, ...) const;

    void doBackup(); 

    class barrel;
    class vault;
    typedef std::map< const char*, column*, lessi > columnList;

    template <typename T> static int
        writeColumn(int fdes, ibis::bitvector::word_t nold,
                    ibis::bitvector::word_t nnew,
                    const array_t<T>& vals, const T& fill,
                    ibis::bitvector& totmask,
                    const ibis::bitvector& newmask);
    static int writeString(int fdes, ibis::bitvector::word_t nold,
                           ibis::bitvector::word_t nnew,
                           const std::vector<std::string>& vals,
                           ibis::bitvector& totmask,
                           const ibis::bitvector& newmask);
    static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold,
                        ibis::bitvector::word_t nnew,
                        const ibis::array_t<unsigned char>& bytes,
                        const ibis::array_t<int64_t>& starts,
                        ibis::bitvector& totmask,
                        const ibis::bitvector& newmask);

protected:
    class cleaner;
    class mutexLock;
    class writeLock;
    class softWriteLock;

    friend struct info;
    friend class cleaner;
    friend class readLock;
    friend class writeLock;
    friend class softWriteLock;
    friend class mutexLock;

    /******************************************************************/
    // protected member variables
    //
    char* m_name;               
    std::string m_desc;         
    ibis::resource::vList metaList;     
    mutable array_t<rid_t>* rids;       
    columnList columns;         
    uint32_t nEvents;           
    char* activeDir;            
    char* backupDir;            
    time_t switchTime;          
    TABLE_STATE state;
    char* idxstr;               

    ibis::bitvector amask;      
    std::vector<const column*> colorder;
    std::vector<std::string> shapeName; 
    std::vector<uint32_t> shapeSize;    

    ibis::part::cleaner* myCleaner;     
    bool readonly;                      


    /******************************************************************/
    // protected member functions
    //
    int  readMetaData(uint32_t &nrows, columnList &plist, const char* dir);
    void writeMetaData(const uint32_t nrows, const columnList &plist,
                       const char* dir) const;
    void readRIDs() const; 
    void freeRIDs() const; 

    // functions to deal with meta tags -- those attributes that are
    // associated with a whole partition but are expected to be processed
    // like a categorical attribute in queries
    void extendMetaTags();
    void setMetaTags(const ibis::resource::vList &mts);
    void setMetaTags(const std::vector<const char*> &mts);

    void readMeshShape(const char* const dir);
    void digestMeshShape(const char* shape);

    void logError(const char* event, const char* fmt, ...) const;

    void makeBackupCopy(); // copy the content of activeDir to backupDir
    long verifyBackupDir(); // minimal consistency check
    void deriveBackupDirName();
    long appendToBackup(const char* dir); // append to the backup directory

    template <typename T>
    long writeValues(const char *fname, const array_t<uint32_t> &ind);
    template <typename T>
    long reorderValues(const char *fname,
                       array_t<uint32_t> &starts,
                       array_t<uint32_t> &indout,
                       const array_t<uint32_t> &indin,
                       bool ascending);
    long append1(const char* dir);
    long append2(const char* dir);

    long deactivate(const ibis::bitvector &rows);
    long reactivate(const ibis::bitvector &rows);
    void numbersToBitvector(const std::vector<uint32_t>&,
                            ibis::bitvector&) const;
    void stringToBitvector(const char*, ibis::bitvector&) const;

    template <typename T> static long
        doCompare(const array_t<T> &array,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::array_t<T> &res, ibis::bitvector &hits);
    template <typename T> static long
        doCompare(const char *file,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::array_t<T> &res, ibis::bitvector &hits);

    template <typename T> static long
        doCompare(const array_t<T> &array,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::array_t<T> &res);
    template <typename T> static long
        doCompare(const char *file,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::array_t<T> &res);

    template <typename T> static long
        doCompare(const array_t<T> &array,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);
    template <typename T> static long
        doCompare(const char *file,
                  const ibis::qRange &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);

    template <typename T> static long
        negativeCompare(const array_t<T> &array,
                        const ibis::qRange &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);
    template <typename T> static long
        negativeCompare(const char *file,
                        const ibis::qRange &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);

    template <typename T> static long
        doCompare(const array_t<T> &array,
                  const ibis::qIntHod &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);
    template <typename T> static long
        doCompare(const char *file,
                  const ibis::qIntHod &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);
    template <typename T> static long
        negativeCompare(const array_t<T> &array,
                        const ibis::qIntHod &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);
    template <typename T> static long
        negativeCompare(const char *file,
                        const ibis::qIntHod &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);

    template <typename T> static long
        doCompare(const array_t<T> &array,
                  const ibis::qUIntHod &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);
    template <typename T> static long
        doCompare(const char *file,
                  const ibis::qUIntHod &cmp,
                  const ibis::bitvector &mask,
                  ibis::bitvector &hits);
    template <typename T> static long
        negativeCompare(const array_t<T> &array,
                        const ibis::qUIntHod &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);
    template <typename T> static long
        negativeCompare(const char *file,
                        const ibis::qUIntHod &cmp,
                        const ibis::bitvector &mask,
                        ibis::bitvector &hits);

    template <typename T, typename F> static long
        doComp(const array_t<T> &vals, F cmp,
               const ibis::bitvector &mask,
               array_t<T> &res, ibis::bitvector &hits);

    template <typename T, typename F> static long
        doComp(const array_t<T> &vals, F cmp,
               const ibis::bitvector &mask,
               array_t<T> &res);

    template <typename T, typename F> static long
        doComp(const array_t<T> &vals, F cmp,
               const ibis::bitvector &mask,
               ibis::bitvector &hits);

    template <typename T, typename F1, typename F2> static long
        doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
               const ibis::bitvector &mask,
               array_t<T> &res, ibis::bitvector &hits);

    template <typename T, typename F1, typename F2> static long
        doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
               const ibis::bitvector &mask,
               array_t<T> &res);

    template <typename T, typename F1, typename F2> static long
        doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
               const ibis::bitvector &mask,
               ibis::bitvector &hits);

    template <typename T, typename F> static long
        doComp0(const array_t<T> &vals, F cmp,
                const ibis::bitvector &mask,
                ibis::bitvector &hits);

    template <typename T, typename F1, typename F2> static long
        doComp0(const array_t<T> &vals, F1 cmp1, F2 cmp2,
                const ibis::bitvector &mask,
                ibis::bitvector &hits);

    template <typename T>
        long doCount(const ibis::qRange &cmp) const;

    template <typename T>
        long doCount(const array_t<T> &vals, const ibis::qIntHod &cmp,
                     const ibis::bitvector &mask) const;

    template <typename T>
        long doCount(const array_t<T> &vals, const ibis::qUIntHod &cmp,
                     const ibis::bitvector &mask) const;

    template <typename T>
        long doCount(const array_t<T> &vals, const ibis::qRange &cmp,
                     const ibis::bitvector &mask) const;

    template <typename T, typename F>
        long doCount(const array_t<T> &vals,
                     const ibis::bitvector &mask, F cmp) const;

    template <typename T, typename F1, typename F2>
        long doCount(const array_t<T> &vals,
                     const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const;

    long packCumulativeDistribution(const std::vector<double> &bounds,
                                    const std::vector<uint32_t> &counts,
                                    uint32_t nbc,
                                    double *bptr, uint32_t *cptr) const;
    long packDistribution(const std::vector<double> &bounds,
                          const std::vector<uint32_t> &counts,
                          uint32_t nbc, double *bptr, uint32_t *cptr) const;

    template <typename T1, typename T2>
    long count2DBins(array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     array_t<T2> &vals2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     std::vector<uint32_t> &counts) const;
    template <typename T1, typename T2, typename T3>
    long count3DBins(const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const array_t<T2> &vals2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     const array_t<T3> &vals3,
                     const double &begin3, const double &end3,
                     const double &stride3,
                     std::vector<uint32_t> &counts) const;
    template <typename T1, typename T2>
    long count2DWeights(array_t<T1> &vals1,
                        const double &begin1, const double &end1,
                        const double &stride1,
                        array_t<T2> &vals2,
                        const double &begin2, const double &end2,
                        const double &stride2,
                        array_t<double> &wts,
                        std::vector<double> &weights) const;
    template <typename T1, typename T2, typename T3>
    long count3DWeights(const array_t<T1> &vals1,
                        const double &begin1, const double &end1,
                        const double &stride1,
                        const array_t<T2> &vals2,
                        const double &begin2, const double &end2,
                        const double &stride2,
                        const array_t<T3> &vals3,
                        const double &begin3, const double &end3,
                        const double &stride3,
                        const array_t<double> &wts,
                        std::vector<double> &weights) const;

    template <typename T1>
    long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    std::vector<ibis::bitvector> &bins) const;
    template <typename T1>
    long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    std::vector<ibis::bitvector*> &bins) const;
    template <typename T1>
    long fill1DBinsWeighted(const ibis::bitvector &mask,
                            const array_t<T1> &vals1,
                            const double &begin1, const double &end1,
                            const double &stride1,
                            const array_t<double> &wts,
                            std::vector<double> &weights,
                            std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2>
    long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    const array_t<T2> &vals2,
                    const double &begin2, const double &end2,
                    const double &stride2,
                    std::vector<ibis::bitvector> &bins) const;
    template <typename T1>
    long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const ibis::column &col2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     std::vector<ibis::bitvector> &bins) const;
    template <typename T1, typename T2>
    long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    const array_t<T2> &vals2,
                    const double &begin2, const double &end2,
                    const double &stride2,
                    std::vector<ibis::bitvector*> &bins) const;
    template <typename T1>
    long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const ibis::column &col2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2>
    long fill2DBinsWeighted(const ibis::bitvector &mask,
                            const array_t<T1> &vals1,
                            const double &begin1, const double &end1,
                            const double &stride1,
                            const array_t<T2> &vals2,
                            const double &begin2, const double &end2,
                            const double &stride2,
                            const array_t<double> &wts,
                            std::vector<double> &weights,
                            std::vector<ibis::bitvector*> &bins) const;
    template <typename T1>
    long fill2DBinsWeighted2(const ibis::bitvector &mask,
                             const array_t<T1> &vals1,
                             const double &begin1, const double &end1,
                             const double &stride1,
                             const ibis::column &col2,
                             const double &begin2, const double &end2,
                             const double &stride2,
                             const array_t<double> &wts,
                             std::vector<double> &weights,
                             std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2, typename T3>
    long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    const array_t<T2> &vals2,
                    const double &begin2, const double &end2,
                    const double &stride2,
                    const array_t<T3> &vals3,
                    const double &begin3, const double &end3,
                    const double &stride3,
                    std::vector<ibis::bitvector> &bins) const;
    template <typename T1>
    long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const ibis::column &col2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     const ibis::column &col3,
                     const double &begin3, const double &end3,
                     const double &stride3,
                     std::vector<bitvector> &bins) const;
    template <typename T1, typename T2>
    long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const array_t<T2> &vals2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     const ibis::column &col3,
                     const double &begin3, const double &end3,
                     const double &stride3,
                     std::vector<bitvector> &bins) const;
    template <typename T1, typename T2, typename T3>
    long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
                    const double &begin1, const double &end1,
                    const double &stride1,
                    const array_t<T2> &vals2,
                    const double &begin2, const double &end2,
                    const double &stride2,
                    const array_t<T3> &vals3,
                    const double &begin3, const double &end3,
                    const double &stride3,
                    std::vector<bitvector*> &bins) const;
    template <typename T1>
    long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const ibis::column &col2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     const ibis::column &col3,
                     const double &begin3, const double &end3,
                     const double &stride3,
                     std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2>
    long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
                     const double &begin1, const double &end1,
                     const double &stride1,
                     const array_t<T2> &vals2,
                     const double &begin2, const double &end2,
                     const double &stride2,
                     const ibis::column &col3,
                     const double &begin3, const double &end3,
                     const double &stride3,
                     std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2, typename T3>
    long fill3DBinsWeighted(const ibis::bitvector &mask,
                            const array_t<T1> &vals1,
                            const double &begin1, const double &end1,
                            const double &stride1,
                            const array_t<T2> &vals2,
                            const double &begin2, const double &end2,
                            const double &stride2,
                            const array_t<T3> &vals3,
                            const double &begin3, const double &end3,
                            const double &stride3,
                            const array_t<double> &wts,
                            std::vector<double> &weights,
                            std::vector<bitvector*> &bins) const;
    template <typename T1>
    long fill3DBinsWeighted2(const ibis::bitvector &mask,
                             const array_t<T1> &vals1,
                             const double &begin1, const double &end1,
                             const double &stride1,
                             const ibis::column &col2,
                             const double &begin2, const double &end2,
                             const double &stride2,
                             const ibis::column &col3,
                             const double &begin3, const double &end3,
                             const double &stride3,
                             const array_t<double> &wts,
                             std::vector<double> &weights,
                             std::vector<ibis::bitvector*> &bins) const;
    template <typename T1, typename T2>
    long fill3DBinsWeighted3(const ibis::bitvector &mask,
                             const array_t<T1> &vals1,
                             const double &begin1, const double &end1,
                             const double &stride1,
                             const array_t<T2> &vals2,
                             const double &begin2, const double &end2,
                             const double &stride2,
                             const ibis::column &col3,
                             const double &begin3, const double &end3,
                             const double &stride3,
                             const array_t<double> &wts,
                             std::vector<double> &weights,
                             std::vector<ibis::bitvector*> &bins) const;

    long get1DBins_(const ibis::bitvector &mask, const ibis::column &col,
                    uint32_t nbin, std::vector<double> &bounds,
                    std::vector<ibis::bitvector> &bins, const char *mesg) const;

    long get1DDistribution(const ibis::column &col, uint32_t nbin,
                           std::vector<double> &bounds,
                           std::vector<uint32_t> &counts) const;
    long get2DDistributionU(const ibis::column &col1,
                            const ibis::column &col2,
                            uint32_t nb1, uint32_t nb2,
                            std::vector<double> &bounds1,
                            std::vector<double> &bounds2,
                            std::vector<uint32_t> &counts) const;
    long get2DDistributionA(const ibis::column &col1,
                            const ibis::column &col2,
                            uint32_t nb1, uint32_t nb2,
                            std::vector<double> &bounds1,
                            std::vector<double> &bounds2,
                            std::vector<uint32_t> &counts) const;
    long get2DDistributionI(const ibis::column &col1,
                            const ibis::column &col2,
                            uint32_t nb1, uint32_t nb2,
                            std::vector<double> &bounds1,
                            std::vector<double> &bounds2,
                            std::vector<uint32_t> &counts) const;
    long old2DDistribution(const char *constraints,
                           const char *name1, const char *name2,
                           uint32_t nb1, uint32_t nb2,
                           std::vector<double> &bounds1,
                           std::vector<double> &bounds2,
                           std::vector<uint32_t> &counts) const;
    int coarsenBins(const ibis::column &col, uint32_t nbin,
                    std::vector<double> &bnds,
                    std::vector<ibis::bitvector*> &btmp) const;
    long get3DDistributionA(const ibis::bitvector &mask,
                            const ibis::column &col1,
                            const ibis::column &col2,
                            const ibis::column &col3,
                            uint32_t nb1, uint32_t nb2, uint32_t nb3,
                            std::vector<double> &bounds1,
                            std::vector<double> &bounds2,
                            std::vector<double> &bounds3,
                            std::vector<uint32_t> &counts) const;
    template <typename E1>
    long get3DDistributionA1(const ibis::bitvector &mask,
                             const array_t<E1> &vals1,
                             const ibis::column &col2,
                             const ibis::column &col3,
                             uint32_t nb1, uint32_t nb2, uint32_t nb3,
                             std::vector<double> &bounds1,
                             std::vector<double> &bounds2,
                             std::vector<double> &bounds3,
                             std::vector<uint32_t> &counts) const;
    template <typename E1, typename E2>
    long get3DDistributionA2(const ibis::bitvector &mask,
                             const array_t<E1> &vals1,
                             const array_t<E2> &vals2,
                             const ibis::column &col3,
                             uint32_t nb1, uint32_t nb2, uint32_t nb3,
                             std::vector<double> &bounds1,
                             std::vector<double> &bounds2,
                             std::vector<double> &bounds3,
                             std::vector<uint32_t> &counts) const;

    template <typename E1, typename E2>
        static void mapValues(array_t<E1> &val1, array_t<E2> &val2,
                              uint32_t nb1, uint32_t nb2,
                              array_t<E1> &bnd1, array_t<E2> &bnd2,
                              std::vector<uint32_t> &cnts);

    template <typename T>
        static void mapValues(const array_t<T> &vals,
                              std::map<T, uint32_t> &hist);

    template <typename T>
        static void equalWeightBins(const array_t<T> &vals,
                                    uint32_t nbins, array_t<T> &bounds);

    template <typename T>
        static long adaptiveInts(const array_t<T> &vals, const T vmin,
                                 const T vmax, uint32_t nbins,
                                 std::vector<double> &bounds,
                                 std::vector<uint32_t> &counts);

    template <typename T>
        static long adaptiveFloats(const array_t<T> &vals, const T vmin,
                                   const T vmax, uint32_t nbins,
                                   std::vector<double> &bounds,
                                   std::vector<uint32_t> &counts);

    template <typename T1, typename T2>
        static long adaptive2DBins(const array_t<T1> &vals1,
                                   const array_t<T2> &vals2,
                                   uint32_t nb1, uint32_t nb2,
                                   std::vector<double> &bounds1,
                                   std::vector<double> &bounds2,
                                   std::vector<uint32_t> &counts);

    template <typename T1, typename T2, typename T3>
        static long adaptive3DBins(const array_t<T1> &vals1,
                                   const array_t<T2> &vals2,
                                   const array_t<T3> &vals3,
                                   uint32_t nb1, uint32_t nb2, uint32_t nb3,
                                   std::vector<double> &bounds1,
                                   std::vector<double> &bounds2,
                                   std::vector<double> &bounds3,
                                   std::vector<uint32_t> &counts);

    template <typename T> static long
        adaptiveIntsDetailed(const ibis::bitvector &mask,
                             const array_t<T> &vals,
                             const T vmin, const T vmax, uint32_t nbins,
                             std::vector<double> &bounds,
                             std::vector<ibis::bitvector> &detail);

    template <typename T> static long
        adaptiveFloatsDetailed(const ibis::bitvector &mask,
                               const array_t<T> &vals,
                               const T vmin, const T vmax, uint32_t nbins,
                               std::vector<double> &bounds,
                               std::vector<ibis::bitvector> &detail);

    void gatherSortKeys(ibis::table::stringList& names);
    void composeQueryString(std::string &str,
                            const ibis::column* col1, const ibis::column* col2,
                            const double &lower1, const double &upper1,
                            const double &lower2, const double &upper2) const;
    void buildQueryList(ibis::part::thrArg &lst,
                        unsigned nc, unsigned nq) const;
    void checkQueryList(const ibis::part::thrArg &lst) const;
    uint32_t recursiveQuery(const char* pref, const column* att,
                            double low, double high, long* nerr) const;

private:

    /******************************************************************/
    // private member variables
    mutable pthread_mutex_t mutex;      
    mutable pthread_rwlock_t rwlock;    

    /******************************************************************/
    // private funcations

    void init(const char* prefix); 

    void gainReadAccess(const char* mesg) const;
    void releaseAccess(const char* mesg) const;
    void gainWriteAccess(const char* mesg) const;

    void   fillRIDs(const char* fn) const; 
    void   sortRIDs() const; 
    uint32_t searchSortedRIDs(const ibis::rid_t &rid) const;
    uint32_t searchRIDs(const ibis::rid_t &rid) const;
    void   searchSortedRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
    void   searchRIDs(const ibis::RIDSet&, ibis::bitvector&) const;

    // functions to perform join operations.
    int64_t equiJoin(const ibis::deprecatedJoin &cmp,
                     const ibis::bitvector64 &trial,
                     ibis::bitvector64 &result) const;
    int64_t deprecatedJoin(const ibis::deprecatedJoin &cmp,
                      const ibis::bitvector64 &trial,
                      ibis::bitvector64 &result) const;
    int64_t compJoin(const ibis::deprecatedJoin &cmp,
                     const ibis::bitvector64 &trial,
                     ibis::bitvector64 &result) const;

    int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
                     const ibis::bitvector &mask,
                     ibis::bitvector64 &pairs) const;
    int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
                     const ibis::bitvector &mask) const;
    int64_t loopJoin(const ibis::deprecatedJoin &cmp,
                     const ibis::bitvector &mask,
                     ibis::bitvector64 &pairs) const;
    int64_t loopJoin(const ibis::deprecatedJoin &cmp,
                     const ibis::bitvector &mask) const;
    int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask,
                          ibis::bitvector64 &pairs) const;
    int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask) const;
    int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask,
                          ibis::bitvector64 &pairs) const;
    int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask) const;
    template <class type1, class type2>
    void deprecatedJoinLoop(const array_t<type1> &arr1,
                       const ibis::bitvector &msk1,
                       const array_t<type2> &arr2,
                       const ibis::bitvector &msk2,
                       const double delta,
                       ibis::bitvector64 &pairs) const;
    template <class type1, class type2>
    int64_t deprecatedJoinLoop(const array_t<type1> &arr1,
                          const ibis::bitvector &msk1,
                          const array_t<type2> &arr2,
                          const ibis::bitvector &msk2,
                          const double delta) const;
    int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask,
                          ibis::bitvector64 &pairs) const;
    int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
                          const ibis::bitvector &mask) const;
    int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector &mask,
                         ibis::bitvector64 &pairs) const;
    int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
                         const ibis::bitvector &mask) const;

    part(const part&);
    const part &operator=(const part&);
}; // class ibis::part

namespace ibis {
    // Extends the name space ibis::util to contain three more functions to
    // deal with the reconstruction of partitions.
    namespace util {
        unsigned int FASTBIT_CXX_DLLSPEC
        gatherParts(ibis::partList &parts,
                    const char *adir, const char *bdir, bool ro=false);
        unsigned int FASTBIT_CXX_DLLSPEC
        gatherParts(ibis::partList &parts, const char *adir, bool ro=false);
        unsigned int FASTBIT_CXX_DLLSPEC
        gatherParts(ibis::partList &parts, const ibis::resource &res,
                    bool ro=false);
    } // namespace util
} // namespace ibis

struct FASTBIT_CXX_DLLSPEC ibis::part::info {
    const char* name;           
    const char* description;    
    const char* metaTags;       
    const uint64_t nrows;       

    std::vector<ibis::column::info*> cols;

    info(const char* na, const char* de, const uint64_t &nr,
         const ibis::part::columnList &co);
    info(const ibis::part &tbl);
    ~info();

private:
    info();     // private default constructor, not implemented!
    info& operator=(const info&);
}; // ibis::part::info

class ibis::part::cleaner : public ibis::fileManager::cleaner {
public:
    virtual void operator()() const;
    cleaner(const part* tbl) : thePart(tbl) {}
    virtual ~cleaner() {}

private:
    const part* thePart;
}; // ibis::part::cleaner

class ibis::part::readLock {
public:
    readLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
        tbl->gainReadAccess(m);
    }
    ~readLock() {thePart->releaseAccess(mesg);}

private:
    const part* thePart;
    const char* mesg;

    readLock() {}; // no default constructor
    readLock(const readLock&) {}; // can not copy
    const readLock &operator=(const readLock&);
}; // ibis::part::readLock

class ibis::part::writeLock {
public:
    writeLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
        tbl->gainWriteAccess(m);
    }
    ~writeLock() {thePart->releaseAccess(mesg);}

private:
    const part* thePart;
    const char* mesg;

    writeLock() {}; // no default constructor
    writeLock(const writeLock&) {}; // can not copy
    const writeLock &operator=(const writeLock&);
}; // ibis::part::writeLock

class ibis::part::softWriteLock {
public:
    softWriteLock(const part* tbl, const char* m);
    ~softWriteLock() {if (lckd==0) thePart->releaseAccess(mesg);}
    bool isLocked() const {return (lckd==0);}

private:
    const part* thePart;
    const char* mesg;
    const int lckd;

    softWriteLock() : thePart(0), mesg(0), lckd(0) {};
    softWriteLock(const softWriteLock &rhs)
        : thePart(rhs.thePart), mesg(rhs.mesg), lckd(0) {};
    const softWriteLock &operator=(const softWriteLock&);
}; // ibis::part::softWriteLock

class ibis::part::mutexLock {
public:
    mutexLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
        if (ibis::gVerbose > 9)
            tbl->logMessage("gainExclusiveAccess",
                            "pthread_mutex_lock for %s", m);
        int ierr = pthread_mutex_lock(&(tbl->mutex));
        if (0 != ierr)
            tbl->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s "
                            "returned %d (%s)", m, ierr, strerror(ierr));
    }
    ~mutexLock() {
        if (ibis::gVerbose > 9)
            thePart->logMessage("releaseExclusiveAccess",
                                "pthread_mutex_unlock for %s", mesg);
        int ierr = pthread_mutex_unlock(&(thePart->mutex));
        if (0 != ierr)
            thePart->logWarning("releaseExclusiveAccess",
                                "pthread_mutex_unlock for %s returned %d (%s)",
                                mesg, ierr, strerror(ierr));
    }

private:
    const part* thePart;
    const char* mesg;

    mutexLock() {}; // no default constructor    
    mutexLock(const mutexLock&) {}; // can not copy      
    const mutexLock &operator=(const mutexLock&);        
}; // ibis::part::mutexLock

class ibis::part::barrel : public ibis::math::barrel {
public:
    barrel(const ibis::part *t=0) : _tbl(t), position(0) {};
    virtual ~barrel() {close();} 

    virtual long open(const ibis::part *t=0); 
    virtual long close(); 
    virtual long read(); 


    virtual long seek(uint32_t pos);
    uint32_t tell() const {return position;}

    void getNullMask(ibis::bitvector &mask) const;
    const ibis::column* getColumn(uint32_t i) const {return cols[i];}

protected:
    const ibis::part *_tbl;
    uint32_t position;
    std::vector<const ibis::column*> cols;
    std::vector<ibis::fileManager::storage*> stores;
    std::vector<int> fdes;
}; // ibis::part::barrel

class ibis::part::vault : public ibis::part::barrel {
public:
    vault(const ibis::roster &r);
    virtual ~vault() {close();}

    virtual long open(const ibis::part *t=0); 
    virtual long read(); 
    virtual long seek(uint32_t pos); 

    long seek(double val);
    uint32_t tellReal() const;

private:
    const ibis::roster &_roster;

    template <class T>
    uint32_t seekValue(int fd, const T &val) const;
    template <class T>
    uint32_t seekValue(const array_t<T>&arr, const T &val) const;

    vault();
    vault& operator=(const vault&);
}; // ibis::part::vault
/*
barrel

Vault \Vault\ (v[add]lt; see Note, below), n. [OE. voute, OF. voute, volte,
F. vo[^u]te, LL. volta, for voluta, volutio, fr. L. volvere, volutum, to
roll, to turn about. See Voluble, and cf. Vault a leap, Volt a turn,
Volute.] 1. (Arch.) An arched structure of masonry, forming a ceiling or
canopy.

The long-drawn aisle and fretted vault. --Gray.

2. An arched apartment; especially, a subterranean room, use for storing
   articles, for a prison, for interment, or the like; a cell; a
   cellar. ``Charnel vaults.'' --Milton.

The silent vaults of death. --Sandys.

To banish rats that haunt our vault. --Swift.

3. The canopy of heaven; the sky.

That heaven's vault should crack. --Shak.

4. [F. volte, It. volta, originally, a turn, and the same word as volta an
   arch. See the Etymology above.] A leap or bound. Specifically: (a)
   (Man.) The bound or leap of a horse; a curvet. (b) A leap by aid of the
   hands, or of a pole, springboard, or the like.

Note: The l in this word was formerly often suppressed in pronunciation.

Barrel, Cradle, Cylindrical, or Wagon, vault (Arch.), a kind of vault
having two parallel abutments, and the same section or profile at all
points. It may be rampant, as over a staircase (see Rampant vault, under
Rampant), or curved in plan, as around the apse of a church.

Coved vault. (Arch.) See under 1st Cove, v. t.

Groined vault (Arch.), a vault having groins, that is, one in which
different cylindrical surfaces intersect one another, as distinguished from
a barrel, or wagon, vault.

Rampant vault. (Arch.) See under Rampant.

Ribbed vault (Arch.), a vault differing from others in having solid ribs
which bear the weight of the vaulted surface. True Gothic vaults are of
this character.

Vault light, a partly glazed plate inserted in a pavement or ceiling to
admit light to a vault below.

Source: Webster's Revised Unabridged Dictionary, (c) 1996, 1998 MICRA, Inc.
 */

namespace ibis {
    // Explicit template specialization for member function
    // ibis::part::equalWeightBins
    template <> void
    part::equalWeightBins(const array_t<float> &vals,
                          uint32_t nbins, array_t<float> &bounds);
    template <> void
    part::equalWeightBins(const array_t<double> &vals,
                          uint32_t nbins, array_t<double> &bounds);

    template <> long
    part::doScan(const array_t<float> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::bitvector &);
    template <> long
    part::doScan(const array_t<double> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::bitvector &);

    template <> long
    part::doScan(const array_t<float> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::array_t<float> &);
    template <> long
    part::doScan(const array_t<double> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::array_t<double> &);

    template <> long
    part::doScan(const array_t<float> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::array_t<float> &,
                 ibis::bitvector &);
    template <> long
    part::doScan(const array_t<double> &,
                 const ibis::qContinuousRange &,
                 const ibis::bitvector &, ibis::array_t<double> &,
                 ibis::bitvector &);

    template <> long
    part::doCount<float>(const ibis::qRange&) const;
    template <> long
    part::doCount<double>(const ibis::qRange&) const;
}

inline ibis::part::info* ibis::part::getInfo() const {
    return new info(*this);
}

inline ibis::column* ibis::part::getColumn(uint32_t ind) const {
    if (ind < columns.size()) {
        if (colorder.size() == columns.size()) {
            return const_cast<ibis::column*>(colorder[ind]);
        }
        else {
            ibis::part::columnList::const_iterator it = columns.begin();
            for (; ind > 0; -- ind) ++ it;
            return (*it).second;
        }
    }
    else {
        return 0;
    }
} // ibis::part::getColumn

inline const char* ibis::part::getMetaTag(const char* name) const {
    ibis::resource::vList::const_iterator it = metaList.find(name);
    if (it != metaList.end())
        return (*it).second;
    else
        return static_cast<const char*>(0);
} // ibis::part::getMetaTag

inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
                                        const ibis::bitvector &mask,
                                        ibis::bitvector64 &pairs) const {
    return loopJoin(cmp, mask, pairs);
} // ibis::part::evaluateJoin

inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &,
                                        const ibis::bitvector &,
                                        const char *) const {
    logWarning("evaluate", "not implemented yet");
    return -1;
} // ibis::part::evaluateJoin

inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
                                        const ibis::bitvector &mask) const {
    return loopJoin(cmp, mask);
} // ibis::part::evaluateJoin

inline int64_t ibis::part::evaluateJoin
(const std::vector<const ibis::deprecatedJoin*> &cmp,
 const ibis::bitvector &mask, ibis::bitvector64 &pairs) const {
    return loopJoin(cmp, mask, pairs);
} // ibis::part::evaluateJoin

inline void ibis::part::cleaner::operator()() const {
    const uint32_t sz = ibis::fileManager::bytesInUse();
    thePart->unloadIndexes();
    if (sz == ibis::fileManager::bytesInUse() &&
        thePart->getStateNoLocking() == ibis::part::STABLE_STATE) {
        thePart->freeRIDs();
    }
} // ibis::part::cleaner::operator
#endif // IBIS_PART_H