Implementing a space efficient SIFT/L2 space #280

similarity_search/include/distcomp.h

@@ -248,13 +248,6 @@ inline float JaccardSparse(const IdType *pArr1, size_t qty1, const IdType *pArr2
   return 1 - qtyInter/(qtyS - qtyInter);
 }
 
-}
-
-/*
- * Edit distances
- */
-#include "distcomp_edist.h"
-
 // For SIFT vectors (whose dim=128) int is enough to store the smallest and the largest difference
 typedef int DistTypeSIFT;
 
@@ -265,5 +258,13 @@ DistTypeSIFT l2SqrSIFTNaive(const uint8_t* pVect1, const uint8_t* pVect2);
 DistTypeSIFT l2SqrSIFTPrecomp(const uint8_t* pVect1, const uint8_t* pVect2);
 DistTypeSIFT l2SqrSIFTPrecompAVX(const uint8_t* pVect1, const uint8_t* pVect2);
 
+}
+
+
+/*
+ * Edit distances not included into the similarity space,
+ * because the namespace is specified in the distcomp_edist.h
+ */
+#include "distcomp_edist.h"
 
 #endif

similarity_search/include/factory/init_spaces.h

@@ -147,6 +147,7 @@ inline void initSpaces() {
   REGISTER_SPACE_CREATOR(float,  SPACE_RENYI_DIVERG_FAST,  CreateRenyiDivergFast)
   REGISTER_SPACE_CREATOR(double, SPACE_RENYI_DIVERG_FAST,  CreateRenyiDivergFast)
 
+  REGISTER_SPACE_CREATOR(int,    SPACE_L2SQR_SIFT,  CreateL2SqrSIFT)
 }
 
 }

similarity_search/include/factory/space/space_lp.h

@@ -16,6 +16,7 @@
 #define FACTORY_SPACE_LP_H
 
 #include <space/space_lp.h>
+#include <space/space_l2sqr_sift.h>
 
 namespace similarity {
 
@@ -49,6 +50,10 @@ Space<dist_t>* CreateL(const AnyParams& AllParams) {
   return new SpaceLp<dist_t>(p);
 }
 
+Space<DistTypeSIFT>* CreateL2SqrSIFT(const AnyParams& ){
+  return new SpaceL2SqrSift();
+}
+
 /*
  * End of creating functions.
  */

similarity_search/include/method/simple_inverted_index.h

@@ -39,8 +39,8 @@ class SimplInvIndex : public Index<dist_t> {
    */
   SimplInvIndex(bool printProgress, 
               Space<dist_t>& space, 
-              const ObjectVector& data) : printProgress_(printProgress),
-                                          Index<dist_t>(data),
+              const ObjectVector& data) : Index<dist_t>(data),
+                                          printProgress_(printProgress),
                                           pSpace_(dynamic_cast<SpaceSparseNegativeScalarProductFast*>(&space)) {
     if (pSpace_ == nullptr) {
       PREPARE_RUNTIME_ERR(err) <<

similarity_search/include/portable_align.h

@@ -20,9 +20,12 @@
  */
 
 #if defined(__GNUC__)
-  #define PORTABLE_ALIGN16 __attribute__((aligned(16)))
-  #define PORTABLE_ALIGN32 __attribute__((aligned(32)))
+#define PORTABLE_ALIGN16 __attribute__((aligned(16)))
 #else
-  #define PORTABLE_ALIGN16 __declspec(align(16))
-  #define PORTABLE_ALIGN32 __declspec(align(32))
+#define PORTABLE_ALIGN16 __declspec(align(16))
+#endif
+#if defined(__GNUC__)
+#define PORTABLE_ALIGN32 __attribute__((aligned(32)))
+#else
+#define PORTABLE_ALIGN32 __declspec(align(32))
 #endif

similarity_search/include/portable_intrinsics.h

@@ -14,8 +14,8 @@
  */
 #pragma once
 
-#include <portable_align.h>
 #include <portable_popcount.h>
+#include <portable_align.h>
 
 // On 64-bit platforms SSE2 is always present, but Windows doesn't set SSE2 flag
 // http://stackoverflow.com/questions/1067630/sse2-option-in-visual-c-x64

similarity_search/include/space/space_l2sqr_sift.h

@@ -0,0 +1,84 @@
+/**
+ * Non-metric Space Library
+ *
+ * Main developers: Bilegsaikhan Naidan, Leonid Boytsov, Yury Malkov, Ben Frederickson, David Novak
+ *
+ * For the complete list of contributors and further details see:
+ * https://github.com/searchivarius/NonMetricSpaceLib
+ *
+ * Copyright (c) 2013-2018
+ *
+ * This code is released under the
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
+ *
+ */
+#ifndef _SPACE_L2_SQR_SIFT_H_
+#define _SPACE_L2_SQR_SIFT_H_
+
+#include <string>
+#include <map>
+#include <stdexcept>
+#include <sstream>
+#include <memory>
+
+#include <string.h>
+#include "global.h"
+#include "object.h"
+#include "utils.h"
+#include "space.h"
+#include "distcomp.h"
+
+#define SPACE_L2SQR_SIFT    "l2sqr_sift"
+
+namespace similarity {
+
+using std::string;
+using std::unique_ptr;
+
+class SpaceL2SqrSift : public Space<DistTypeSIFT> {
+ public:
+  explicit SpaceL2SqrSift() {}
+  virtual ~SpaceL2SqrSift() {}
+
+  /** Standard functions to read/write/create objects */ 
+  virtual unique_ptr<Object> CreateObjFromStr(IdType id, LabelType label, const string& s,
+                                                DataFileInputState* pInpState) const override;
+  // Create a string representation of an object.
+  virtual string CreateStrFromObj(const Object* pObj, const string& externId /* ignored */) const override;
+  // Open a file for reading, fetch a header (if there is any) and memorize an input state
+  virtual unique_ptr<DataFileInputState> OpenReadFileHeader(const string& inputFile) const override;
+  // Open a file for writing, write a header (if there is any) and memorize an output state
+  virtual unique_ptr<DataFileOutputState> OpenWriteFileHeader(const ObjectVector& dataset,
+                                                                const string& outputFile) const override;
+  /*
+   * Read a string representation of the next object in a file as well
+   * as its label. Return false, on EOF.
+   */
+  virtual bool ReadNextObjStr(DataFileInputState &, string& strObj, LabelType& label, string& externId) const override;
+  /** End of standard functions to read/write/create objects */ 
+
+  virtual Object* CreateObjFromUint8Vect(IdType id, LabelType label, const std::vector<uint8_t>& InpVect) const;
+  virtual size_t GetElemQty(const Object* object) const override { return SIFT_DIM; }
+
+  virtual string StrDesc() const override { return SPACE_L2SQR_SIFT; }
+
+  virtual bool ApproxEqual(const Object& obj1, const Object& obj2) const override;
+
+  virtual void CreateDenseVectFromObj(const Object* obj, DistTypeSIFT* pVect, size_t nElem) const override;
+
+  static void ReadUint8Vec(std::string line, LabelType& label, std::vector<uint8_t>& v);
+
+protected:
+  DISABLE_COPY_AND_ASSIGN(SpaceL2SqrSift);
+
+  virtual DistTypeSIFT HiddenDistance(const Object* obj1, const Object* obj2) const override {
+    const uint8_t* pVect1 = reinterpret_cast<const uint8_t*>(obj1->data());
+    const uint8_t* pVect2 = reinterpret_cast<const uint8_t*>(obj2->data());
+
+    return l2SqrSIFTPrecompAVX(pVect1, pVect2);
+  }
+};
+
+}  // namespace similarity
+
+#endif