use forward index in snapshot input mappings

platform/lang-impl/src/com/intellij/util/indexing/HashIdForwardIndexAccessor.java

+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing;
+
+import com.intellij.util.indexing.impl.InputDataDiffBuilder;
+import com.intellij.util.indexing.impl.MapInputDataDiffBuilder;
+import com.intellij.util.indexing.impl.forward.AbstractForwardIndexAccessor;
+import com.intellij.util.io.EnumeratorIntegerDescriptor;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.Map;
+
+class HashIdForwardIndexAccessor<Key, Value, Input> extends AbstractForwardIndexAccessor<Key, Value, Integer, Input> {
+  private final SnapshotInputMappingIndex<Key, Value, Input> mySnapshotInputMappingIndex;
+
+  HashIdForwardIndexAccessor(@NotNull SnapshotInputMappingIndex<Key, Value, Input> snapshotInputMappingIndex) {
+    super(EnumeratorIntegerDescriptor.INSTANCE);
+    mySnapshotInputMappingIndex = snapshotInputMappingIndex;
+  }
+
+  @Override
+  protected InputDataDiffBuilder<Key, Value> createDiffBuilder(int inputId, @Nullable Integer hashId) throws IOException {
+    return new MapInputDataDiffBuilder<>(inputId, hashId == null ? null : mySnapshotInputMappingIndex.readData(hashId));
+  }
+
+  @Override
+  public Integer convertToDataType(@Nullable Map<Key, Value> map, @Nullable Input content) {
+    try {
+      return content == null ? null : mySnapshotInputMappingIndex.getHashId(content);
+    }
+    catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/SharedMapForwardIndex.java

+// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing;
+
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.Comparing;
+import com.intellij.openapi.util.io.ByteArraySequence;
+import com.intellij.openapi.vfs.newvfs.persistent.PersistentFS;
+import com.intellij.util.indexing.impl.forward.AbstractForwardIndexAccessor;
+import com.intellij.util.indexing.impl.forward.ForwardIndex;
+import com.intellij.util.indexing.impl.forward.PersistentMapBasedForwardIndex;
+import com.intellij.util.io.ByteSequenceDataExternalizer;
+import com.intellij.util.io.EnumeratorIntegerDescriptor;
+import com.intellij.util.io.IOUtil;
+import com.intellij.util.io.PersistentHashMap;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+
+class SharedMapForwardIndex implements ForwardIndex {
+  private static final Logger LOG = Logger.getInstance(SharedMapForwardIndex.class);
+  private final ID<?, ?> myIndexId;
+
+  // only verification purpose
+  @Nullable
+  private final PersistentMapBasedForwardIndex mySanityVerificationIndex;
+
+  // only verification purpose
+  @Nullable
+  private final AbstractForwardIndexAccessor<?, ?, ?, ?> myAccessor;
+
+  SharedMapForwardIndex(@NotNull IndexExtension<?, ?, ?> extension,
+                        @Nullable AbstractForwardIndexAccessor<?, ?, ?, ?> accessor) throws IOException {
+    myIndexId = (ID<?, ?>)extension.getName();
+    if (!SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS) {
+      File sanityVerificationIndexFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
+      try {
+        mySanityVerificationIndex = new PersistentMapBasedForwardIndex(sanityVerificationIndexFile) {
+          @NotNull
+          @Override
+          protected PersistentHashMap<Integer, ByteArraySequence> createMap(File file) throws IOException {
+            return new PersistentHashMap<Integer, ByteArraySequence>(file,
+                                                                     EnumeratorIntegerDescriptor.INSTANCE,
+                                                                     ByteSequenceDataExternalizer.INSTANCE,
+                                                                     4096) {
+              @Override
+              protected boolean wantNonNegativeIntegralValues() {
+                return true;
+              }
+            };
+          }
+        };
+      }
+      catch (IOException e) {
+        IOUtil.deleteAllFilesStartingWith(sanityVerificationIndexFile);
+        throw e;
+      }
+    }
+    else {
+      mySanityVerificationIndex = null;
+    }
+    myAccessor = accessor;
+  }
+
+  @Nullable
+  @Override
+  public ByteArraySequence get(@NotNull Integer inputId) throws IOException {
+    if (!SharedIndicesData.ourFileSharedIndicesEnabled) {
+      assert mySanityVerificationIndex != null;
+      return mySanityVerificationIndex.get(inputId);
+    }
+    ByteArraySequence data = SharedIndicesData.recallFileData(inputId, myIndexId, ByteSequenceDataExternalizer.INSTANCE);
+    if (mySanityVerificationIndex != null) {
+      ByteArraySequence verificationData = mySanityVerificationIndex.get(inputId);
+      if (!Comparing.equal(verificationData, data)) {
+        if (verificationData != null) {
+          SharedIndicesData.associateFileData(inputId, myIndexId, verificationData, ByteSequenceDataExternalizer.INSTANCE);
+          if (data != null) {
+            LOG.error("Unexpected indexing diff with hash id " +
+                      myIndexId +
+                      ", file:" +
+                      IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
+                      +
+                      "," +
+                      deserializeOrRawString(verificationData) +
+                      "," +
+                      deserializeOrRawString(data));
+          }
+        }
+        data = verificationData;
+      }
+    }
+    return data;
+  }
+
+  @Override
+  public void put(@NotNull Integer inputId, @Nullable ByteArraySequence value) throws IOException {
+    if (value != null) {
+      if (SharedIndicesData.ourFileSharedIndicesEnabled) {
+        SharedIndicesData.associateFileData(inputId, myIndexId, value, ByteSequenceDataExternalizer.INSTANCE);
+      }
+      if (mySanityVerificationIndex != null) {
+        mySanityVerificationIndex.put(inputId, value);
+      }
+    }
+  }
+
+  @Override
+  public void force() {
+    if (mySanityVerificationIndex != null) mySanityVerificationIndex.force();
+  }
+
+  @Override
+  public void clear() throws IOException {
+    if (mySanityVerificationIndex != null) mySanityVerificationIndex.clear();
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (mySanityVerificationIndex != null) mySanityVerificationIndex.close();
+  }
+
+  private Object deserializeOrRawString(ByteArraySequence seq) throws IOException {
+    if (myAccessor != null) return myAccessor.deserializeData(seq);
+    if (seq == null) return null;
+    return Arrays.toString(seq.getBytes());
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/SnapshotInputMappings.java

@@ -37,9 +37,11 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
   private final InputMapExternalizer<Key, Value> myMapExternalizer;
   private final DataIndexer<Key, Value, Input> myIndexer;
   private final PersistentMapBasedForwardIndex myContents;
-  private volatile PersistentHashMap<Integer, Integer> myInputsSnapshotMapping;
   private volatile PersistentHashMap<Integer, String> myIndexingTrace;
 
+  private final SharedMapForwardIndex myForwardIndex;
+  private final HashIdForwardIndexAccessor<Key, Value, Input> myHashIdForwardIndexAccessor;
+
   private final boolean myIsPsiBackedIndex;
 
   SnapshotInputMappings(IndexExtension<Key, Value, Input> indexExtension) throws IOException {
@@ -48,7 +50,9 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
     myMapExternalizer = new InputMapExternalizer<>(indexExtension);
     myIndexer = indexExtension.getIndexer();
     myContents = createContentsIndex();
-    createMaps();
+    myHashIdForwardIndexAccessor = new HashIdForwardIndexAccessor<>(this);
+    myForwardIndex = new SharedMapForwardIndex(indexExtension, myHashIdForwardIndexAccessor);
+    myIndexingTrace = createIndexingTrace();
   }
 
   @NotNull
@@ -186,10 +190,7 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
 
   void putInputHash(int inputId, int hashId) {
     try {
-      if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-        SharedIndicesData.associateFileData(inputId, myIndexId, hashId, EnumeratorIntegerDescriptor.INSTANCE);
-      }
-      if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.put(inputId, hashId);
+      myForwardIndex.put(inputId, myHashIdForwardIndexAccessor.serializeIndexedData(hashId));
     }
     catch (IOException ex) {
       throw new RuntimeException(ex);
@@ -199,57 +200,46 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
   @Override
   public void flush() {
     if (myContents != null) myContents.force();
-    if (myInputsSnapshotMapping != null) myInputsSnapshotMapping.force();
+    if (myForwardIndex != null) myForwardIndex.force();
     if (myIndexingTrace != null) myIndexingTrace.force();
   }
 
   @Override
   public void clear() throws IOException {
     try {
-      List<File> baseDirs = Stream.of(myIndexingTrace, myInputsSnapshotMapping)
-        .filter(Objects::nonNull)
-        .map(PersistentHashMap::getBaseFile)
-        .collect(Collectors.toList());
-      try {
-        closeInternalMaps();
-      }
-      catch (Exception e) {
-        LOG.error(e);
+      if (myIndexingTrace != null) {
+        File baseFile = myIndexingTrace.getBaseFile();
+        try {
+          myIndexingTrace.close();
+        }
+        catch (Exception e) {
+          LOG.error(e);
+        }
+        PersistentHashMap.deleteFilesStartingWith(baseFile);
+        myIndexingTrace = createIndexingTrace();
       }
-      baseDirs.forEach(PersistentHashMap::deleteFilesStartingWith);
-      createMaps();
     } finally {
-      if (myContents != null) {
-        myContents.clear();
-      }
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    closeInternalMaps();
-    if (myContents != null) {
-      myContents.close();
-    }
-  }
-
-  private void closeInternalMaps() {
-    Stream.of(myContents, myIndexingTrace)
-      .filter(Objects::nonNull)
-      .forEach(store -> {
+      Stream.of(myContents, myForwardIndex).filter(Objects::nonNull).forEach(index -> {
         try {
-          store.close();
+          index.clear();
         }
-        catch (Exception e) {
+        catch (IOException e) {
           LOG.error(e);
         }
       });
+    }
   }
 
-  private void createMaps() throws IOException {
-    myIndexingTrace = DebugAssertions.EXTRA_SANITY_CHECKS ? createIndexingTrace() : null;
-    myInputsSnapshotMapping =
-      !SharedIndicesData.ourFileSharedIndicesEnabled || SharedIndicesData.DO_CHECKS ? createInputSnapshotMapping() : null;
+  @Override
+  public void close() throws IOException {
+    Stream.of(myContents, myForwardIndex, myIndexingTrace).filter(Objects::nonNull).forEach(index -> {
+      try {
+        index.close();
+      }
+      catch (IOException e) {
+        LOG.error(e);
+      }
+    });
   }
 
   private PersistentMapBasedForwardIndex createContentsIndex() throws IOException {
@@ -264,23 +254,6 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
     }
   }
 
-  private PersistentHashMap<Integer, Integer> createInputSnapshotMapping() throws IOException {
-    final File fileIdToHashIdFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "fileIdToHashId");
-    try {
-      return new PersistentHashMap<Integer, Integer>(fileIdToHashIdFile, EnumeratorIntegerDescriptor.INSTANCE,
-                                                     EnumeratorIntegerDescriptor.INSTANCE, 4096) {
-        @Override
-        protected boolean wantNonNegativeIntegralValues() {
-          return true;
-        }
-      };
-    }
-    catch (IOException ex) {
-      IOUtil.deleteAllFilesStartingWith(fileIdToHashIdFile);
-      throw ex;
-    }
-  }
-
   private PersistentHashMap<Integer, String> createIndexingTrace() throws IOException {
     final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace");
     try {
@@ -306,32 +279,7 @@ class SnapshotInputMappings<Key, Value, Input> implements SnapshotInputMappingIn
   }
 
   private Integer readInputHashId(int inputId) throws IOException {
-    if (SharedIndicesData.ourFileSharedIndicesEnabled) {
-      Integer hashId = SharedIndicesData.recallFileData(inputId, myIndexId, EnumeratorIntegerDescriptor.INSTANCE);
-      if (hashId == null) hashId = 0;
-      if (myInputsSnapshotMapping == null) return hashId;
-
-      Integer hashIdFromInputSnapshotMapping = myInputsSnapshotMapping.get(inputId);
-      if (hashId == 0 && hashIdFromInputSnapshotMapping != 0 ||
-          !Comparing.equal(hashIdFromInputSnapshotMapping, hashId)) {
-        SharedIndicesData.associateFileData(inputId, myIndexId, hashIdFromInputSnapshotMapping,
-                                            EnumeratorIntegerDescriptor.INSTANCE);
-        if (hashId != 0) {
-          LOG.error("Unexpected indexing diff with hash id " +
-                    myIndexId +
-                    ", file:" +
-                    IndexInfrastructure.findFileById(PersistentFS.getInstance(), inputId)
-                    +
-                    "," +
-                    hashIdFromInputSnapshotMapping +
-                    "," +
-                    hashId);
-        }
-        hashId = hashIdFromInputSnapshotMapping;
-      }
-      return hashId;
-    }
-    return myInputsSnapshotMapping.get(inputId);
+    return myHashIdForwardIndexAccessor.deserializeData(myForwardIndex.get(inputId));
   }
 
   private ByteArraySequence readContents(Integer hashId) throws IOException {