shared index: drop shared-index storage memory data on cache invalidation, do...

shared index: drop shared-index storage memory data on cache invalidation, do not wipe excluded chunk list, drop download history to ensure indexes could be download again IDEA-266495, IDEA-266989

(cherry picked from commit 3887e9059f4001fb20bcb01ce5568edaa581c0c2)

IJ-CR-10430

GitOrigin-RevId: 9944b12b9018dd5c654953f67c4e5953a2646e5e

platform/lang-impl/src/com/intellij/util/indexing/CorruptionMarker.kt

@@ -5,34 +5,81 @@ import com.intellij.openapi.application.PathManager
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.psi.stubs.SerializationManagerEx
 import com.intellij.util.indexing.impl.storage.FileBasedIndexLayoutSettings
+import com.intellij.util.io.directoryStreamIfExists
+import com.intellij.util.io.directoryStreamIfExists
+import com.intellij.util.io.exists
+import com.intellij.util.io.readText
+import com.intellij.util.io.write
 import java.nio.file.Files
+import kotlin.io.path.deleteExisting
 
 internal object CorruptionMarker {
   private const val CORRUPTION_MARKER_NAME = "corruption.marker"
+  private const val MARKED_AS_DIRTY_REASON = "Indexes marked as dirty (IDE is expected to be work)"
+  private const val EXPLICIT_INVALIDATION_REASON = "Explicit index invalidation"
 
   private val corruptionMarker
     get() = PathManager.getIndexRoot().resolve(CORRUPTION_MARKER_NAME)
 
   @JvmStatic
-  fun requestInvalidation() {
-    FileBasedIndexImpl.LOG.info("Requesting explicit indices invalidation")
-    try {
-      Files.newOutputStream(corruptionMarker).close()
-    }
-    catch (ignore: Throwable) {
+  fun markIndexesAsDirty() {
+    createCorruptionMarker(MARKED_AS_DIRTY_REASON)
+  }
+
+  @JvmStatic
+  fun markIndexesAsClosed() {
+    val corruptionMarkerExists = corruptionMarker.exists()
+    if (corruptionMarkerExists) {
+      try {
+        if (corruptionMarker.readText() == MARKED_AS_DIRTY_REASON) {
+          corruptionMarker.deleteExisting()
+        }
+      }
+      catch (ignored: Exception) { }
     }
   }
 
+  @JvmStatic
+  fun requestInvalidation() {
+    FileBasedIndexImpl.LOG.info("Explicit index invalidation has been requested")
+    createCorruptionMarker(EXPLICIT_INVALIDATION_REASON)
+  }
+
   @JvmStatic
   fun requireInvalidation(): Boolean {
-    return IndexInfrastructure.hasIndices() && Files.exists(corruptionMarker)
+    val corruptionMarkerExists = corruptionMarker.exists()
+    if (corruptionMarkerExists) {
+      val message = "Indexes are corrupted and will be rebuilt"
+      try {
+        val corruptionReason = corruptionMarker.readText()
+        FileBasedIndexImpl.LOG.info("$message (reason = $corruptionReason)")
+      }
+      catch (e: Exception) {
+        FileBasedIndexImpl.LOG.info(message)
+      }
+    }
+    return IndexInfrastructure.hasIndices() && corruptionMarkerExists
   }
 
   @JvmStatic
   fun dropIndexes() {
-    val indexRoot = PathManager.getIndexRoot().toFile()
-    FileUtil.deleteWithRenaming(indexRoot)
-    indexRoot.mkdirs()
+    val indexRoot = PathManager.getIndexRoot()
+
+    if (Files.exists(indexRoot)) {
+      val filesToBeIgnored = FileBasedIndexInfrastructureExtension.EP_NAME.extensions.mapNotNull { it.persistentStateRoot }.toSet()
+      indexRoot.directoryStreamIfExists { dirStream ->
+        dirStream.forEach {
+          if (!filesToBeIgnored.contains(it.fileName.toString())) {
+            FileUtil.deleteWithRenaming(it.toFile())
+          }
+        }
+      }
+    }
+    else {
+      Files.createDirectories(indexRoot)
+    }
+
+
     // serialization manager is initialized before and use removed index root so we need to reinitialize it
     SerializationManagerEx.getInstanceEx().reinitializeNameStorage()
     ID.reinitializeDiskStorage()
@@ -41,4 +88,13 @@ internal object CorruptionMarker {
     FileBasedIndexInfrastructureExtension.EP_NAME.extensions.forEach { it.resetPersistentState() }
     FileBasedIndexLayoutSettings.saveCurrentLayout()
   }
+
+  private fun createCorruptionMarker(reason: String) {
+    try {
+      corruptionMarker.write(reason)
+    }
+    catch (e: Exception) {
+      FileBasedIndexImpl.LOG.warn(e)
+    }
+  }
 }
\ No newline at end of file

platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexInfrastructureExtension.java

 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.util.indexing;
 
+import com.intellij.openapi.application.PathManager;
 import com.intellij.openapi.extensions.ExtensionPointName;
 import com.intellij.openapi.project.Project;
 import com.intellij.openapi.vfs.VirtualFile;
@@ -85,6 +86,15 @@ public interface FileBasedIndexInfrastructureExtension {
   @NotNull
   InitializationResult initialize();
 
+  /**
+   * @return index persistent state root for given extension, namely a place where all cached data will be stored.
+   * Every index extension persistent data should be stored in `{@link PathManager#getIndexRoot()}/getPersistentStateRoot()` dir.
+   */
+  @Nullable
+  default String getPersistentStateRoot() {
+    return null;
+  }
+
   /**
    * Executed when IntelliJ is requested to clear indexes. Each extension should reset its caches.
    * For example, it may happen on index invalidation.