Commit eeea5e2e authored by Sergey Patrikeev's avatar Sergey Patrikeev Committed by intellij-monorepo-bot
Browse files

HTML indexing diagnostics [IDEA-267366]: detect slowly indexed files (>500 ms).

(cherry picked from commit 7984d4488b082689768023ec7097216b34406cf8)

GitOrigin-RevId: 449e196b92fc64bc317d1dde9e9f341c4abf30d9
parent 1410d71f
Showing with 88 additions and 11 deletions
+88 -11
......@@ -75,6 +75,7 @@ class IndexDiagnosticTest : JavaCodeInsightFixtureTestCase() {
JsonDuration(789),
JsonDuration(234),
JsonDuration(345),
JsonDuration(345),
JsonDateTime(ZonedDateTime.now()),
JsonDateTime(ZonedDateTime.now()),
JsonDuration(333),
......@@ -131,13 +132,21 @@ class IndexDiagnosticTest : JavaCodeInsightFixtureTestCase() {
),
fileProviderStatistics = listOf(
JsonFileProviderIndexStatistics(
"providerName",
444,
33,
JsonDuration(123),
JsonDuration(456),
1,
listOf(
providerName = "providerName",
totalNumberOfIndexedFiles = 444,
totalNumberOfFilesFullyIndexedByExtensions = 33,
totalIndexingTime = JsonDuration(123),
contentLoadingTime = JsonDuration(456),
numberOfTooLargeForIndexingFiles = 1,
slowIndexedFiles = listOf(
JsonFileProviderIndexStatistics.JsonSlowIndexedFile(
"file",
JsonDuration(123),
JsonDuration(456),
JsonDuration(789)
)
),
indexedFiles = listOf(
JsonFileProviderIndexStatistics.JsonIndexedFile(
path = PortableFilePath.RelativePath(PortableFilePath.ProjectRoot, "src/a.java"),
wasFullyIndexedByExtensions = true
......
......@@ -13,6 +13,12 @@ import com.intellij.util.indexing.diagnostic.dump.paths.PortableFilePaths
*/
class IndexingJobStatistics(private val project: Project, val fileSetName: String) {
companion object {
const val SLOW_FILES_LIMIT = 10
const val SLOW_FILE_PROCESSING_THRESHOLD_MS = 500
}
var indexingVisibleTime: TimeNano = 0
var processingTimeInAllThreads: TimeNano = 0
......@@ -35,6 +41,8 @@ class IndexingJobStatistics(private val project: Project, val fileSetName: Strin
val indexedFiles = arrayListOf<IndexedFile>()
val slowIndexedFiles = LimitedPriorityQueue<SlowIndexedFile>(SLOW_FILES_LIMIT, compareBy { it.processingTime })
data class IndexedFile(val portableFilePath: PortableFilePath, val wasFullyIndexedByExtensions: Boolean)
data class StatsPerIndexer(
......@@ -54,9 +62,9 @@ class IndexingJobStatistics(private val project: Project, val fileSetName: Strin
fun addFileStatistics(
file: VirtualFile,
fileStatistics: FileIndexingStatistics,
processingTime: Long,
contentLoadingTime: Long,
fileSize: Long
processingTime: TimeNano,
contentLoadingTime: TimeNano,
fileSize: BytesNumber
) {
numberOfIndexedFiles++
if (fileStatistics.wasFullyIndexedByExtensions) {
......@@ -81,12 +89,16 @@ class IndexingJobStatistics(private val project: Project, val fileSetName: Strin
StatsPerFileType(0, 0, 0, 0)
}
stats.contentLoadingTimeInAllThreads += contentLoadingTime
stats.indexingTimeInAllThreads += perIndexerTimes.values.sum()
val indexingTime = perIndexerTimes.values.sum()
stats.indexingTimeInAllThreads += indexingTime
stats.totalBytes += fileSize
stats.numberOfFiles++
if (IndexDiagnosticDumper.shouldDumpPathsOfIndexedFiles) {
indexedFiles += IndexedFile(getIndexedFilePath(file), fileStatistics.wasFullyIndexedByExtensions)
}
if (processingTime > SLOW_FILE_PROCESSING_THRESHOLD_MS) {
slowIndexedFiles.addElement(SlowIndexedFile(file.name, processingTime, indexingTime, contentLoadingTime))
}
}
fun addTooLargeForIndexingFile(file: VirtualFile) {
......
// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util.indexing.diagnostic
data class SlowIndexedFile(
val fileName: String,
val processingTime: TimeNano,
val indexingTime: TimeNano,
val contentLoadingTime: TimeNano
)
\ No newline at end of file
......@@ -55,10 +55,18 @@ fun IndexingJobStatistics.toJsonStatistics(): JsonFileProviderIndexStatistics {
totalIndexingTime = JsonDuration(indexingVisibleTime),
contentLoadingTime = JsonDuration(contentLoadingVisibleTime),
numberOfTooLargeForIndexingFiles = numberOfTooLargeForIndexingFiles,
slowIndexedFiles = slowIndexedFiles.biggestElements.map { it.toJson() },
indexedFiles = jsonIndexedFiles
)
}
fun SlowIndexedFile.toJson() = JsonFileProviderIndexStatistics.JsonSlowIndexedFile(
fileName = fileName,
processingTime = JsonDuration(processingTime),
indexingTime = JsonDuration(indexingTime),
contentLoadingTime = JsonDuration(contentLoadingTime)
)
fun IndexingJobStatistics.IndexedFile.toJson() = JsonFileProviderIndexStatistics.JsonIndexedFile(
path = portableFilePath,
wasFullyIndexedByExtensions = wasFullyIndexedByExtensions
......
......@@ -13,9 +13,17 @@ data class JsonFileProviderIndexStatistics(
val totalIndexingTime: JsonDuration,
val contentLoadingTime: JsonDuration,
val numberOfTooLargeForIndexingFiles: Int,
val slowIndexedFiles: List<JsonSlowIndexedFile>,
// Available only if [com.intellij.util.indexing.diagnostic.IndexDiagnosticDumper.shouldDumpPathsOfIndexedFiles] is enabled.
val indexedFiles: List<JsonIndexedFile>?
) {
data class JsonSlowIndexedFile(
val fileName: String,
val processingTime: JsonDuration,
val indexingTime: JsonDuration,
val contentLoadingTime: JsonDuration
)
data class JsonIndexedFile(
val path: PortableFilePath,
@JsonProperty("wfibe")
......
......@@ -8,6 +8,7 @@ import com.intellij.openapi.util.text.HtmlChunk
import com.intellij.openapi.util.text.HtmlChunk.*
import com.intellij.openapi.util.text.StringUtil
import com.intellij.util.indexing.diagnostic.IndexDiagnosticDumper
import com.intellij.util.indexing.diagnostic.IndexingJobStatistics
import com.intellij.util.indexing.diagnostic.dto.*
import org.jetbrains.annotations.Nls
......@@ -162,6 +163,36 @@ fun JsonIndexDiagnostic.generateHtml(): String {
}
}
val hasSlowIndexedFiles = projectIndexingHistory.fileProviderStatistics.any { it.slowIndexedFiles.isNotEmpty() }
if (hasSlowIndexedFiles) {
h1("Slowly indexed files (> ${IndexingJobStatistics.SLOW_FILE_PROCESSING_THRESHOLD_MS} ms)")
table {
thead {
tr {
th("Provider name")
th("File")
th("Content loading time")
th("Indexing time")
th("Total processing time")
}
}
tbody {
for (providerStatistic in projectIndexingHistory.fileProviderStatistics.filter { it.slowIndexedFiles.isNotEmpty() }) {
val slowIndexedFiles = providerStatistic.slowIndexedFiles
for ((index, slowFile) in slowIndexedFiles.sortedByDescending { it.processingTime.nano }.withIndex()) {
tr {
td(if (index == 0) providerStatistic.providerName else "^")
td(slowFile.fileName)
td(slowFile.contentLoadingTime.presentableDuration())
td(slowFile.indexingTime.presentableDuration())
td(slowFile.processingTime.presentableDuration())
}
}
}
}
}
}
h1("Statistics per file type")
table {
thead {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment