diff --git a/app/src/main/java/com/itsaky/androidide/services/builder/GradleBuildService.kt b/app/src/main/java/com/itsaky/androidide/services/builder/GradleBuildService.kt index 587f875d9e..3882e8789f 100644 --- a/app/src/main/java/com/itsaky/androidide/services/builder/GradleBuildService.kt +++ b/app/src/main/java/com/itsaky/androidide/services/builder/GradleBuildService.kt @@ -440,6 +440,12 @@ class GradleBuildService : buildResult = result, ), ) + + buildServiceScope.launch { + ProjectManagerImpl.getInstance() + .indexingServiceManager + .onBuildCompleted() + } } override fun onProgressEvent(event: ProgressEvent) { diff --git a/editor/src/main/java/com/itsaky/androidide/editor/language/treesitter/TreeSitterLanguage.kt b/editor/src/main/java/com/itsaky/androidide/editor/language/treesitter/TreeSitterLanguage.kt index 4c7d677fc3..bd641f3a64 100644 --- a/editor/src/main/java/com/itsaky/androidide/editor/language/treesitter/TreeSitterLanguage.kt +++ b/editor/src/main/java/com/itsaky/androidide/editor/language/treesitter/TreeSitterLanguage.kt @@ -27,6 +27,7 @@ import com.itsaky.androidide.editor.schemes.LanguageSpecProvider.getLanguageSpec import com.itsaky.androidide.editor.schemes.LocalCaptureSpecProvider.newLocalCaptureSpec import com.itsaky.androidide.editor.utils.isNonBlankLine import com.itsaky.androidide.treesitter.TSLanguage +import com.itsaky.androidide.treesitter.TreeSitter import com.itsaky.androidide.utils.IntPair import io.github.rosemoe.sora.editor.ts.TsTheme import io.github.rosemoe.sora.lang.Language.INTERRUPTION_LEVEL_STRONG @@ -42,156 +43,167 @@ import java.io.File * @author Akash Yadav */ abstract class TreeSitterLanguage( - context: Context, - lang: TSLanguage, - private val langType: String + context: Context, + lang: TSLanguage, + private val langType: String ) : IDELanguage() { - private var languageSpec = - getLanguageSpec(context, langType, lang, newLocalCaptureSpec(langType)) - private var tsTheme = TsTheme(languageSpec.spec.tsQuery) - private lateinit var _indentProvider: TreeSitterIndentProvider - private val analyzer by lazy { TreeSitterAnalyzeManager(languageSpec.spec, tsTheme) } - private val newlineHandlersLazy by lazy { createNewlineHandlers() } - - private var languageScheme: LanguageScheme? = null - - private val indentProvider: TreeSitterIndentProvider - get() { - if (!this::_indentProvider.isInitialized) { - this._indentProvider = TreeSitterIndentProvider( - languageSpec, - analyzer.analyzeWorker!!, - getTabSize() - ) - } - - return _indentProvider - } - - companion object { - - private val log = LoggerFactory.getLogger(TreeSitterLanguage::class.java) - private const val DEF_IDENT_ADV = 0 - } - - fun setupWith(scheme: IDEColorScheme?) { - val langScheme = scheme?.languages?.get(langType) - this.languageScheme = langScheme - this.analyzer.langScheme = languageScheme - langScheme?.styles?.forEach { tsTheme.putStyleRule(it.key, it.value.makeStyle()) } - } - - override fun addBreakpoint(line: Int) { - this.analyzer.addBreakpoint(line) - } - - override fun removeBreakpoint(line: Int) { - this.analyzer.removeBreakpoint(line) - } - - override fun removeAllBreakpoints() { - this.analyzer.removeAllBreakpoints() - } - - override fun toggleBreakpoint(line: Int) { - this.analyzer.toggleBreakpoint(line) - } - - override fun highlightLine(line: Int) { - this.analyzer.highlightLine(line) - } - - override fun unhighlightLines() { - this.analyzer.unhighlightLines() - } - - override fun getAnalyzeManager(): AnalyzeManager { - return this.analyzer - } - - override fun getSymbolPairs(): SymbolPairMatch { - return CommonSymbolPairs() - } - - open fun createNewlineHandlers(): Array { - return emptyArray() - } - - override fun getNewlineHandlers(): Array { - return newlineHandlersLazy - } - - override fun getInterruptionLevel(): Int { - return INTERRUPTION_LEVEL_STRONG - } - - override fun getIndentAdvance( - content: ContentReference, - line: Int, - column: Int, - spaceCountOnLine: Int, - tabCountOnLine: Int - ): Int { - return try { - if (line == content.reference.lineCount - 1) { - // line + 1 does not exist - // TODO(itsaky): Update this implementation when this behavior is fixed in sora-editor - return DEF_IDENT_ADV - } - - var linesToReq = LongArray(1) - linesToReq[0] = IntPair.pack(line, column) - - if (content.reference.isNonBlankLine(line + 1)) { - // consider the indentation of the next line only if it is non-blank - linesToReq += IntPair.pack(line + 1, 0) - } - - val indents = this.indentProvider.getIndentsForLines( - content = content.reference, - positions = linesToReq, - ) - - if (indents.size == 1) { - val indent = indents[0] - if (indent == TreeSitterIndentProvider.INDENTATION_ERR) { - return DEF_IDENT_ADV - } - - return indent - (spaceCountOnLine + (tabCountOnLine * getTabSize())) - } - - val (indentLine, indentNxtLine) = indents - if (indentLine == TreeSitterIndentProvider.INDENTATION_ERR - || indentNxtLine == TreeSitterIndentProvider.INDENTATION_ERR) { - log.debug( - "expectedIndent[{}]={}, expectedIndentNextLine[{}]={}, returning default indent advance", - line, indentLine, line + 1, indentNxtLine) - return DEF_IDENT_ADV - } - - return indentNxtLine - indentLine - } catch (e: Exception) { - log.error("An error occurred computing indentation at line:column::{}:{}", line, column, e) - DEF_IDENT_ADV - } - - } - - override fun destroy() { - this.languageSpec.close() - this.languageScheme = null - } - - /** A [Factory] creates instance of a specific [TreeSitterLanguage] implementation. */ - fun interface Factory { - - /** - * Create the instance of the [TreeSitterLanguage] implementation. - * - * @param context The current context. - */ - fun create(context: Context): T - } + private var languageSpec = + getLanguageSpec(context, langType, lang, newLocalCaptureSpec(langType)) + private var tsTheme = TsTheme(languageSpec.spec.tsQuery) + private lateinit var _indentProvider: TreeSitterIndentProvider + private val analyzer by lazy { TreeSitterAnalyzeManager(languageSpec.spec, tsTheme) } + private val newlineHandlersLazy by lazy { createNewlineHandlers() } + + private var languageScheme: LanguageScheme? = null + + private val indentProvider: TreeSitterIndentProvider + get() { + if (!this::_indentProvider.isInitialized) { + this._indentProvider = TreeSitterIndentProvider( + languageSpec, + analyzer.analyzeWorker!!, + getTabSize() + ) + } + + return _indentProvider + } + + companion object { + + init { + TreeSitter.loadLibrary() + } + + private val log = LoggerFactory.getLogger(TreeSitterLanguage::class.java) + private const val DEF_IDENT_ADV = 0 + } + + fun setupWith(scheme: IDEColorScheme?) { + val langScheme = scheme?.languages?.get(langType) + this.languageScheme = langScheme + this.analyzer.langScheme = languageScheme + langScheme?.styles?.forEach { tsTheme.putStyleRule(it.key, it.value.makeStyle()) } + } + + override fun addBreakpoint(line: Int) { + this.analyzer.addBreakpoint(line) + } + + override fun removeBreakpoint(line: Int) { + this.analyzer.removeBreakpoint(line) + } + + override fun removeAllBreakpoints() { + this.analyzer.removeAllBreakpoints() + } + + override fun toggleBreakpoint(line: Int) { + this.analyzer.toggleBreakpoint(line) + } + + override fun highlightLine(line: Int) { + this.analyzer.highlightLine(line) + } + + override fun unhighlightLines() { + this.analyzer.unhighlightLines() + } + + override fun getAnalyzeManager(): AnalyzeManager { + return this.analyzer + } + + override fun getSymbolPairs(): SymbolPairMatch { + return CommonSymbolPairs() + } + + open fun createNewlineHandlers(): Array { + return emptyArray() + } + + override fun getNewlineHandlers(): Array { + return newlineHandlersLazy + } + + override fun getInterruptionLevel(): Int { + return INTERRUPTION_LEVEL_STRONG + } + + override fun getIndentAdvance( + content: ContentReference, + line: Int, + column: Int, + spaceCountOnLine: Int, + tabCountOnLine: Int + ): Int { + return try { + if (line == content.reference.lineCount - 1) { + // line + 1 does not exist + // TODO(itsaky): Update this implementation when this behavior is fixed in sora-editor + return DEF_IDENT_ADV + } + + var linesToReq = LongArray(1) + linesToReq[0] = IntPair.pack(line, column) + + if (content.reference.isNonBlankLine(line + 1)) { + // consider the indentation of the next line only if it is non-blank + linesToReq += IntPair.pack(line + 1, 0) + } + + val indents = this.indentProvider.getIndentsForLines( + content = content.reference, + positions = linesToReq, + ) + + if (indents.size == 1) { + val indent = indents[0] + if (indent == TreeSitterIndentProvider.INDENTATION_ERR) { + return DEF_IDENT_ADV + } + + return indent - (spaceCountOnLine + (tabCountOnLine * getTabSize())) + } + + val (indentLine, indentNxtLine) = indents + if (indentLine == TreeSitterIndentProvider.INDENTATION_ERR + || indentNxtLine == TreeSitterIndentProvider.INDENTATION_ERR + ) { + log.debug( + "expectedIndent[{}]={}, expectedIndentNextLine[{}]={}, returning default indent advance", + line, indentLine, line + 1, indentNxtLine + ) + return DEF_IDENT_ADV + } + + return indentNxtLine - indentLine + } catch (e: Exception) { + log.error( + "An error occurred computing indentation at line:column::{}:{}", + line, + column, + e + ) + DEF_IDENT_ADV + } + + } + + override fun destroy() { + this.languageSpec.close() + this.languageScheme = null + } + + /** A [Factory] creates instance of a specific [TreeSitterLanguage] implementation. */ + fun interface Factory { + + /** + * Create the instance of the [TreeSitterLanguage] implementation. + * + * @param context The current context. + */ + fun create(context: Context): T + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 144a3736dd..a62003b2bc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ activityKtx = "1.8.2" agp = "8.8.2" agp-tooling = "8.11.0" -appcompat = "1.6.1" +androidx-sqlite = "2.6.2" appcompatVersion = "1.7.1" bcprovJdk18on = "1.80" colorpickerview = "2.3.0" @@ -97,6 +97,8 @@ androidx-palette-ktx = { module = "androidx.palette:palette-ktx", version.ref = androidx-preference-ktx = { module = "androidx.preference:preference-ktx", version.ref = "preferenceKtxVersion" } androidx-recyclerview-v132 = { module = "androidx.recyclerview:recyclerview", version.ref = "recyclerview" } androidx-security-crypto = { module = "androidx.security:security-crypto", version.ref = "securityCrypto" } +androidx-sqlite-ktx = { module = "androidx.sqlite:sqlite-ktx", version.ref = "androidx-sqlite" } +androidx-sqlite-framework = { module = "androidx.sqlite:sqlite-framework", version.ref = "androidx-sqlite" } androidx-viewpager2-v110beta02 = { module = "androidx.viewpager2:viewpager2", version.ref = "viewpager2" } bcpkix-jdk18on = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bcprovJdk18on" } bcprov-jdk18on = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bcprovJdk18on" } @@ -121,10 +123,8 @@ desugar_jdk_libs-v215 = { module = "com.android.tools:desugar_jdk_libs", version google-genai = { module = "com.google.genai:google-genai", version.ref = "googleGenai" } gson-v2101 = { module = "com.google.code.gson:gson", version.ref = "gson" } koin-android = { module = "io.insert-koin:koin-android", version.ref = "koinAndroid" } -kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib" } kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutinesCore" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } -play-services-oss-licenses = { module = "com.google.android.gms:play-services-oss-licenses", version.ref = "playServicesOssLicenses" } room-compiler = { module = "androidx.room:room-compiler", version.ref = "room" } room-ktx = { module = "androidx.room:room-ktx", version.ref = "room" } @@ -311,6 +311,7 @@ monitor = { group = "androidx.test", name = "monitor", version.ref = "monitorVer org-json = { module = "org.json:json", version = "20210307"} pebble = { module = "io.pebbletemplates:pebble", version.ref = "pebble" } +kotlinx-metadata = { module = "org.jetbrains.kotlin:kotlin-metadata-jvm", version.ref = "kotlin" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/lsp/indexing/build.gradle.kts b/lsp/indexing/build.gradle.kts new file mode 100644 index 0000000000..b81c2d47b3 --- /dev/null +++ b/lsp/indexing/build.gradle.kts @@ -0,0 +1,19 @@ +import com.itsaky.androidide.build.config.BuildConfig + +plugins { + id("com.android.library") + id("kotlin-android") +} + +android { + namespace = "${BuildConfig.PACKAGE_NAME}.lsp.indexing" +} + +dependencies { + api(libs.androidx.annotation) + api(libs.androidx.sqlite.ktx) + api(libs.androidx.sqlite.framework) + api(libs.kotlinx.coroutines.core) + + api(projects.logger) +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt new file mode 100644 index 0000000000..d5a8527ca3 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -0,0 +1,112 @@ +package org.appdevforall.codeonthego.indexing + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filter +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.ReadableIndex +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * A read-only view over an index that only exposes entries + * from a set of active sources. + * + * The underlying index retains ALL data (it's a persistent cache). + * This view controls which subset is visible based on which + * sources (JAR paths, etc.) are currently "active." + * + * @param T The indexed type. + * @param backing The underlying index that holds all data. + */ +open class FilteredIndex( + private val backing: ReadableIndex, +) : ReadableIndex, Closeable { + + /** + * The set of source IDs whose entries are visible. + * Uses a concurrent set for thread-safe reads during queries. + */ + private val activeSources = ConcurrentHashMap.newKeySet() + + /** + * Make a source's entries visible in query results. + */ + fun activateSource(sourceId: String) { + activeSources.add(sourceId) + } + + /** + * Hide a source's entries from query results. + * The data remains in the backing index. + */ + fun deactivateSource(sourceId: String) { + activeSources.remove(sourceId) + } + + /** + * Replace the entire active set. Sources not in [sourceIds] + * become invisible; sources in [sourceIds] become visible. + * + * This is the typical call on project sync: pass in all + * current classpath JAR paths. + */ + fun setActiveSources(sourceIds: Set) { + activeSources.clear() + activeSources.addAll(sourceIds) + } + + /** + * Returns the current set of active source IDs. + */ + fun activeSources(): Set = + activeSources.toSet() + + /** + * Returns true if the source is currently active (visible). + */ + fun isActive(sourceId: String): Boolean = + sourceId in activeSources + + /** + * Returns true if the source exists in the backing index, + * regardless of whether it's active. + * + * Use this to check if a JAR needs indexing at all. + */ + suspend fun isCached(sourceId: String): Boolean = + backing.containsSource(sourceId) + + override fun query(query: IndexQuery): Flow { + // If the query already specifies a sourceId, check if it's active + if (query.sourceId != null && query.sourceId !in activeSources) { + return kotlinx.coroutines.flow.emptyFlow() + } + + return backing.query(query).filter { it.sourceId in activeSources } + } + + override suspend fun get(key: String): T? { + val entry = backing.get(key) ?: return null + return if (entry.sourceId in activeSources) entry else null + } + + override suspend fun containsSource(sourceId: String): Boolean { + return sourceId in activeSources && backing.containsSource(sourceId) + } + + override fun distinctValues(fieldName: String): Flow { + // This is imprecise — the backing index may return values + // from inactive sources. For exact results, we'd need to + // query all entries and filter. For package enumeration + // (the main use case), this approximation is acceptable + // since packages from inactive JARs are harmless — they + // just produce empty results when queried further. + return backing.distinctValues(fieldName) + } + + override fun close() { + activeSources.clear() + if (backing is Closeable) backing.close() + } +} \ No newline at end of file diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt new file mode 100644 index 0000000000..20067e998e --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -0,0 +1,226 @@ +package org.appdevforall.codeonthego.indexing + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.collections.iterator +import kotlin.concurrent.read +import kotlin.concurrent.write + +/** + * A thread-safe, in-memory [Index] backed by [ConcurrentHashMap]. + * + * Optimized for small-to-medium datasets (source files, typically + * hundreds to low thousands of entries) that change frequently. + * + * Data layout: + * - [primaryMap]: key → entry (O(1) point lookup) + * - [sourceMap]: sourceId → set of keys (O(1) bulk removal) + * - [fieldMaps]: fieldName → (fieldValue → set of keys) (equality filter) + * - [prefixBuckets]: fieldName → (lowercased first char → list of (value, key)) + * Provides a ~36-way partition for prefix search. + * + * All mutations go through [lock] in write mode for consistency + * across the multiple maps. Reads use read mode. + * + * @param T The indexed entry type. + * @param descriptor Defines queryable fields and serialization. + */ +class InMemoryIndex( + override val descriptor: IndexDescriptor, + override val name: String = "memory:${descriptor.name}", +) : Index { + + private val primaryMap = ConcurrentHashMap(256) + private val sourceMap = ConcurrentHashMap>(32) + private val fieldMaps = ConcurrentHashMap>>() + private val prefixBuckets = ConcurrentHashMap>>() + + private val lock = ReentrantReadWriteLock() + + private data class PrefixEntry(val lowerValue: String, val key: String) + + init { + for (field in descriptor.fields) { + fieldMaps[field.name] = ConcurrentHashMap() + if (field.prefixSearchable) { + prefixBuckets[field.name] = ConcurrentHashMap() + } + } + } + + override fun query(query: IndexQuery): Flow = flow { + val keys = resolveMatchingKeys(query) + var emitted = 0 + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + + for (key in keys) { + if (emitted >= limit) break + val entry = primaryMap[key] ?: continue + emit(entry) + emitted++ + } + } + + override suspend fun get(key: String): T? = primaryMap[key] + + override suspend fun containsSource(sourceId: String): Boolean = + sourceMap.containsKey(sourceId) + + override fun distinctValues(fieldName: String): Flow = flow { + val fieldMap = fieldMaps[fieldName] ?: return@flow + lock.read { + for (value in fieldMap.keys) { + emit(value) + } + } + } + + override suspend fun insert(entries: Flow) { + entries.collect { entry -> insertSingle(entry) } + } + + override suspend fun insertAll(entries: Sequence) { + lock.write { + for (entry in entries) { + insertSingleLocked(entry) + } + } + } + + override suspend fun insert(entry: T) = insertSingle(entry) + + override suspend fun removeBySource(sourceId: String) = lock.write { + val keys = sourceMap.remove(sourceId) ?: return@write + for (key in keys) { + val entry = primaryMap.remove(key) ?: continue + removeFromSecondaryIndexes(entry) + } + } + + override suspend fun clear() = lock.write { + primaryMap.clear() + sourceMap.clear() + fieldMaps.values.forEach { it.clear() } + prefixBuckets.values.forEach { it.clear() } + } + + val size: Int get() = primaryMap.size + val sourceCount: Int get() = sourceMap.size + + /** + * Resolves the set of keys matching the query by intersecting + * the results of each predicate. + * + * Starts with the most selective predicate to minimize the + * intersection set. + */ + private fun resolveMatchingKeys(query: IndexQuery): Sequence = lock.read { + var candidates: Set? = null + + if (query.key != null) { + return@read if (primaryMap.containsKey(query.key)) { + sequenceOf(query.key) + } else { + emptySequence() + } + } + + if (query.sourceId != null) { + candidates = intersect(candidates, sourceMap[query.sourceId]) + } + + for ((field, value) in query.exactMatch) { + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + candidates = intersect(candidates, fieldMap[value]) + } + + for ((field, prefix) in query.prefixMatch) { + val buckets = prefixBuckets[field] ?: return@read emptySequence() + val lowerPrefix = prefix.lowercase() + val firstChar = lowerPrefix.firstOrNull() ?: continue + val bucket = buckets[firstChar] ?: return@read emptySequence() + + val matching = bucket.asSequence() + .filter { it.lowerValue.startsWith(lowerPrefix) } + .map { it.key } + .toSet() + + candidates = intersect(candidates, matching) + } + + for ((field, mustExist) in query.presence) { + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + val allKeysWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } + candidates = if (mustExist) { + intersect(candidates, allKeysWithField) + } else { + // Keys that DON'T have this field + val allKeys = primaryMap.keys.toMutableSet() + allKeys.removeAll(allKeysWithField) + intersect(candidates, allKeys) + } + } + + candidates?.asSequence() ?: primaryMap.keys.asSequence() + } + + private fun intersect(current: Set?, other: Set?): Set? { + if (other == null) return current + if (current == null) return other + return current.intersect(other) + } + + private fun insertSingle(entry: T) = lock.write { + insertSingleLocked(entry) + } + + private fun insertSingleLocked(entry: T) { + val existing = primaryMap[entry.key] + if (existing != null) { + removeFromSecondaryIndexes(existing) + } + + primaryMap[entry.key] = entry + sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(entry.key) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName] + ?.getOrPut(value) { mutableSetOf() } + ?.add(entry.key) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets.getOrPut(firstChar) { mutableListOf() } + .add(PrefixEntry(lower, entry.key)) + } + } + } + + private fun removeFromSecondaryIndexes(entry: T) { + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName]?.get(value)?.remove(entry.key) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets[firstChar]?.removeAll { it.key == entry.key } + } + } + // Note: sourceMap is handled by the caller + } +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt new file mode 100644 index 0000000000..af39930033 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt @@ -0,0 +1,82 @@ +package org.appdevforall.codeonthego.indexing + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.launch +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.ReadableIndex +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * Merges query results from multiple [ReadableIndex] instances. + * + * @param T The indexed type. + * @param indexes The indexes to merge, in priority order. + */ +class MergedIndex( + private val indexes: List>, +) : ReadableIndex, Closeable { + + constructor(vararg indexes: ReadableIndex) : this(indexes.toList()) + + override fun query(query: IndexQuery): Flow = channelFlow { + val seen = ConcurrentHashMap.newKeySet() + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + val emitted = java.util.concurrent.atomic.AtomicInteger(0) + + // Launch a producer coroutine per index. + // channelFlow provides structured concurrency: when the + // collector stops (limit reached), all producers are cancelled. + for (index in indexes) { + launch { + index.query(query).collect { entry -> + if (emitted.get() >= limit) { + return@collect + } + if (seen.add(entry.key)) { + send(entry) + if (emitted.incrementAndGet() >= limit) { + // Close the channel - cancels other producers + channel.close() + return@collect + } + } + } + } + } + } + + override suspend fun get(key: String): T? { + // First match wins (priority order) + for (index in indexes) { + val result = index.get(key) + if (result != null) return result + } + return null + } + + override suspend fun containsSource(sourceId: String): Boolean { + return indexes.any { it.containsSource(sourceId) } + } + + override fun distinctValues(fieldName: String): Flow = channelFlow { + val seen = ConcurrentHashMap.newKeySet() + for (index in indexes) { + launch { + index.distinctValues(fieldName).collect { value -> + if (seen.add(value)) { + send(value) + } + } + } + } + } + + override fun close() { + for (index in indexes) { + if (index is Closeable) index.close() + } + } +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/PersistentIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/PersistentIndex.kt new file mode 100644 index 0000000000..f3b0cf539b --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/PersistentIndex.kt @@ -0,0 +1,336 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.ContentValues +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.sqlite.db.SupportSQLiteOpenHelper +import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import kotlin.collections.iterator + +/** + * A persistent [Index] backed by SQLite via AndroidX. + * + * Creates a table dynamically based on the [IndexDescriptor]: + * ``` + * CREATE TABLE IF NOT EXISTS {name} ( + * _key TEXT PRIMARY KEY, + * _source_id TEXT NOT NULL, + * f_{field1} TEXT, + * f_{field1}_lower TEXT, -- if prefix-searchable + * f_{field2} TEXT, + * ... + * _payload BLOB NOT NULL + * ); + * ``` + * + * SQL indexes are created on: + * - `_source_id` (for bulk removal) + * - Each `f_{field}` (for equality filter) + * - Each `f_{field}_lower` (for prefix search via `LIKE 'prefix%'`) + * + * Uses WAL journal mode for concurrent read/write performance. + * Inserts are batched inside transactions for throughput. + * + * @param T The indexed entry type. + * @param descriptor Defines fields and serialization. + * @param context Android context (for database file location). + * @param dbName Database file name. Different index types can share + * a database (each gets its own table) or use separate files. + * @param batchSize Number of rows per INSERT transaction. + */ +class PersistentIndex( + override val descriptor: IndexDescriptor, + context: Context, + dbName: String, + override val name: String = "persistent:${descriptor.name}", + private val batchSize: Int = 500, +) : Index { + + private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") + + // Field column names: "f_{fieldName}" + private val fieldColumns = descriptor.fields.associate { field -> + field.name to "f_${field.name}" + } + + // Prefix-searchable fields also get a "_lower" column + private val prefixColumns = descriptor.fields + .filter { it.prefixSearchable } + .associate { it.name to "f_${it.name}_lower" } + + private val db: SupportSQLiteDatabase + + init { + val config = SupportSQLiteOpenHelper.Configuration.builder(context) + .name(dbName) + .callback(object : SupportSQLiteOpenHelper.Callback(1) { + override fun onCreate(db: SupportSQLiteDatabase) { + createTable(db) + } + + override fun onUpgrade( + db: SupportSQLiteDatabase, oldVersion: Int, newVersion: Int, + ) { + // TODO: Add migration support + db.execSQL("DROP TABLE IF EXISTS $tableName") + createTable(db) + } + + override fun onOpen(db: SupportSQLiteDatabase) { + } + }) + .build() + + db = FrameworkSQLiteOpenHelperFactory() + .create(config) + .writableDatabase + + // Ensure table exists (for shared databases) + createTable(db) + } + + override fun query(query: IndexQuery): Flow = flow { + val (sql, args) = buildSelectQuery(query) + val cursor = db.query(sql, args.toTypedArray()) + + cursor.use { + val payloadIdx = it.getColumnIndexOrThrow("_payload") + while (it.moveToNext()) { + val bytes = it.getBlob(payloadIdx) + emit(descriptor.deserialize(bytes)) + } + } + }.flowOn(Dispatchers.IO) + + override suspend fun get(key: String): T? = withContext(Dispatchers.IO) { + val cursor = db.query( + "SELECT _payload FROM $tableName WHERE _key = ? LIMIT 1", + arrayOf(key), + ) + cursor.use { + if (it.moveToFirst()) { + descriptor.deserialize(it.getBlob(0)) + } else null + } + } + + override suspend fun containsSource(sourceId: String): Boolean = + withContext(Dispatchers.IO) { + val cursor = db.query( + "SELECT 1 FROM $tableName WHERE _source_id = ? LIMIT 1", + arrayOf(sourceId), + ) + cursor.use { it.moveToFirst() } + } + + override fun distinctValues(fieldName: String): Flow = flow { + val col = fieldColumns[fieldName] + ?: throw IllegalArgumentException("Unknown field: $fieldName") + + val cursor = db.query("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") + cursor.use { + val idx = 0 + while (it.moveToNext()) { + emit(it.getString(idx)) + } + } + }.flowOn(Dispatchers.IO) + + /** + * Streaming insert from a [Flow]. + * + * Collects entries from the flow and inserts them in batched + * transactions. Each batch is a single SQLite transaction - + * this is orders of magnitude faster than one transaction per row. + * + * The flow is collected on [Dispatchers.IO]. + */ + override suspend fun insert(entries: Flow) = withContext(Dispatchers.IO) { + val batch = mutableListOf() + entries.collect { entry -> + batch.add(entry) + if (batch.size >= batchSize) { + insertBatch(batch) + batch.clear() + } + } + // Flush remaining + if (batch.isNotEmpty()) { + insertBatch(batch) + } + } + + override suspend fun insertAll(entries: Sequence) = withContext(Dispatchers.IO) { + val batch = mutableListOf() + for (entry in entries) { + batch.add(entry) + if (batch.size >= batchSize) { + insertBatch(batch) + batch.clear() + } + } + if (batch.isNotEmpty()) { + insertBatch(batch) + } + } + + override suspend fun insert(entry: T) = withContext(Dispatchers.IO) { + insertBatch(listOf(entry)) + } + + override suspend fun removeBySource(sourceId: String) = withContext(Dispatchers.IO) { + db.execSQL("DELETE FROM $tableName WHERE _source_id = ?", arrayOf(sourceId)) + } + + override suspend fun clear() = withContext(Dispatchers.IO) { + db.execSQL("DELETE FROM $tableName") + } + + override fun close() { + db.close() + } + + suspend fun size(): Int = withContext(Dispatchers.IO) { + val cursor = db.query("SELECT COUNT(*) FROM $tableName") + cursor.use { if (it.moveToFirst()) it.getInt(0) else 0 } + } + + private fun createTable(db: SupportSQLiteDatabase) { + val columns = buildString { + append("_key TEXT PRIMARY KEY, ") + append("_source_id TEXT NOT NULL, ") + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + append("$col TEXT, ") + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + append("$lowerCol TEXT, ") + } + } + + append("_payload BLOB NOT NULL") + } + + db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") + + // Indexes + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_source ON $tableName(_source_id)" + ) + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$col ON $tableName($col)" + ) + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$lowerCol ON $tableName($lowerCol)" + ) + } + } + } + + private fun insertBatch(entries: List) { + db.beginTransaction() + try { + for (entry in entries) { + val cv = ContentValues().apply { + put("_key", entry.key) + put("_source_id", entry.sourceId) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + val col = fieldColumns[fieldName] ?: continue + put(col, value) + + val lowerCol = prefixColumns[fieldName] + if (lowerCol != null) { + put(lowerCol, value?.lowercase()) + } + } + + put("_payload", descriptor.serialize(entry)) + } + + db.insert( + tableName, + SQLiteDatabase.CONFLICT_REPLACE, + cv, + ) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + + private data class SqlQuery(val sql: String, val args: List) + + private fun buildSelectQuery(query: IndexQuery): SqlQuery { + val where = StringBuilder() + val args = mutableListOf() + + fun and(clause: String, vararg values: String) { + if (where.isNotEmpty()) where.append(" AND ") + where.append(clause) + args.addAll(values) + } + + query.key?.let { and("_key = ?", it) } + query.sourceId?.let { and("_source_id = ?", it) } + + for ((field, value) in query.exactMatch) { + val col = fieldColumns[field] ?: continue + and("$col = ?", value) + } + + for ((field, prefix) in query.prefixMatch) { + val lowerCol = prefixColumns[field] + if (lowerCol != null) { + // Use the pre-lowercased column for index-friendly LIKE + and("$lowerCol LIKE ?", "${prefix.lowercase()}%") + } else { + // Fallback: case-sensitive prefix on the regular column + val col = fieldColumns[field] ?: continue + and("$col LIKE ?", "$prefix%") + } + } + + for ((field, mustExist) in query.presence) { + val col = fieldColumns[field] ?: continue + if (mustExist) { + and("$col IS NOT NULL") + } else { + and("$col IS NULL") + } + } + + val sql = buildString { + append("SELECT _payload FROM $tableName") + if (where.isNotEmpty()) { + append(" WHERE ") + append(where) + } + if (query.limit > 0) { + append(" LIMIT ${query.limit}") + } + } + + return SqlQuery(sql, args) + } +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt new file mode 100644 index 0000000000..b595e604ba --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -0,0 +1,91 @@ +package org.appdevforall.codeonthego.indexing.api + +/** + * Any object that can be stored in an index. + * + * The only requirements are a unique key (for deduplication and + * point lookups) and a source identifier (for bulk operations + * when the source changes). + * + * What constitutes a "key" and "source" depends entirely on + * the consumer: + * - For Kotlin symbols: key = FQN, source = JAR path or file path + * - For Android resources: key = resource ID, source = AAR path + * - For Python symbols: key = qualified name, source = .py file path + */ +interface Indexable { + + /** Unique identifier within the index. */ + val key: String + + /** + * Identifies the origin of this entry. + * All entries sharing a [sourceId] can be removed atomically + * via [WritableIndex.removeBySource]. + */ + val sourceId: String +} + +/** + * Describes how to index, serialize, and query a specific type. + * + * Acts as the bridge between domain objects and the storage layer. + * A single index instance is parameterized by one descriptor - + * different data types get different index instances. + * + * @param T The domain type being indexed. + */ +interface IndexDescriptor { + + /** + * A unique name for this index type. Used as the table name + * in persistent storage and the namespace in composite indexes. + */ + val name: String + + /** + * The fields that should be queryable. + * Defines the "schema" for this index type. + * + * The persistent layer will create SQL columns and indexes + * for each declared field. + */ + val fields: List + + /** + * Extract the queryable field values from an entry. + * + * The returned map's keys must be a subset of [fields]'s names. + * Null values mean the field is not applicable for this entry + * (e.g. receiverType is null for a non-extension function). + */ + fun fieldValues(entry: T): Map + + /** + * Serialize an entry to bytes for persistent storage. + * + * Use whatever format is appropriate - protobuf, JSON, + * custom binary. Called once on insert; the bytes are + * stored opaquely. + */ + fun serialize(entry: T): ByteArray + + /** + * Deserialize bytes back into an entry. + * Must be the inverse of [serialize]. + */ + fun deserialize(bytes: ByteArray): T +} + +/** + * Declares a queryable field on an [IndexDescriptor]. + * + * @param name The field name (used in queries and as the column name). + * @param prefixSearchable Whether this field supports prefix queries + * (e.g. name prefix for completions). Affects how + * the persistent layer creates SQL indexes. + */ +data class IndexField( + val name: String, + val prefixSearchable: Boolean = false, +) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt new file mode 100644 index 0000000000..39f9846494 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -0,0 +1,105 @@ +package org.appdevforall.codeonthego.indexing.api + +import kotlinx.coroutines.flow.Flow +import java.io.Closeable + +/** + * Read-only view of an index. + * + * All query methods return [Flow]s and results are produced lazily. + * The consumer decides how many to take, which dispatcher to + * collect on, and whether to buffer. + * + * @param T The indexed type. + */ +interface ReadableIndex { + + /** + * Query the index. Returns a lazy [Flow] of matching entries. + * + * Results are not guaranteed to be in any particular order + * unless the implementation specifies otherwise. + * + * If [IndexQuery.limit] is 0, all matches are emitted. + */ + fun query(query: IndexQuery): Flow + + /** + * Point lookup by key. Returns null if not found. + */ + suspend fun get(key: String): T? + + /** + * Fast existence check for a source. + */ + suspend fun containsSource(sourceId: String): Boolean + + /** + * Returns distinct values for a given field across all entries. + * + * Useful for enumerating packages, kinds, etc. without + * deserializing full entries. + * + * @param fieldName Must be one of the fields declared in the + * [IndexDescriptor]. + */ + fun distinctValues(fieldName: String): Flow +} + +/** + * Write interface for mutating an index. + * + * Accepts [Flow]s for streaming inserts so that the producer can + * yield entries one at a time without holding the entire set + * in memory. + */ +interface WritableIndex { + + /** + * Insert entries from a [Flow]. + * + * Entries are consumed lazily from the flow and batched + * internally for throughput. If an entry with the same key + * already exists, it is replaced. + * + * The flow is collected on the caller's dispatcher; the + * implementation handles its own threading for storage I/O. + */ + suspend fun insert(entries: Flow) + + /** + * Convenience: insert a sequence (also lazy). + */ + suspend fun insertAll(entries: Sequence) + + /** + * Convenience: insert a single entry. + */ + suspend fun insert(entry: T) + + /** + * Remove all entries from the given source. + */ + suspend fun removeBySource(sourceId: String) + + /** + * Remove all entries. + */ + suspend fun clear() +} + +/** + * A complete index with read, write, and lifecycle management. + * + * @param T The indexed type. + */ +interface Index : ReadableIndex, WritableIndex, Closeable { + + /** Human-readable name for logging. */ + val name: String + + /** The descriptor governing serialization and field extraction. */ + val descriptor: IndexDescriptor + + override fun close() {} +} diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt new file mode 100644 index 0000000000..8b3e1a6a12 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -0,0 +1,78 @@ +package org.appdevforall.codeonthego.indexing.api + +/** + * A query against an index. + * + * All predicates are ANDed together. The query is intentionally + * field-based (not type-specific) so the same query engine works + * for Kotlin symbols, Android resources, Python declarations, etc. + */ +data class IndexQuery( + /** Exact match predicates: field name → expected value. */ + val exactMatch: Map = emptyMap(), + + /** Prefix match predicates: field name → prefix (case-insensitive). */ + val prefixMatch: Map = emptyMap(), + + /** + * Presence predicates: field name → whether the field must be + * non-null (true) or null (false). + */ + val presence: Map = emptyMap(), + + /** Filter by source ID. */ + val sourceId: String? = null, + + /** Filter by key (exact). */ + val key: String? = null, + + /** Maximum number of results. 0 = unlimited (use with care). */ + val limit: Int = 200, +) { + companion object { + /** Match everything up to [limit]. */ + val ALL = IndexQuery() + + /** Exact key lookup. */ + fun byKey(key: String) = IndexQuery(key = key, limit = 1) + + /** All entries from a specific source. */ + fun bySource(sourceId: String) = IndexQuery(sourceId = sourceId, limit = 0) + } +} + +/** + * DSL builder for [IndexQuery]. + */ +class IndexQueryBuilder { + private val exact = mutableMapOf() + private val prefix = mutableMapOf() + private val pres = mutableMapOf() + var sourceId: String? = null + var key: String? = null + var limit: Int = 200 + + /** Exact match on a field. */ + fun eq(field: String, value: String) { exact[field] = value } + + /** Prefix match on a field (case-insensitive). */ + fun prefix(field: String, value: String) { prefix[field] = value } + + /** Field must be non-null. */ + fun exists(field: String) { pres[field] = true } + + /** Field must be null. */ + fun notExists(field: String) { pres[field] = false } + + fun build() = IndexQuery( + exactMatch = exact.toMap(), + prefixMatch = prefix.toMap(), + presence = pres.toMap(), + sourceId = sourceId, + key = key, + limit = limit, + ) +} + +inline fun indexQuery(block: IndexQueryBuilder.() -> Unit): IndexQuery = + IndexQueryBuilder().apply(block).build() diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexRegistry.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexRegistry.kt new file mode 100644 index 0000000000..e2639c5cae --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexRegistry.kt @@ -0,0 +1,119 @@ +package org.appdevforall.codeonthego.indexing.service + +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * A typed key for retrieving an index from the [IndexRegistry]. + * + * @param T The index type. Not restricted to [org.appdevforall.codeonthego.indexing.api.Index], can be a + * domain-specific facade. + */ +data class IndexKey( + val name: String, +) + +/** + * Central registry where [IndexingService]s publish their indexes + * and consumers (LSPs, etc.) retrieve them. + */ +class IndexRegistry : Closeable { + + private val indexes = ConcurrentHashMap() + private val listeners = ConcurrentHashMap Unit>>() + + /** + * Register an index. Replaces any previously registered index + * with the same key. + * + * If there are listeners waiting for this key, they are notified + * immediately. + */ + fun register(key: IndexKey, index: T) { + val old = indexes.put(key.name, index) + + // Close the old index if it's Closeable + if (old is Closeable && old !== index) { + old.close() + } + + // Notify listeners + listeners[key.name]?.forEach { listener -> + @Suppress("UNCHECKED_CAST") + (listener as (T) -> Unit).invoke(index) + } + } + + /** + * Retrieve an index by key. Returns null if not yet registered. + */ + @Suppress("UNCHECKED_CAST") + fun get(key: IndexKey): T? = + indexes[key.name] as? T + + /** + * Retrieve an index, throwing if not available. + */ + fun require(key: IndexKey): T = + get(key) ?: throw IllegalStateException( + "Index '${key.name}' is not registered. " + + "Has the corresponding IndexingService been initialized?" + ) + + /** + * Execute a block if the index is available. + */ + inline fun ifAvailable( + key: IndexKey, + block: (T) -> R, + ): R? { + val index = get(key) ?: return null + return block(index) + } + + /** + * Register a listener that will be called when an index + * is registered (or re-registered) with the given key. + * + * If the index is already registered, the listener is + * called immediately. + */ + fun onAvailable(key: IndexKey, listener: (T) -> Unit) { + @Suppress("UNCHECKED_CAST") + listeners.getOrPut(key.name) { mutableListOf() } + .add(listener as (Any) -> Unit) + + // If already registered, notify immediately + get(key)?.let { listener(it) } + } + + /** + * Unregister an index. The caller is responsible for closing it. + */ + fun unregister(key: IndexKey): T? { + @Suppress("UNCHECKED_CAST") + return indexes.remove(key.name) as? T + } + + /** + * Returns true if an index is registered for this key. + */ + fun isRegistered(key: IndexKey): Boolean = + indexes.containsKey(key.name) + + /** + * Returns all registered keys. + */ + fun registeredKeys(): Set = indexes.keys.toSet() + + /** + * Close and remove all registered indexes. + */ + override fun close() { + indexes.values.forEach { index -> + if (index is Closeable) index.close() + } + indexes.clear() + listeners.clear() + } +} \ No newline at end of file diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingService.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingService.kt new file mode 100644 index 0000000000..cbf074cce2 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingService.kt @@ -0,0 +1,41 @@ +package org.appdevforall.codeonthego.indexing.service +import java.io.Closeable + +/** + * A service that knows how to build and maintain an index for a + * specific domain. + * + * Implementations should be stateless with respect to the project + * model because they receive it as a parameter, not as a constructor + * argument. This allows the same service instance to handle + * re-syncs without recreation. + */ +interface IndexingService : Closeable { + + /** + * Unique identifier for this service. + * Used for logging and debugging. + */ + val id: String + + /** + * The keys of the indexes this service registers. + * Used by the manager to verify all expected indexes + * are available after initialization. + */ + val providedKeys: List> + + /** + * Called once after the service is registered. + * + * Create your index instances here and register them + * with the [registry]. + */ + suspend fun initialize(registry: IndexRegistry) + + /** + * Called when the project is closed or the IDE shuts down. + * Release all resources. + */ + override fun close() {} +} \ No newline at end of file diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt new file mode 100644 index 0000000000..6575c5df63 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/service/IndexingServiceManager.kt @@ -0,0 +1,158 @@ +package org.appdevforall.codeonthego.indexing.service + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.launch +import org.slf4j.LoggerFactory +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * Manages the lifecycle of [IndexingService]s and the [IndexRegistry]. + */ +class IndexingServiceManager( + private val scope: CoroutineScope = CoroutineScope( + SupervisorJob() + Dispatchers.Default + ), +) : Closeable { + + companion object { + private val log = LoggerFactory.getLogger(IndexingServiceManager::class.java) + } + + /** + * The central registry. All services register their indexes here. + * Consumers (LSPs, etc.) retrieve indexes from here. + */ + val registry = IndexRegistry() + + private val services = ConcurrentHashMap() + private var initialized = false + + /** + * Register an [IndexingService]. + * + * Must be called before [onProjectSynced]. Services are initialized + * in registration order. + * + * @throws IllegalStateException if called after initialization. + */ + fun register(service: IndexingService) { + check(!initialized) { + "Cannot register services after initialization. " + + "Register all services before the first onProjectSynced call." + } + + if (services.putIfAbsent(service.id, service) != null) { + log.warn("Attempt to re-register service with ID: {}", service.id) + return + } + + log.info("Registered indexing service: {}", service.id) + } + + /** + * Called after project sync (e.g. Gradle sync) completes. + * + * On the first call, initializes all registered services + * (creates indexes, registers them). On subsequent calls, + * notifies services of the updated project model. + * + * Services process the event concurrently. Failures in one + * service don't affect others (SupervisorJob). + */ + fun onProjectSynced() { + scope.launch { + if (!initialized) { + initializeServices() + initialized = true + } + } + } + + /** + * Called after a build completes. + */ + fun onBuildCompleted() { + if (!initialized) { + log.warn("onBuildCompleted called before initialization, ignoring") + return + } + } + + /** + * Called when source files change. + */ + fun onSourceChanged() { + if (!initialized) return + } + + /** + * Returns the registered service with the given ID, or null. + */ + fun getService(id: String): IndexingService? = + services[id] + + /** + * Returns all registered services. + */ + fun allServices(): List = + services.values.toList() + + /** + * Shut down all services and clear the registry. + */ + override fun close() { + log.info("Shutting down indexing services") + + // Cancel in-flight work + scope.coroutineContext.cancelChildren() + + // Close services in reverse registration order + services.values.reversed().forEach { service -> + try { + service.close() + log.debug("Closed service: {}", service.id) + } catch (e: Exception) { + log.error("Failed to close service: {}", service.id, e) + } + } + + services.clear() + registry.close() + initialized = false + + log.info("Indexing services shut down") + } + + private suspend fun initializeServices() { + log.info("Initializing {} indexing services", services.size) + + val allServices = allServices() + for (service in allServices) { + try { + service.initialize(registry) + log.info("Initialized service: {} (provides: {})", + service.id, + service.providedKeys.joinToString { it.name }, + ) + } catch (e: Exception) { + log.error("Failed to initialize service: {}", service.id, e) + } + } + + // Verify all promised keys are registered + for (service in allServices) { + for (key in service.providedKeys) { + if (!registry.isRegistered(key)) { + log.warn( + "Service '{}' promised index '{}' but did not register it", + service.id, key.name, + ) + } + } + } + } +} \ No newline at end of file diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt new file mode 100644 index 0000000000..1ab75e4074 --- /dev/null +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt @@ -0,0 +1,214 @@ +package org.appdevforall.codeonthego.indexing.util + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.onCompletion +import kotlinx.coroutines.flow.onStart +import kotlinx.coroutines.isActive +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.slf4j.LoggerFactory +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * Callback for tracking indexing progress. + * Implementations must be thread-safe. + */ +fun interface IndexingProgressListener { + + /** + * Called with progress updates during indexing. + * + * @param sourceId The source being indexed. + * @param event What happened. + */ + fun onProgress(sourceId: String, event: IndexingEvent) +} + +sealed class IndexingEvent { + data object Started : IndexingEvent() + data class Progress(val processed: Int) : IndexingEvent() + data class Completed(val totalIndexed: Int) : IndexingEvent() + data class Failed(val error: Throwable) : IndexingEvent() + data object Skipped : IndexingEvent() +} + +/** + * Runs indexing operations in the background. + */ +class BackgroundIndexer( + private val index: Index, + private val scope: CoroutineScope = CoroutineScope( + SupervisorJob() + Dispatchers.Default + ), + /** + * Buffer capacity between the producer flow and the index writer. + * Higher values use more memory but tolerate more producer/consumer + * speed mismatch. + */ + private val bufferCapacity: Int = 64, +) : Closeable { + + companion object { + private val log = LoggerFactory.getLogger(BackgroundIndexer::class.java) + } + + var progressListener: IndexingProgressListener? = null + + private val activeJobs = ConcurrentHashMap() + + /** + * Index a single source. The [provider] returns a [Flow] that + * lazily produces entries so that it is NOT collected eagerly. + * + * If [skipIfExists] is true and the source is already indexed, + * this is a no-op. + * + * @param sourceId Identifies the source. + * @param skipIfExists Skip if already indexed. + * @param provider Lambda returning a lazy [Flow] of entries. + * Runs on [Dispatchers.IO]. + * @return The launched job, or null if skipped. + */ + fun indexSource( + sourceId: String, + skipIfExists: Boolean = true, + provider: (sourceId: String) -> Flow, + ): Job { + // Cancel any in-flight job for this source + activeJobs[sourceId]?.cancel() + + val job = scope.launch { + try { + if (skipIfExists && index.containsSource(sourceId)) { + log.debug("Skipping already-indexed: {}", sourceId) + progressListener?.onProgress(sourceId, IndexingEvent.Skipped) + return@launch + } + + log.info("Indexing: {}", sourceId) + + // Remove stale entries first + index.removeBySource(sourceId) + + if (!isActive) return@launch + + // Streaming pipeline: + // producer (IO) → buffer → consumer (index.insert) + // + // The producer emits entries lazily on Dispatchers.IO. + // The buffer decouples producer and consumer speeds. + // The index.insert collects from the buffered flow + // and batches into transactions internally. + var count = 0 + + val tracked = provider(sourceId) + .flowOn(Dispatchers.IO) + .buffer(bufferCapacity) + .onStart { + progressListener?.onProgress( + sourceId, IndexingEvent.Started + ) + } + .onCompletion { error -> + if (error == null) { + progressListener?.onProgress( + sourceId, IndexingEvent.Completed(count) + ) + log.info("Indexed {} entries from {}", count, sourceId) + } + } + .catch { error -> + log.error("Indexing failed for {}", sourceId, error) + progressListener?.onProgress( + sourceId, IndexingEvent.Failed(error) + ) + } + + // Wrap in a counting flow that reports progress + val counted = kotlinx.coroutines.flow.flow { + tracked.collect { entry -> + emit(entry) + count++ + if (count % 1000 == 0) { + progressListener?.onProgress( + sourceId, IndexingEvent.Progress(count) + ) + } + } + } + + index.insert(counted) + + } catch (e: CancellationException) { + log.debug("Indexing cancelled: {}", sourceId) + throw e + } catch (e: Exception) { + log.error("Indexing failed: {}", sourceId, e) + progressListener?.onProgress( + sourceId, IndexingEvent.Failed(e) + ) + } finally { + activeJobs.remove(sourceId) + } + } + + activeJobs[sourceId] = job + return job + } + + /** + * Index multiple sources in parallel. + * + * Each source gets its own coroutine. The [SupervisorJob] ensures + * that one failure doesn't cancel the others. + * + * @param sources The sources to index (e.g. a list of JAR paths). + * @param mapper Maps each source to a (sourceId, Flow) pair. + */ + fun indexSources( + sources: Collection, + skipIfExists: Boolean = true, + mapper: (S) -> Pair>, + ): List { + return sources.map { source -> + val (sourceId, flow) = mapper(source) + indexSource(sourceId, skipIfExists) { flow } + }.filterNotNull() + } + + /** + * Cancel all in-flight indexing and wait for completion. + */ + suspend fun cancelAll() { + activeJobs.values.toList().forEach { it.cancelAndJoin() } + } + + /** + * Wait for all in-flight indexing to complete. + */ + suspend fun awaitAll() { + activeJobs.values.toList().joinAll() + } + + /** + * Returns the number of currently active indexing jobs. + */ + val activeJobCount: Int get() = activeJobs.size + + override fun close() { + activeJobs.values.forEach { it.cancel() } + activeJobs.clear() + } +} diff --git a/lsp/java/build.gradle.kts b/lsp/java/build.gradle.kts index 7d43f1aefd..73322f24be 100644 --- a/lsp/java/build.gradle.kts +++ b/lsp/java/build.gradle.kts @@ -55,6 +55,7 @@ dependencies { implementation(projects.editorApi) implementation(projects.resources) implementation(projects.lsp.api) + implementation(projects.lsp.jvmSymbolIndex) implementation(projects.subprojects.libjdwp) implementation(projects.subprojects.javacServices) implementation(projects.idetooltips) diff --git a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt index bf0e04417d..a31a330d7e 100644 --- a/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt +++ b/lsp/java/src/main/java/com/itsaky/androidide/lsp/java/JavaLanguageServer.kt @@ -17,6 +17,7 @@ package com.itsaky.androidide.lsp.java import androidx.annotation.RestrictTo +import com.itsaky.androidide.app.BaseApplication import com.itsaky.androidide.eventbus.events.editor.DocumentChangeEvent import com.itsaky.androidide.eventbus.events.editor.DocumentCloseEvent import com.itsaky.androidide.eventbus.events.editor.DocumentOpenEvent @@ -44,7 +45,7 @@ import com.itsaky.androidide.lsp.java.providers.JavaDiagnosticProvider import com.itsaky.androidide.lsp.java.providers.JavaSelectionProvider import com.itsaky.androidide.lsp.java.providers.ReferenceProvider import com.itsaky.androidide.lsp.java.providers.SignatureProvider -import com.itsaky.androidide.lsp.java.providers.snippet.JavaSnippetRepository.init +import com.itsaky.androidide.lsp.java.providers.snippet.JavaSnippetRepository import com.itsaky.androidide.lsp.java.utils.AnalyzeTimer import com.itsaky.androidide.lsp.java.utils.CancelChecker.Companion.isCancelled import com.itsaky.androidide.lsp.models.CodeFormatResult @@ -74,6 +75,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JvmIndexingService import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode @@ -118,7 +120,12 @@ class JavaLanguageServer : ILanguageServer { EventBus.getDefault().register(this) } - init() + val projectManager = ProjectManagerImpl.getInstance() + projectManager.indexingServiceManager.register( + service = JvmIndexingService(context = BaseApplication.baseInstance) + ) + + JavaSnippetRepository.init() } override fun shutdown() { @@ -152,6 +159,11 @@ class JavaLanguageServer : ILanguageServer { override fun setupWithProject(workspace: Workspace) { LSPEditorActions.ensureActionsMenuRegistered(JavaCodeActionsMenu) + (ProjectManagerImpl.getInstance() + .indexingServiceManager + .getService(JvmIndexingService.ID) as? JvmIndexingService?) + ?.refresh() + // Once we have project initialized // Destory the NO_MODULE_COMPILER instance JavaCompilerService.NO_MODULE_COMPILER.destroy() @@ -249,7 +261,8 @@ class JavaLanguageServer : ILanguageServer { } } - override fun formatCode(params: FormatCodeParams?): CodeFormatResult = CodeFormatProvider(settings).format(params) + override fun formatCode(params: FormatCodeParams?): CodeFormatResult = + CodeFormatProvider(settings).format(params) override fun handleFailure(failure: LSPFailure?): Boolean { return when (failure!!.type) { diff --git a/lsp/jvm-symbol-index/build.gradle.kts b/lsp/jvm-symbol-index/build.gradle.kts new file mode 100644 index 0000000000..959f2264be --- /dev/null +++ b/lsp/jvm-symbol-index/build.gradle.kts @@ -0,0 +1,24 @@ +import com.itsaky.androidide.build.config.BuildConfig + +plugins { + alias(libs.plugins.android.library) + id("kotlin-android") +} + +android { + namespace = "${BuildConfig.PACKAGE_NAME}.lsp.java.indexing" +} + +dependencies { + api(libs.google.protobuf.java) + api(libs.google.protobuf.kotlin) + api(libs.kotlinx.coroutines.core) + api(libs.kotlinx.metadata) + + api(projects.common) + api(projects.logger) + api(projects.lsp.indexing) + api(projects.lsp.jvmSymbolModels) + api(projects.subprojects.kotlinAnalysisApi) + api(projects.subprojects.projects) +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt new file mode 100644 index 0000000000..f0bea84939 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt @@ -0,0 +1,78 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream +import java.nio.file.Path +import java.util.jar.JarFile +import kotlin.io.path.pathString + +/** + * Scans a JAR and routes each class to the appropriate scanner: + * [KotlinMetadataScanner] for Kotlin classes, [JarSymbolScanner] for Java. + */ +object CombinedJarScanner { + + private val log = LoggerFactory.getLogger(CombinedJarScanner::class.java) + + fun scan(jarPath: Path, sourceId: String = jarPath.pathString): Flow = flow { + val jar = try { + JarFile(jarPath.toFile()) + } catch (e: Exception) { + log.warn("Failed to open JAR: {}", jarPath, e) + return@flow + } + + jar.use { + val entries = jar.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + if (!entry.name.endsWith(".class")) continue + if (entry.name == "module-info.class" || entry.name == "package-info.class") continue + + try { + val bytes = jar.getInputStream(entry).use { input -> + val buf = ByteArrayOutputStream(entry.size.toInt().coerceAtLeast(1024)) + input.copyTo(buf) + buf.toByteArray() + } + + val symbols = if (hasKotlinMetadata(bytes)) { + KotlinMetadataScanner.parseKotlinClass(bytes.inputStream(), sourceId) + } else { + JarSymbolScanner.parseClassFile(bytes.inputStream(), sourceId) + } + + symbols?.forEach { emit(it) } + } catch (e: Exception) { + log.debug("Failed to parse {}: {}", entry.name, e.message) + } + } + } + } + .flowOn(Dispatchers.IO) + + private fun hasKotlinMetadata(classBytes: ByteArray): Boolean { + var found = false + try { + ClassReader(classBytes).accept(object : ClassVisitor(Opcodes.ASM9) { + override fun visitAnnotation( + descriptor: String?, + visible: Boolean + ): AnnotationVisitor? { + if (descriptor == "Lkotlin/Metadata;") found = true + return null + } + }, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) + } catch (_: Exception) { + } + return found + } +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt new file mode 100644 index 0000000000..89fd0ab492 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -0,0 +1,305 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.FieldVisitor +import org.jetbrains.org.objectweb.asm.MethodVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.jetbrains.org.objectweb.asm.Type +import org.slf4j.LoggerFactory +import java.io.InputStream +import java.nio.file.Path +import java.util.jar.JarFile +import kotlin.io.path.pathString + +/** + * Scans JAR files using ASM and produces [JvmSymbol]s lazily. + * + * For Java class files, this gives complete information. + * For Kotlin class files, use [KotlinMetadataScanner] or + * [CombinedJarScanner] instead — ASM cannot see Kotlin-specific + * semantics like extensions, suspend, or nullable types. + */ +object JarSymbolScanner { + + private val log = LoggerFactory.getLogger(JarSymbolScanner::class.java) + + fun scan(jarPath: Path, sourceId: String = jarPath.pathString): Flow = flow { + val jar = try { + JarFile(jarPath.toFile()) + } catch (e: Exception) { + log.warn("Failed to open JAR: {}", jarPath, e) + return@flow + } + + jar.use { + val entries = jar.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + if (!entry.name.endsWith(".class")) continue + if (entry.name == "module-info.class") continue + if (entry.name == "package-info.class") continue + + try { + jar.getInputStream(entry).use { input -> + for (symbol in parseClassFile(input, sourceId)) { + emit(symbol) + } + } + } catch (e: Exception) { + log.debug("Failed to parse {}: {}", entry.name, e.message) + } + } + } + } + .flowOn(Dispatchers.IO) + + internal fun parseClassFile(input: InputStream, sourceId: String): List { + val reader = ClassReader(input) + val collector = SymbolCollector(sourceId) + reader.accept(collector, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) + return collector.symbols + } + + private class SymbolCollector( + private val sourceId: String, + ) : ClassVisitor(Opcodes.ASM9) { + + val symbols = mutableListOf() + + private var className = "" + private var classFqName = "" + private var packageName = "" + private var shortClassName = "" + private var classAccess = 0 + private var isKotlinClass = false + private var superName: String? = null + private var interfaces: Array? = null + private var isInnerClass = false + private var classDeprecated = false + + override fun visit( + version: Int, access: Int, name: String, + signature: String?, superName: String?, + interfaces: Array?, + ) { + className = name + classFqName = name.replace('/', '.').replace('$', '.') + classAccess = access + this.superName = superName + this.interfaces = interfaces + classDeprecated = false + + val lastSlash = name.lastIndexOf('/') + packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" + + val afterPackage = if (lastSlash >= 0) name.substring(lastSlash + 1) else name + shortClassName = afterPackage.replace('$', '.') + + isInnerClass = name.contains('$') + } + + override fun visitAnnotation(descriptor: String?, visible: Boolean): AnnotationVisitor? { + if (descriptor == "Ljava/lang/Deprecated;") classDeprecated = true + if (descriptor == "Lkotlin/Metadata;") isKotlinClass = true + return null + } + + override fun visitEnd() { + if (!isPublicOrProtected(classAccess)) return + + val isAnonymous = isInnerClass && + shortClassName.split('.').last().firstOrNull()?.isDigit() == true + if (isAnonymous) return + + val kind = classKindFromAccess(classAccess) + val language = if (isKotlinClass) JvmSourceLanguage.KOTLIN else JvmSourceLanguage.JAVA + + val supertypes = buildList { + superName?.let { + if (it != "java/lang/Object") add(it.replace('/', '.')) + } + interfaces?.forEach { add(it.replace('/', '.')) } + } + + val containingClass = if (isInnerClass) { + classFqName.split('.').dropLast(1).joinToString(".") + } else "" + + symbols.add( + JvmSymbol( + key = classFqName, + sourceId = sourceId, + fqName = classFqName, + shortName = shortClassName.split('.').last(), + packageName = packageName, + kind = kind, + language = language, + visibility = visibilityFromAccess(classAccess), + isDeprecated = classDeprecated, + data = JvmClassInfo( + containingClassFqName = containingClass, + supertypeFqNames = supertypes, + isAbstract = hasFlag(classAccess, Opcodes.ACC_ABSTRACT), + isFinal = hasFlag(classAccess, Opcodes.ACC_FINAL), + isInner = isInnerClass && !hasFlag(classAccess, Opcodes.ACC_STATIC), + isStatic = isInnerClass && hasFlag(classAccess, Opcodes.ACC_STATIC), + ), + ) + ) + } + + override fun visitMethod( + access: Int, name: String, descriptor: String, + signature: String?, exceptions: Array?, + ): MethodVisitor? { + if (!isPublicOrProtected(access)) return null + if (!isPublicOrProtected(classAccess)) return null + if (name.startsWith("access$")) return null + if (hasFlag(access, Opcodes.ACC_BRIDGE)) return null + if (hasFlag(access, Opcodes.ACC_SYNTHETIC)) return null + if (name == "") return null + + val methodType = Type.getMethodType(descriptor) + val paramTypes = methodType.argumentTypes + val returnType = methodType.returnType + + val isConstructor = name == "" + val methodName = if (isConstructor) shortClassName.split('.').last() else name + val kind = if (isConstructor) JvmSymbolKind.CONSTRUCTOR else JvmSymbolKind.FUNCTION + val language = if (isKotlinClass) JvmSourceLanguage.KOTLIN else JvmSourceLanguage.JAVA + + val parameters = paramTypes.map { type -> + JvmParameterInfo( + name = "", // not available without -parameters flag + typeFqName = typeToFqName(type), + typeDisplay = typeToDisplay(type), + ) + } + + val fqName = "$classFqName.$methodName" + val key = "$fqName(${parameters.joinToString(",") { it.typeFqName }})" + + val signatureDisplay = buildString { + append("(") + append(parameters.joinToString(", ") { it.typeDisplay }) + append(")") + if (!isConstructor) { + append(": ") + append(typeToDisplay(returnType)) + } + } + + symbols.add( + JvmSymbol( + key = key, + sourceId = sourceId, + fqName = fqName, + shortName = methodName, + packageName = packageName, + kind = kind, + language = language, + visibility = visibilityFromAccess(access), + isDeprecated = classDeprecated, + data = JvmFunctionInfo( + containingClassFqName = classFqName, + returnTypeFqName = typeToFqName(returnType), + returnTypeDisplay = typeToDisplay(returnType), + parameterCount = paramTypes.size, + parameters = parameters, + signatureDisplay = signatureDisplay, + isStatic = hasFlag(access, Opcodes.ACC_STATIC), + isAbstract = hasFlag(access, Opcodes.ACC_ABSTRACT), + isFinal = hasFlag(access, Opcodes.ACC_FINAL), + ), + ) + ) + + return null + } + + override fun visitField( + access: Int, name: String, descriptor: String, + signature: String?, value: Any?, + ): FieldVisitor? { + if (!isPublicOrProtected(access)) return null + if (!isPublicOrProtected(classAccess)) return null + if (hasFlag(access, Opcodes.ACC_SYNTHETIC)) return null + + val fieldType = Type.getType(descriptor) + val kind = if (isKotlinClass) JvmSymbolKind.PROPERTY else JvmSymbolKind.FIELD + val language = if (isKotlinClass) JvmSourceLanguage.KOTLIN else JvmSourceLanguage.JAVA + val fqName = "$classFqName.$name" + + symbols.add( + JvmSymbol( + key = fqName, + sourceId = sourceId, + fqName = fqName, + shortName = name, + packageName = packageName, + kind = kind, + language = language, + visibility = visibilityFromAccess(access), + isDeprecated = classDeprecated, + data = JvmFieldInfo( + containingClassFqName = classFqName, + typeFqName = typeToFqName(fieldType), + typeDisplay = typeToDisplay(fieldType), + isStatic = hasFlag(access, Opcodes.ACC_STATIC), + isFinal = hasFlag(access, Opcodes.ACC_FINAL), + constantValue = value?.toString() ?: "", + ), + ) + ) + + return null + } + + private fun isPublicOrProtected(access: Int) = + hasFlag(access, Opcodes.ACC_PUBLIC) || hasFlag(access, Opcodes.ACC_PROTECTED) + + private fun hasFlag(access: Int, flag: Int) = (access and flag) != 0 + + private fun classKindFromAccess(access: Int) = when { + hasFlag(access, Opcodes.ACC_ANNOTATION) -> JvmSymbolKind.ANNOTATION_CLASS + hasFlag(access, Opcodes.ACC_ENUM) -> JvmSymbolKind.ENUM + hasFlag(access, Opcodes.ACC_INTERFACE) -> JvmSymbolKind.INTERFACE + else -> JvmSymbolKind.CLASS + } + + private fun visibilityFromAccess(access: Int) = when { + hasFlag(access, Opcodes.ACC_PUBLIC) -> JvmVisibility.PUBLIC + hasFlag(access, Opcodes.ACC_PROTECTED) -> JvmVisibility.PROTECTED + hasFlag(access, Opcodes.ACC_PRIVATE) -> JvmVisibility.PRIVATE + else -> JvmVisibility.PACKAGE_PRIVATE + } + + private fun typeToFqName(type: Type): String = when (type.sort) { + Type.VOID -> "void" + Type.BOOLEAN -> "boolean" + Type.BYTE -> "byte" + Type.CHAR -> "char" + Type.SHORT -> "short" + Type.INT -> "int" + Type.LONG -> "long" + Type.FLOAT -> "float" + Type.DOUBLE -> "double" + Type.ARRAY -> typeToFqName(type.elementType) + "[]".repeat(type.dimensions) + Type.OBJECT -> type.className + else -> type.className + } + + private fun typeToDisplay(type: Type): String = when (type.sort) { + Type.VOID -> "void" + Type.ARRAY -> typeToDisplay(type.elementType) + "[]".repeat(type.dimensions) + Type.OBJECT -> type.className.substringAfterLast('.') + else -> typeToFqName(type) + } + } +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmIndexingService.kt new file mode 100644 index 0000000000..a85c2c271e --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmIndexingService.kt @@ -0,0 +1,146 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import com.itsaky.androidide.projects.ProjectManagerImpl +import com.itsaky.androidide.projects.api.AndroidModule +import com.itsaky.androidide.projects.api.ModuleProject +import com.itsaky.androidide.projects.models.bootClassPaths +import com.itsaky.androidide.tasks.cancelIfActive +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.appdevforall.codeonthego.indexing.service.IndexKey +import org.appdevforall.codeonthego.indexing.service.IndexRegistry +import org.appdevforall.codeonthego.indexing.service.IndexingService +import org.greenrobot.eventbus.Subscribe +import org.greenrobot.eventbus.ThreadMode +import org.slf4j.LoggerFactory +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.extension + +/** + * Well-known key for the JVM library symbol index. + * + * Both the Kotlin and Java LSPs use this key to retrieve the + * shared index from the [IndexRegistry]. + */ +val JVM_LIBRARY_SYMBOL_INDEX = IndexKey("jvm-library-symbols") + +/** + * [IndexingService] that scans classpath JARs/AARs and builds + * a [JvmLibrarySymbolIndex]. + * + * Thread safety: all methods are called from the + * [IndexingServiceManager][org.appdevforall.codeonthego.indexing.service.IndexingServiceManager]'s + * coroutine scope. The [JvmLibrarySymbolIndex] handles its own internal thread safety. + */ +class JvmIndexingService( + private val context: Context, +) : IndexingService { + + companion object { + const val ID = "jvm-indexing-service" + private val log = LoggerFactory.getLogger(JvmIndexingService::class.java) + } + + override val id = ID + + override val providedKeys = listOf(JVM_LIBRARY_SYMBOL_INDEX) + + private var index: JvmLibrarySymbolIndex? = null + private var indexingMutex = Mutex() + private val coroutineScope = CoroutineScope(Dispatchers.Default) + + override suspend fun initialize(registry: IndexRegistry) { + val jvmIndex = JvmLibrarySymbolIndex.create(context) + this.index = jvmIndex + registry.register(JVM_LIBRARY_SYMBOL_INDEX, jvmIndex) + log.info("JVM symbol index initialized") + } + + @Subscribe(threadMode = ThreadMode.ASYNC) + @Suppress("UNUSED") + fun onProjectSynced() { + refresh() + } + + fun refresh() { + coroutineScope.launch { + indexingMutex.withLock { + reindexLibraries() + } + } + } + + private suspend fun reindexLibraries() { + val index = this.index ?: run { + log.warn("Not indexing libraries. Index not initialized.") + return + } + + val workspace = ProjectManagerImpl.getInstance().workspace ?: run { + log.warn("Not indexing libraries. Workspace model not available.") + return + } + + val currentJars = + workspace.subProjects + .asSequence() + .filterIsInstance() + .filter { it.path != workspace.rootProject.path } + .flatMap { project -> + buildList { + if (project is AndroidModule) { + addAll(project.bootClassPaths) + } + + addAll(project.getCompileClasspaths(excludeSourceGeneratedClassPath = true)) + } + } + .filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } + .map { jar -> jar.absolutePath } + .toSet() + + log.info("{} JARs on classpath", currentJars.size) + + // Step 1: Set the active set - this is instant. + // JARs not in the set become invisible to queries. + // JARs in the set that are already cached become + // visible immediately. + index.setActiveLibraries(currentJars) + + // Step 2: Index any JARs not yet in the cache. + // Already-cached JARs are skipped (cheap existence check). + // Newly cached JARs are automatically visible because + // they're already in the active set. + var newCount = 0 + for (jarPath in currentJars) { + if (!index.isLibraryCached(jarPath)) { + newCount++ + index.indexLibrary(jarPath) { sourceId -> + CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + } + } + } + + if (newCount > 0) { + log.info("{} new JARs submitted for background indexing", newCount) + } else { + log.info("All JARs already cached, nothing to index") + } + } + + override fun close() { + coroutineScope.cancelIfActive("indexing service closed") + index?.close() + index = null + } + + private fun isIndexableJar(path: Path): Boolean { + val ext = path.extension.lowercase() + return ext == "jar" || ext == "aar" + } +} \ No newline at end of file diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibrarySymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibrarySymbolIndex.kt new file mode 100644 index 0000000000..ec52e5d633 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibrarySymbolIndex.kt @@ -0,0 +1,166 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import android.content.Context +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.take +import org.appdevforall.codeonthego.indexing.FilteredIndex +import org.appdevforall.codeonthego.indexing.PersistentIndex +import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_CONTAINING_CLASS +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_NAME +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_PACKAGE +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_RECEIVER_TYPE +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import java.io.Closeable + +/** + * An index of symbols from external Java libraries (JARs). + */ +class JvmLibrarySymbolIndex private constructor( + /** Persistent cache — stores every JAR ever indexed. */ + val libraryCache: PersistentIndex, + + /** Filtered view — only shows JARs on the current classpath. */ + val libraryView: FilteredIndex, + + /** Background indexer writing to the cache. */ + val libraryIndexer: BackgroundIndexer, +) : Closeable { + + companion object { + + const val DB_NAME_DEFAULT = "jvm_symbol_index.db" + const val INDEX_NAME_LIBRARY = "jvm-library-cache" + + fun create( + context: Context, + dbName: String = DB_NAME_DEFAULT, + ): JvmLibrarySymbolIndex { + val cache = PersistentIndex( + descriptor = JvmSymbolDescriptor, + context = context, + dbName = dbName, + name = INDEX_NAME_LIBRARY, + ) + + val view = FilteredIndex(cache) + + val indexer = BackgroundIndexer(cache) + return JvmLibrarySymbolIndex( + libraryCache = cache, + libraryView = view, + libraryIndexer = indexer + ) + } + } + + /** + * Make a library visible in query results. + * + * If the library is already cached (indexed previously), + * this is instant. If not, call [indexLibrary] first. + */ + fun activateLibrary(sourceId: String) { + libraryView.activateSource(sourceId) + } + + /** + * Hide a library from query results. + * The cached index data is retained for future reuse. + */ + fun deactivateLibrary(sourceId: String) { + libraryView.deactivateSource(sourceId) + } + + /** + * Replace the entire active library set. + * + * Typical call after project sync: pass all current classpath + * JAR paths. Libraries not in the set become invisible. + * Libraries in the set that are already cached become + * instantly visible. + */ + fun setActiveLibraries(sourceIds: Set) { + libraryView.setActiveSources(sourceIds) + } + + /** + * Check if a library is already cached (regardless of whether + * it's currently active). + */ + suspend fun isLibraryCached(sourceId: String): Boolean = + libraryView.isCached(sourceId) + + /** + * Index a library JAR/AAR into the persistent cache. + * + * This does NOT make the library visible in queries — + * call [activateLibrary] after indexing completes. + * + * Skips if already cached. Call [reindexLibrary] to force. + */ + fun indexLibrary( + sourceId: String, + provider: (sourceId: String) -> Flow, + ) = libraryIndexer.indexSource(sourceId, skipIfExists = true, provider) + + fun reindexLibrary( + sourceId: String, + provider: (sourceId: String) -> Flow, + ) = libraryIndexer.indexSource(sourceId, skipIfExists = false, provider) + + fun findByPrefix(prefix: String, limit: Int = 200): Flow = + libraryView.query(indexQuery { prefix(KEY_NAME, prefix); this.limit = limit }) + + fun findByPrefix( + prefix: String, kinds: Set, limit: Int = 200, + ): Flow = + libraryView.query(indexQuery { prefix(KEY_NAME, prefix); this.limit = 0 }) + .filter { it.kind in kinds } + .take(limit) + + fun findExtensionsFor( + receiverTypeFqName: String, namePrefix: String = "", limit: Int = 200, + ): Flow = libraryView.query(indexQuery { + eq(KEY_RECEIVER_TYPE, receiverTypeFqName) + if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) + this.limit = limit + }) + + fun findTopLevelCallablesInPackage( + packageName: String, namePrefix: String = "", limit: Int = 200, + ): Flow = libraryView.query(indexQuery { + eq(KEY_PACKAGE, packageName) + if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) + this.limit = 0 + }).filter { it.kind.isCallable && it.isTopLevel }.take(limit) + + fun findClassifiersInPackage( + packageName: String, namePrefix: String = "", limit: Int = 200, + ): Flow = libraryView.query(indexQuery { + eq(KEY_PACKAGE, packageName) + if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) + this.limit = 0 + }).filter { it.kind.isClassifier }.take(limit) + + fun findMembersOf( + classFqName: String, namePrefix: String = "", limit: Int = 200, + ): Flow = libraryView.query(indexQuery { + eq(KEY_CONTAINING_CLASS, classFqName) + if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) + this.limit = limit + }) + + suspend fun findByFqName(fqName: String): JvmSymbol? = libraryView.get(fqName) + + fun allPackages(): Flow = libraryView.distinctValues(KEY_PACKAGE) + + suspend fun awaitLibraryIndexing() = libraryIndexer.awaitAll() + + override fun close() { + libraryCache.close() + libraryIndexer.close() + libraryView.close() + } +} \ No newline at end of file diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt new file mode 100644 index 0000000000..fdbd2c20bd --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -0,0 +1,204 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import org.appdevforall.codeonthego.indexing.api.Indexable + +enum class JvmSymbolKind { + CLASS, INTERFACE, ENUM, ENUM_ENTRY, ANNOTATION_CLASS, + OBJECT, COMPANION_OBJECT, DATA_CLASS, VALUE_CLASS, + SEALED_CLASS, SEALED_INTERFACE, + FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, + PROPERTY, EXTENSION_PROPERTY, FIELD, + TYPE_ALIAS; + + val isCallable: Boolean + get() = this in CALLABLE_KINDS + + val isClassifier: Boolean + get() = this in CLASSIFIER_KINDS + + val isExtension: Boolean + get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY + + companion object { + val CALLABLE_KINDS = setOf( + FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, + PROPERTY, EXTENSION_PROPERTY, FIELD, + ) + val CLASSIFIER_KINDS = setOf( + CLASS, INTERFACE, ENUM, ANNOTATION_CLASS, + OBJECT, COMPANION_OBJECT, DATA_CLASS, + VALUE_CLASS, SEALED_CLASS, SEALED_INTERFACE, + TYPE_ALIAS, + ) + } +} + +enum class JvmSourceLanguage { JAVA, KOTLIN } + +enum class JvmVisibility { + PUBLIC, PROTECTED, INTERNAL, PRIVATE, PACKAGE_PRIVATE; + + val isAccessibleOutsideClass: Boolean + get() = this == PUBLIC || this == PROTECTED || this == INTERNAL +} + +/** + * A symbol from a JVM class file (JAR/AAR). + * + * Common identity fields live here. Type-specific details live in + * [data], which is one of: + * - [JvmClassInfo] for classes, interfaces, enums, objects, etc. + * - [JvmFunctionInfo] for functions, extension functions, constructors + * - [JvmFieldInfo] for Java fields and Kotlin properties + * - [JvmEnumEntryInfo] for enum constants + * - [JvmTypeAliasInfo] for Kotlin type aliases + */ +data class JvmSymbol( + override val key: String, + override val sourceId: String, + + val fqName: String, + val shortName: String, + val packageName: String, + val kind: JvmSymbolKind, + val language: JvmSourceLanguage, + val visibility: JvmVisibility = JvmVisibility.PUBLIC, + val isDeprecated: Boolean = false, + + val data: JvmSymbolInfo, +) : Indexable { + + val isTopLevel: Boolean + get() = data.containingClassFqName.isEmpty() + + val isExtension: Boolean + get() = kind.isExtension + + val receiverTypeFqName: String? + get() = when (val d = data) { + is JvmFunctionInfo -> d.kotlin?.receiverTypeFqName?.takeIf { it.isNotEmpty() } + is JvmFieldInfo -> d.kotlin?.receiverTypeFqName?.takeIf { it.isNotEmpty() } + else -> null + } + + val containingClassFqName: String + get() = data.containingClassFqName + + val returnTypeDisplay: String + get() = when (val d = data) { + is JvmFunctionInfo -> d.returnTypeDisplay + is JvmFieldInfo -> d.typeDisplay + else -> "" + } + + val signatureDisplay: String + get() = when (val d = data) { + is JvmFunctionInfo -> d.signatureDisplay + else -> "" + } +} + +/** + * Base for all type-specific symbol data. + * Every variant provides [containingClassFqName] (empty for top-level). + */ +sealed interface JvmSymbolInfo { + val containingClassFqName: String +} + +data class JvmClassInfo( + override val containingClassFqName: String = "", + val supertypeFqNames: List = emptyList(), + val typeParameters: List = emptyList(), + val isAbstract: Boolean = false, + val isFinal: Boolean = false, + val isInner: Boolean = false, + val isStatic: Boolean = false, + val kotlin: KotlinClassInfo? = null, +) : JvmSymbolInfo + +data class KotlinClassInfo( + val isData: Boolean = false, + val isValue: Boolean = false, + val isSealed: Boolean = false, + val isFunInterface: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val sealedSubclasses: List = emptyList(), + val companionObjectName: String = "", +) + +data class JvmFunctionInfo( + override val containingClassFqName: String = "", + val returnTypeFqName: String = "", + val returnTypeDisplay: String = "", + val parameterCount: Int = 0, + val parameters: List = emptyList(), + val signatureDisplay: String = "", + val typeParameters: List = emptyList(), + val isStatic: Boolean = false, + val isAbstract: Boolean = false, + val isFinal: Boolean = false, + val kotlin: KotlinFunctionInfo? = null, +) : JvmSymbolInfo + +data class JvmParameterInfo( + val name: String, + val typeFqName: String, + val typeDisplay: String, + val hasDefaultValue: Boolean = false, + val isCrossinline: Boolean = false, + val isNoinline: Boolean = false, + val isVararg: Boolean = false, +) + +data class KotlinFunctionInfo( + val receiverTypeFqName: String = "", + val receiverTypeDisplay: String = "", + val isSuspend: Boolean = false, + val isInline: Boolean = false, + val isInfix: Boolean = false, + val isOperator: Boolean = false, + val isTailrec: Boolean = false, + val isExternal: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isReturnTypeNullable: Boolean = false, +) + +data class JvmFieldInfo( + override val containingClassFqName: String = "", + val typeFqName: String = "", + val typeDisplay: String = "", + val isStatic: Boolean = false, + val isFinal: Boolean = false, + val constantValue: String = "", + val kotlin: KotlinPropertyInfo? = null, +) : JvmSymbolInfo + +data class KotlinPropertyInfo( + val receiverTypeFqName: String = "", + val receiverTypeDisplay: String = "", + val isConst: Boolean = false, + val isLateinit: Boolean = false, + val hasGetter: Boolean = false, + val hasSetter: Boolean = false, + val isDelegated: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val isTypeNullable: Boolean = false, +) + +data class JvmEnumEntryInfo( + override val containingClassFqName: String = "", + val ordinal: Int = 0, +) : JvmSymbolInfo + +data class JvmTypeAliasInfo( + override val containingClassFqName: String = "", + val expandedTypeFqName: String = "", + val expandedTypeDisplay: String = "", + val typeParameters: List = emptyList(), +) : JvmSymbolInfo diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt new file mode 100644 index 0000000000..4d34d1b55d --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -0,0 +1,416 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.jvm.proto.JvmSymbolProtos +import org.appdevforall.codeonthego.indexing.jvm.proto.JvmSymbolProtos.JvmSymbolData + +/** + * [IndexDescriptor] for [JvmSymbol]. + * + * Queryable fields: + * - `name` : prefix-searchable, for completion + * - `package` : exact, for package-scoped queries + * - `kind` : exact, for filtering by CLASS/FUNCTION/etc. + * - `receiverType` : exact, for extension function matching + * - `containingClass`: exact, for member lookup + * - `language` : exact, for Java-only or Kotlin-only queries + * + * Blob serialization uses Protobuf with `oneof` for type-specific data. + */ +object JvmSymbolDescriptor : IndexDescriptor { + + const val KEY_NAME = "name" + const val KEY_PACKAGE = "package" + const val KEY_KIND = "kind" + const val KEY_RECEIVER_TYPE = "receiverType" + const val KEY_CONTAINING_CLASS = "containingClass" + const val KEY_LANGUAGE = "language" + + override val name: String = "jvm_symbols" + + override val fields: List = listOf( + IndexField(name = KEY_NAME, prefixSearchable = true), + IndexField(name = KEY_PACKAGE), + IndexField(name = KEY_KIND), + IndexField(name = KEY_RECEIVER_TYPE), + IndexField(name = KEY_CONTAINING_CLASS), + IndexField(name = KEY_LANGUAGE), + ) + + override fun fieldValues(entry: JvmSymbol): Map = mapOf( + KEY_NAME to entry.shortName, + KEY_PACKAGE to entry.packageName, + KEY_KIND to entry.kind.name, + KEY_RECEIVER_TYPE to entry.receiverTypeFqName, + KEY_CONTAINING_CLASS to entry.containingClassFqName.ifEmpty { null }, + KEY_LANGUAGE to entry.language.name, + ) + + override fun serialize(entry: JvmSymbol): ByteArray = + toProto(entry).toByteArray() + + override fun deserialize(bytes: ByteArray): JvmSymbol = + fromProto(JvmSymbolData.parseFrom(bytes)) + + private fun toProto(s: JvmSymbol): JvmSymbolData { + val builder = JvmSymbolData.newBuilder() + .setFqName(s.fqName) + .setShortName(s.shortName) + .setPackageName(s.packageName) + .setSourceId(s.sourceId) + .setKind(kindToProto(s.kind)) + .setLanguage(languageToProto(s.language)) + .setVisibility(visibilityToProto(s.visibility)) + .setIsDeprecated(s.isDeprecated) + + when (val data = s.data) { + is JvmClassInfo -> builder.setClassData(classInfoToProto(data)) + is JvmFunctionInfo -> builder.setFunctionData(functionInfoToProto(data)) + is JvmFieldInfo -> builder.setFieldData(fieldInfoToProto(data)) + is JvmEnumEntryInfo -> builder.setEnumEntryData(enumEntryToProto(data)) + is JvmTypeAliasInfo -> builder.setTypeAliasData(typeAliasToProto(data)) + } + + return builder.build() + } + + private fun classInfoToProto(d: JvmClassInfo): JvmSymbolProtos.ClassData { + val builder = JvmSymbolProtos.ClassData.newBuilder() + .setContainingClassFqName(d.containingClassFqName) + .addAllSupertypeFqNames(d.supertypeFqNames) + .addAllTypeParameters(d.typeParameters) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + .setIsInner(d.isInner) + .setIsStatic(d.isStatic) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinClassData.newBuilder() + .setIsData(kd.isData) + .setIsValue(kd.isValue) + .setIsSealed(kd.isSealed) + .setIsFunInterface(kd.isFunInterface) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .addAllSealedSubclasses(kd.sealedSubclasses) + .setCompanionObjectName(kd.companionObjectName) + ) + } + + return builder.build() + } + + private fun functionInfoToProto(d: JvmFunctionInfo): JvmSymbolProtos.FunctionData { + val builder = JvmSymbolProtos.FunctionData.newBuilder() + .setContainingClassFqName(d.containingClassFqName) + .setReturnTypeFqName(d.returnTypeFqName) + .setReturnTypeDisplay(d.returnTypeDisplay) + .setParameterCount(d.parameterCount) + .addAllParameters(d.parameters.map { paramToProto(it) }) + .setSignatureDisplay(d.signatureDisplay) + .addAllTypeParameters(d.typeParameters) + .setIsStatic(d.isStatic) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinFunctionData.newBuilder() + .setReceiverTypeFqName(kd.receiverTypeFqName) + .setReceiverTypeDisplay(kd.receiverTypeDisplay) + .setIsSuspend(kd.isSuspend) + .setIsInline(kd.isInline) + .setIsInfix(kd.isInfix) + .setIsOperator(kd.isOperator) + .setIsTailrec(kd.isTailrec) + .setIsExternal(kd.isExternal) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsReturnTypeNullable(kd.isReturnTypeNullable) + ) + } + + return builder.build() + } + + private fun paramToProto(p: JvmParameterInfo): JvmSymbolProtos.ParameterData = + JvmSymbolProtos.ParameterData.newBuilder() + .setName(p.name) + .setTypeFqName(p.typeFqName) + .setTypeDisplay(p.typeDisplay) + .setHasDefaultValue(p.hasDefaultValue) + .setIsCrossinline(p.isCrossinline) + .setIsNoinline(p.isNoinline) + .setIsVararg(p.isVararg) + .build() + + private fun fieldInfoToProto(d: JvmFieldInfo): JvmSymbolProtos.FieldData { + val builder = JvmSymbolProtos.FieldData.newBuilder() + .setContainingClassFqName(d.containingClassFqName) + .setTypeFqName(d.typeFqName) + .setTypeDisplay(d.typeDisplay) + .setIsStatic(d.isStatic) + .setIsFinal(d.isFinal) + .setConstantValue(d.constantValue) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinPropertyData.newBuilder() + .setReceiverTypeFqName(kd.receiverTypeFqName) + .setReceiverTypeDisplay(kd.receiverTypeDisplay) + .setIsConst(kd.isConst) + .setIsLateinit(kd.isLateinit) + .setHasGetter(kd.hasGetter) + .setHasSetter(kd.hasSetter) + .setIsDelegated(kd.isDelegated) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .setIsTypeNullable(kd.isTypeNullable) + ) + } + + return builder.build() + } + + private fun enumEntryToProto(d: JvmEnumEntryInfo): JvmSymbolProtos.EnumEntryData = + JvmSymbolProtos.EnumEntryData.newBuilder() + .setContainingEnumFqName(d.containingClassFqName) + .setOrdinal(d.ordinal) + .build() + + private fun typeAliasToProto(d: JvmTypeAliasInfo): JvmSymbolProtos.TypeAliasData = + JvmSymbolProtos.TypeAliasData.newBuilder() + .setExpandedTypeFqName(d.expandedTypeFqName) + .setExpandedTypeDisplay(d.expandedTypeDisplay) + .addAllTypeParameters(d.typeParameters) + .build() + + private fun fromProto(p: JvmSymbolData): JvmSymbol { + val kind = kindFromProto(p.kind) + val data = dataFromProto(p) + + val key = when { + kind.isCallable && kind != JvmSymbolKind.PROPERTY + && kind != JvmSymbolKind.EXTENSION_PROPERTY + && kind != JvmSymbolKind.FIELD -> { + val params = (data as? JvmFunctionInfo) + ?.parameters + ?.joinToString(",") { it.typeFqName } + ?: "" + "${p.fqName}($params)" + } + else -> p.fqName + } + + return JvmSymbol( + key = key, + sourceId = p.sourceId, + fqName = p.fqName, + shortName = p.shortName, + packageName = p.packageName, + kind = kind, + language = languageFromProto(p.language), + visibility = visibilityFromProto(p.visibility), + isDeprecated = p.isDeprecated, + data = data, + ) + } + + private fun dataFromProto(p: JvmSymbolData): JvmSymbolInfo = when (p.dataCase) { + JvmSymbolData.DataCase.CLASS_DATA -> classInfoFromProto(p.classData) + JvmSymbolData.DataCase.FUNCTION_DATA -> functionInfoFromProto(p.functionData) + JvmSymbolData.DataCase.FIELD_DATA -> fieldInfoFromProto(p.fieldData) + JvmSymbolData.DataCase.ENUM_ENTRY_DATA -> enumEntryFromProto(p.enumEntryData) + JvmSymbolData.DataCase.TYPE_ALIAS_DATA -> typeAliasFromProto(p.typeAliasData) + else -> JvmClassInfo() // fallback + } + + private fun classInfoFromProto(p: JvmSymbolProtos.ClassData): JvmClassInfo { + val kotlin = if (p.hasKotlin()) { + val kd = p.kotlin + KotlinClassInfo( + isData = kd.isData, + isValue = kd.isValue, + isSealed = kd.isSealed, + isFunInterface = kd.isFunInterface, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + sealedSubclasses = kd.sealedSubclassesList.toList(), + companionObjectName = kd.companionObjectName, + ) + } else null + + return JvmClassInfo( + containingClassFqName = p.containingClassFqName, + supertypeFqNames = p.supertypeFqNamesList.toList(), + typeParameters = p.typeParametersList.toList(), + isAbstract = p.isAbstract, + isFinal = p.isFinal, + isInner = p.isInner, + isStatic = p.isStatic, + kotlin = kotlin, + ) + } + + private fun functionInfoFromProto(p: JvmSymbolProtos.FunctionData): JvmFunctionInfo { + val kotlin = if (p.hasKotlin()) { + val kd = p.kotlin + KotlinFunctionInfo( + receiverTypeFqName = kd.receiverTypeFqName, + receiverTypeDisplay = kd.receiverTypeDisplay, + isSuspend = kd.isSuspend, + isInline = kd.isInline, + isInfix = kd.isInfix, + isOperator = kd.isOperator, + isTailrec = kd.isTailrec, + isExternal = kd.isExternal, + isExpect = kd.isExpect, + isActual = kd.isActual, + isReturnTypeNullable = kd.isReturnTypeNullable, + ) + } else null + + return JvmFunctionInfo( + containingClassFqName = p.containingClassFqName, + returnTypeFqName = p.returnTypeFqName, + returnTypeDisplay = p.returnTypeDisplay, + parameterCount = p.parameterCount, + parameters = p.parametersList.map { paramFromProto(it) }, + signatureDisplay = p.signatureDisplay, + typeParameters = p.typeParametersList.toList(), + isStatic = p.isStatic, + isAbstract = p.isAbstract, + isFinal = p.isFinal, + kotlin = kotlin, + ) + } + + private fun paramFromProto(p: JvmSymbolProtos.ParameterData): JvmParameterInfo = + JvmParameterInfo( + name = p.name, + typeFqName = p.typeFqName, + typeDisplay = p.typeDisplay, + hasDefaultValue = p.hasDefaultValue, + isCrossinline = p.isCrossinline, + isNoinline = p.isNoinline, + isVararg = p.isVararg, + ) + + private fun fieldInfoFromProto(p: JvmSymbolProtos.FieldData): JvmFieldInfo { + val kotlin = if (p.hasKotlin()) { + val kd = p.kotlin + KotlinPropertyInfo( + receiverTypeFqName = kd.receiverTypeFqName, + receiverTypeDisplay = kd.receiverTypeDisplay, + isConst = kd.isConst, + isLateinit = kd.isLateinit, + hasGetter = kd.hasGetter, + hasSetter = kd.hasSetter, + isDelegated = kd.isDelegated, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + isTypeNullable = kd.isTypeNullable, + ) + } else null + + return JvmFieldInfo( + containingClassFqName = p.containingClassFqName, + typeFqName = p.typeFqName, + typeDisplay = p.typeDisplay, + isStatic = p.isStatic, + isFinal = p.isFinal, + constantValue = p.constantValue, + kotlin = kotlin, + ) + } + + private fun enumEntryFromProto(p: JvmSymbolProtos.EnumEntryData): JvmEnumEntryInfo = + JvmEnumEntryInfo( + containingClassFqName = p.containingEnumFqName, + ordinal = p.ordinal, + ) + + private fun typeAliasFromProto(p: JvmSymbolProtos.TypeAliasData): JvmTypeAliasInfo = + JvmTypeAliasInfo( + expandedTypeFqName = p.expandedTypeFqName, + expandedTypeDisplay = p.expandedTypeDisplay, + typeParameters = p.typeParametersList.toList(), + ) + + private fun kindToProto(k: JvmSymbolKind) = when (k) { + JvmSymbolKind.CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_CLASS + JvmSymbolKind.INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE + JvmSymbolKind.ENUM -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM + JvmSymbolKind.ENUM_ENTRY -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY + JvmSymbolKind.ANNOTATION_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS + JvmSymbolKind.OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT + JvmSymbolKind.COMPANION_OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT + JvmSymbolKind.DATA_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS + JvmSymbolKind.VALUE_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS + JvmSymbolKind.SEALED_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS + JvmSymbolKind.SEALED_INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE + JvmSymbolKind.FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION + JvmSymbolKind.EXTENSION_FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION + JvmSymbolKind.CONSTRUCTOR -> JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR + JvmSymbolKind.PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY + JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY + JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD + JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS + } + + private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = when (k) { + JvmSymbolProtos.JvmSymbolKind.KIND_CLASS -> JvmSymbolKind.CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE -> JvmSymbolKind.INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM -> JvmSymbolKind.ENUM + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY -> JvmSymbolKind.ENUM_ENTRY + JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT -> JvmSymbolKind.OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS -> JvmSymbolKind.DATA_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS -> JvmSymbolKind.VALUE_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS -> JvmSymbolKind.SEALED_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE -> JvmSymbolKind.SEALED_INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION -> JvmSymbolKind.FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION -> JvmSymbolKind.EXTENSION_FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR -> JvmSymbolKind.CONSTRUCTOR + JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY -> JvmSymbolKind.PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD + JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS + else -> JvmSymbolKind.CLASS + } + + private fun languageToProto(l: JvmSourceLanguage) = when (l) { + JvmSourceLanguage.JAVA -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA + JvmSourceLanguage.KOTLIN -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN + } + + private fun languageFromProto(l: JvmSymbolProtos.JvmSourceLanguage) = when (l) { + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA -> JvmSourceLanguage.JAVA + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN -> JvmSourceLanguage.KOTLIN + else -> JvmSourceLanguage.JAVA + } + + private fun visibilityToProto(v: JvmVisibility) = when (v) { + JvmVisibility.PUBLIC -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC + JvmVisibility.PROTECTED -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED + JvmVisibility.INTERNAL -> JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL + JvmVisibility.PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE + JvmVisibility.PACKAGE_PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE + } + + private fun visibilityFromProto(v: JvmSymbolProtos.JvmVisibility) = when (v) { + JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC -> JvmVisibility.PUBLIC + JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED -> JvmVisibility.PROTECTED + JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL -> JvmVisibility.INTERNAL + JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE -> JvmVisibility.PRIVATE + JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE -> JvmVisibility.PACKAGE_PRIVATE + else -> JvmVisibility.PUBLIC + } +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt new file mode 100644 index 0000000000..691d50dd25 --- /dev/null +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -0,0 +1,433 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.slf4j.LoggerFactory +import java.io.InputStream +import java.nio.file.Path +import java.util.jar.JarFile +import kotlin.io.path.pathString +import kotlin.metadata.ClassKind +import kotlin.metadata.KmClass +import kotlin.metadata.KmClassifier +import kotlin.metadata.KmFunction +import kotlin.metadata.KmPackage +import kotlin.metadata.KmProperty +import kotlin.metadata.KmType +import kotlin.metadata.Modality +import kotlin.metadata.Visibility +import kotlin.metadata.declaresDefaultValue +import kotlin.metadata.isConst +import kotlin.metadata.isDelegated +import kotlin.metadata.isExpect +import kotlin.metadata.isExternal +import kotlin.metadata.isInfix +import kotlin.metadata.isInline +import kotlin.metadata.isLateinit +import kotlin.metadata.isNullable +import kotlin.metadata.isOperator +import kotlin.metadata.isSuspend +import kotlin.metadata.isTailrec +import kotlin.metadata.jvm.KotlinClassMetadata +import kotlin.metadata.jvm.Metadata +import kotlin.metadata.kind +import kotlin.metadata.modality +import kotlin.metadata.visibility + +/** + * Scans JAR files using Kotlin metadata to produce [JvmSymbol]s + * with full Kotlin semantics (extensions, suspend, inline, etc.). + * + * Skips non-Kotlin class files (no `@Metadata` annotation). + */ +object KotlinMetadataScanner { + + private val log = LoggerFactory.getLogger(KotlinMetadataScanner::class.java) + + fun scan(jarPath: Path, sourceId: String = jarPath.pathString): Flow = flow { + val jar = try { + JarFile(jarPath.toFile()) + } catch (e: Exception) { + log.warn("Failed to open JAR: {}", jarPath, e) + return@flow + } + + jar.use { + val entries = jar.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + if (!entry.name.endsWith(".class")) continue + if (entry.name == "module-info.class") continue + + try { + jar.getInputStream(entry).use { input -> + parseKotlinClass(input, sourceId)?.forEach { emit(it) } + } + } catch (e: Exception) { + log.debug("Failed to parse {}: {}", entry.name, e.message) + } + } + } + } + .flowOn(Dispatchers.IO) + + internal fun parseKotlinClass(input: InputStream, sourceId: String): List? { + val reader = ClassReader(input) + val collector = MetadataCollector() + reader.accept( + collector, + ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES + ) + + val header = collector.metadataHeader ?: return null + + val metadata = try { + KotlinClassMetadata.readStrict(header) + } catch (e: Exception) { + log.debug("Failed to read Kotlin metadata: {}", e.message) + return null + } + + return when (metadata) { + is KotlinClassMetadata.Class -> + extractFromClass(metadata.kmClass, sourceId) + + is KotlinClassMetadata.FileFacade -> + extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) + + is KotlinClassMetadata.MultiFileClassPart -> + extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) + + else -> null + } + } + + private fun extractFromClass( + klass: KmClass, sourceId: String, + ): List { + val symbols = mutableListOf() + val classFqName = klass.name.replace('/', '.') + val packageName = classFqName.substringBeforeLast('.', "") + val shortName = classFqName.substringAfterLast('.') + + val kind = when (klass.kind) { + ClassKind.INTERFACE -> JvmSymbolKind.INTERFACE + ClassKind.ENUM_CLASS -> JvmSymbolKind.ENUM + ClassKind.ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS + ClassKind.OBJECT -> JvmSymbolKind.OBJECT + ClassKind.COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT + ClassKind.CLASS -> JvmSymbolKind.CLASS + else -> JvmSymbolKind.CLASS + } + + val supertypes = klass.supertypes.mapNotNull { supertype -> + when (val c = supertype.classifier) { + is KmClassifier.Class -> c.name.replace('/', '.') + else -> null + } + } + + symbols.add( + JvmSymbol( + key = classFqName, + sourceId = sourceId, + fqName = classFqName, + shortName = shortName, + packageName = packageName, + kind = kind, + language = JvmSourceLanguage.KOTLIN, + visibility = kmVisibility(klass.visibility), + data = JvmClassInfo( + supertypeFqNames = supertypes, + typeParameters = klass.typeParameters.map { it.name }, + isAbstract = klass.modality == Modality.ABSTRACT, + isFinal = klass.modality == Modality.FINAL, + kotlin = KotlinClassInfo( + isSealed = klass.modality == Modality.SEALED, + sealedSubclasses = klass.sealedSubclasses.map { it.replace('/', '.') }, + ), + ), + ) + ) + + for (fn in klass.functions) { + extractFunction(fn, classFqName, packageName, sourceId)?.let { symbols.add(it) } + } + + for (prop in klass.properties) { + extractProperty(prop, classFqName, packageName, sourceId)?.let { symbols.add(it) } + } + + if (kind == JvmSymbolKind.ENUM) { + klass.kmEnumEntries.forEachIndexed { ordinal, entry -> + symbols.add( + JvmSymbol( + key = "$classFqName.$entry", + sourceId = sourceId, + fqName = "$classFqName.$entry", + shortName = entry.name, + packageName = packageName, + kind = JvmSymbolKind.ENUM_ENTRY, + language = JvmSourceLanguage.KOTLIN, + data = JvmEnumEntryInfo( + containingClassFqName = classFqName, + ordinal = ordinal, + ), + ) + ) + } + } + + return symbols + } + + private fun extractFromPackage( + pkg: KmPackage, + packageName: String, + sourceId: String, + ): List { + val symbols = mutableListOf() + + for (fn in pkg.functions) { + extractFunction(fn, "", packageName, sourceId)?.let { symbols.add(it) } + } + + for (prop in pkg.properties) { + extractProperty(prop, "", packageName, sourceId)?.let { symbols.add(it) } + } + + for (alias in pkg.typeAliases) { + val fqName = if (packageName.isEmpty()) alias.name else "$packageName.${alias.name}" + symbols.add( + JvmSymbol( + key = fqName, + sourceId = sourceId, + fqName = fqName, + shortName = alias.name, + packageName = packageName, + kind = JvmSymbolKind.TYPE_ALIAS, + language = JvmSourceLanguage.KOTLIN, + visibility = kmVisibility(alias.visibility), + data = JvmTypeAliasInfo( + expandedTypeFqName = kmTypeToFqName(alias.expandedType), + expandedTypeDisplay = kmTypeToDisplay(alias.expandedType), + typeParameters = alias.typeParameters.map { it.name }, + ), + ) + ) + } + + return symbols + } + + private fun extractFunction( + fn: KmFunction, + containingClass: String, + packageName: String, + sourceId: String, + ): JvmSymbol? { + val vis = kmVisibility(fn.visibility) + if (vis == JvmVisibility.PRIVATE) return null + + val receiverType = fn.receiverParameterType + val isExtension = receiverType != null + val kind = if (isExtension) JvmSymbolKind.EXTENSION_FUNCTION else JvmSymbolKind.FUNCTION + + val parameters = fn.valueParameters.map { param -> + JvmParameterInfo( + name = param.name, + typeFqName = kmTypeToFqName(param.type), + typeDisplay = kmTypeToDisplay(param.type), + hasDefaultValue = param.declaresDefaultValue, + isVararg = param.varargElementType != null, + ) + } + + val baseFqName = if (containingClass.isNotEmpty()) + "$containingClass.${fn.name}" else "$packageName.${fn.name}" + val key = "$baseFqName(${parameters.joinToString(",") { it.typeFqName }})" + + val signatureDisplay = buildString { + append("(") + append(parameters.joinToString(", ") { "${it.name}: ${it.typeDisplay}" }) + append("): ") + append(kmTypeToDisplay(fn.returnType)) + } + + return JvmSymbol( + key = key, + sourceId = sourceId, + fqName = baseFqName, + shortName = fn.name, + packageName = packageName, + kind = kind, + language = JvmSourceLanguage.KOTLIN, + visibility = vis, + data = JvmFunctionInfo( + containingClassFqName = containingClass, + returnTypeFqName = kmTypeToFqName(fn.returnType), + returnTypeDisplay = kmTypeToDisplay(fn.returnType), + parameterCount = parameters.size, + parameters = parameters, + signatureDisplay = signatureDisplay, + typeParameters = fn.typeParameters.map { it.name }, + kotlin = KotlinFunctionInfo( + receiverTypeFqName = receiverType?.let { kmTypeToFqName(it) } ?: "", + receiverTypeDisplay = receiverType?.let { kmTypeToDisplay(it) } ?: "", + isSuspend = fn.isSuspend, + isInline = fn.isInline, + isInfix = fn.isInfix, + isOperator = fn.isOperator, + isTailrec = fn.isTailrec, + isExternal = fn.isExternal, + isExpect = fn.isExpect, + isReturnTypeNullable = fn.returnType.isNullable, + ), + ), + ) + } + + private fun extractProperty( + prop: KmProperty, + containingClass: String, + packageName: String, + sourceId: String, + ): JvmSymbol? { + val vis = kmVisibility(prop.visibility) + if (vis == JvmVisibility.PRIVATE) return null + + val receiverType = prop.receiverParameterType + val isExtension = receiverType != null + val kind = if (isExtension) JvmSymbolKind.EXTENSION_PROPERTY else JvmSymbolKind.PROPERTY + + val fqName = if (containingClass.isNotEmpty()) + "$containingClass.${prop.name}" else "$packageName.${prop.name}" + + return JvmSymbol( + key = fqName, + sourceId = sourceId, + fqName = fqName, + shortName = prop.name, + packageName = packageName, + kind = kind, + language = JvmSourceLanguage.KOTLIN, + visibility = vis, + data = JvmFieldInfo( + containingClassFqName = containingClass, + typeFqName = kmTypeToFqName(prop.returnType), + typeDisplay = kmTypeToDisplay(prop.returnType), + kotlin = KotlinPropertyInfo( + receiverTypeFqName = receiverType?.let { kmTypeToFqName(it) } ?: "", + receiverTypeDisplay = receiverType?.let { kmTypeToDisplay(it) } ?: "", + isConst = prop.isConst, + isLateinit = prop.isLateinit, + hasGetter = prop.getter != null, + hasSetter = prop.setter != null, + isDelegated = prop.isDelegated, + isTypeNullable = prop.returnType.isNullable, + ), + ), + ) + } + + private fun kmTypeToFqName(type: KmType): String = when (val c = type.classifier) { + is KmClassifier.Class -> c.name.replace('/', '.') + is KmClassifier.TypeAlias -> c.name.replace('/', '.') + is KmClassifier.TypeParameter -> "T${c.id}" + } + + private fun kmTypeToDisplay(type: KmType): String { + val base = kmTypeToFqName(type).substringAfterLast('.') + val args = type.arguments.mapNotNull { it.type?.let { t -> kmTypeToDisplay(t) } } + return buildString { + append(base) + if (args.isNotEmpty()) append("<${args.joinToString(", ")}>") + if (type.isNullable) append("?") + } + } + + private fun kmVisibility(vis: Visibility) = when (vis) { + Visibility.PUBLIC -> JvmVisibility.PUBLIC + Visibility.PROTECTED -> JvmVisibility.PROTECTED + Visibility.INTERNAL -> JvmVisibility.INTERNAL + Visibility.PRIVATE, Visibility.PRIVATE_TO_THIS, Visibility.LOCAL -> JvmVisibility.PRIVATE + } + + private class MetadataCollector : ClassVisitor(Opcodes.ASM9) { + var metadataHeader: Metadata? = null + var packageName = "" + + private var metadataKind: Int? = null + private var metadataVersion: IntArray? = null + private var data1: Array? = null + private var data2: Array? = null + private var extraString: String? = null + private var pn: String? = null + private var extraInt: Int? = null + + override fun visit( + version: Int, access: Int, name: String, + signature: String?, superName: String?, interfaces: Array?, + ) { + val lastSlash = name.lastIndexOf('/') + packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" + } + + override fun visitAnnotation(descriptor: String?, visible: Boolean): AnnotationVisitor? { + if (descriptor != "Lkotlin/Metadata;") return null + + return object : AnnotationVisitor(Opcodes.ASM9) { + override fun visit(name: String?, value: Any?) { + when (name) { + "mv" -> { + if (value is IntArray) { + metadataVersion = value.copyOf() + } + } + "k" -> metadataKind = value as? Int + "xi" -> extraInt = value as? Int + "xs" -> extraString = value as? String + "pn" -> pn = value as? String + } + } + + override fun visitArray(name: String?): AnnotationVisitor = + object : AnnotationVisitor(Opcodes.ASM9) { + private val values = mutableListOf() + override fun visit(n: String?, value: Any?) { + value?.let { values.add(it) } + } + + override fun visitEnd() { + when (name) { + "mv" -> metadataVersion = + values.filterIsInstance().toIntArray() + + "d1" -> data1 = values.filterIsInstance().toTypedArray() + "d2" -> data2 = values.filterIsInstance().toTypedArray() + } + } + } + + override fun visitEnd() { + val kind = metadataKind ?: return + metadataHeader = Metadata( + kind = kind, + metadataVersion = metadataVersion ?: intArrayOf(), + data1 = data1 ?: emptyArray(), + data2 = data2 ?: emptyArray(), + extraString = extraString ?: "", + packageName = pn ?: "", + extraInt = extraInt ?: 0, + ) + } + } + } + } +} diff --git a/lsp/jvm-symbol-models/build.gradle.kts b/lsp/jvm-symbol-models/build.gradle.kts new file mode 100644 index 0000000000..40417fdcbf --- /dev/null +++ b/lsp/jvm-symbol-models/build.gradle.kts @@ -0,0 +1,37 @@ +import com.google.protobuf.gradle.id +import com.itsaky.androidide.plugins.conf.configureProtoc + +plugins { + id("java-library") + id("org.jetbrains.kotlin.jvm") + alias(libs.plugins.google.protobuf) +} + +configureProtoc(protobuf = protobuf, protocVersion = libs.versions.protobuf.asProvider()) + +protobuf { + plugins { + id("kotlin-ext") { + artifact = "dev.hsbrysk:protoc-gen-kotlin-ext:${libs.versions.protoc.gen.kotlin.ext.get()}:jdk8@jar" + } + } + generateProtoTasks { + all().forEach { task -> + task.plugins { + id("kotlin-ext") { + outputSubDir = "kotlin" + } + } + task.builtins { + getByName("java") { + option("lite") + } + } + } + } +} + +dependencies { + api(libs.google.protobuf.java) + api(libs.google.protobuf.kotlin) +} diff --git a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto new file mode 100644 index 0000000000..a925e45979 --- /dev/null +++ b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto @@ -0,0 +1,210 @@ +syntax = "proto3"; + +package org.appdevforall.codeonthego.indexing.jvm; + +option java_package = "org.appdevforall.codeonthego.indexing.jvm.proto"; +option java_outer_classname = "JvmSymbolProtos"; +option java_multiple_files = false; + +message JvmSymbolData { + + string fq_name = 1; + string short_name = 2; + string package_name = 3; + string source_id = 4; + + JvmSymbolKind kind = 5; + JvmSourceLanguage language = 6; + JvmVisibility visibility = 7; + bool is_deprecated = 8; + + oneof data { + ClassData class_data = 20; + FunctionData function_data = 21; + FieldData field_data = 22; + EnumEntryData enum_entry_data = 23; + TypeAliasData type_alias_data = 24; + } +} + +message ClassData { + + // FQN of the enclosing class (empty for top-level classes) + string containing_class_fq_name = 1; + + // Direct supertypes + repeated string supertype_fq_names = 2; + + // Type parameters: ["T", "R : Comparable"] + repeated string type_parameters = 3; + + // Modifiers + bool is_abstract = 4; + bool is_final = 5; + bool is_inner = 6; + bool is_static = 7; // static nested class in Java + + KotlinClassData kotlin = 10; +} + +message KotlinClassData { + bool is_data = 1; + bool is_value = 2; // inline/value class + bool is_sealed = 3; + bool is_fun_interface = 4; + bool is_expect = 5; + bool is_actual = 6; + bool is_external = 7; + + // Sealed subclass FQNs (only for sealed classes/interfaces) + repeated string sealed_subclasses = 8; + + // Companion object name (empty if none or uses default "Companion") + string companion_object_name = 9; +} + +message FunctionData { + + // FQN of the containing class (empty for top-level functions) + string containing_class_fq_name = 1; + + // Return type + string return_type_fq_name = 2; + string return_type_display = 3; + + // Parameters + int32 parameter_count = 4; + repeated ParameterData parameters = 5; + + // Human-readable signature: "(count: Int, sep: String): String" + string signature_display = 6; + + // Type parameters: ["T", "R : Comparable"] + repeated string type_parameters = 7; + + // Modifiers + bool is_static = 8; + bool is_abstract = 9; + bool is_final = 10; + + KotlinFunctionData kotlin = 20; +} + +message ParameterData { + string name = 1; + string type_fq_name = 2; + string type_display = 3; + bool has_default_value = 4; + + bool is_crossinline = 5; + bool is_noinline = 6; + bool is_vararg = 7; +} + +message KotlinFunctionData { + // Extension receiver type + string receiver_type_fq_name = 1; + string receiver_type_display = 2; + + // Modifiers + bool is_suspend = 3; + bool is_inline = 4; + bool is_infix = 5; + bool is_operator = 6; + bool is_tailrec = 7; + bool is_external = 8; + bool is_expect = 9; + bool is_actual = 10; + + bool is_return_type_nullable = 11; +} + +message FieldData { + + // FQN of the containing class (empty for top-level properties) + string containing_class_fq_name = 1; + + // Type of the field/property + string type_fq_name = 2; + string type_display = 3; + + // Modifiers + bool is_static = 4; + bool is_final = 5; + + // Constant value (for compile-time constants, as string repr) + string constant_value = 6; + + KotlinPropertyData kotlin = 20; +} + +message KotlinPropertyData { + // Extension receiver type + string receiver_type_fq_name = 1; + string receiver_type_display = 2; + + bool is_const = 3; + bool is_lateinit = 4; + bool has_getter = 5; + bool has_setter = 6; + bool is_delegated = 7; + bool is_expect = 8; + bool is_actual = 9; + bool is_external = 10; + + bool is_type_nullable = 11; +} + +message EnumEntryData { + // FQN of the containing enum class + string containing_enum_fq_name = 1; + + // Ordinal position + int32 ordinal = 2; +} + +message TypeAliasData { + // The type this alias expands to + string expanded_type_fq_name = 1; + string expanded_type_display = 2; + + // Type parameters: ["T"] + repeated string type_parameters = 3; +} + +enum JvmSymbolKind { + KIND_UNKNOWN = 0; + KIND_CLASS = 1; + KIND_INTERFACE = 2; + KIND_ENUM = 3; + KIND_ENUM_ENTRY = 4; + KIND_ANNOTATION_CLASS = 5; + KIND_OBJECT = 6; + KIND_COMPANION_OBJECT = 7; + KIND_DATA_CLASS = 8; + KIND_VALUE_CLASS = 9; + KIND_SEALED_CLASS = 10; + KIND_SEALED_INTERFACE = 11; + KIND_FUNCTION = 12; + KIND_EXTENSION_FUNCTION = 13; + KIND_CONSTRUCTOR = 14; + KIND_PROPERTY = 15; + KIND_EXTENSION_PROPERTY = 16; + KIND_FIELD = 17; + KIND_TYPE_ALIAS = 18; +} + +enum JvmSourceLanguage { + LANGUAGE_UNKNOWN = 0; + LANGUAGE_JAVA = 1; + LANGUAGE_KOTLIN = 2; +} + +enum JvmVisibility { + VISIBILITY_UNKNOWN = 0; + VISIBILITY_PUBLIC = 1; + VISIBILITY_PROTECTED = 2; + VISIBILITY_INTERNAL = 3; + VISIBILITY_PRIVATE = 4; + VISIBILITY_PACKAGE_PRIVATE = 5; +} diff --git a/lsp/kotlin/build.gradle.kts b/lsp/kotlin/build.gradle.kts index 66f2a74f4b..54c0fa6206 100644 --- a/lsp/kotlin/build.gradle.kts +++ b/lsp/kotlin/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { kapt(projects.annotationProcessors) implementation(projects.lsp.api) + implementation(projects.lsp.jvmSymbolIndex) implementation(projects.lsp.models) implementation(projects.editorApi) implementation(projects.eventbusEvents) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/KotlinLanguageServer.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/KotlinLanguageServer.kt index 3bd433a429..bf0e4d9ddd 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/KotlinLanguageServer.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/KotlinLanguageServer.kt @@ -43,6 +43,7 @@ import com.itsaky.androidide.lsp.models.SignatureHelp import com.itsaky.androidide.lsp.models.SignatureHelpParams import com.itsaky.androidide.models.Range import com.itsaky.androidide.projects.FileManager +import com.itsaky.androidide.projects.ProjectManagerImpl import com.itsaky.androidide.projects.api.Workspace import com.itsaky.androidide.utils.DocumentUtils import com.itsaky.androidide.utils.Environment @@ -55,6 +56,7 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JvmIndexingService import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode @@ -122,6 +124,11 @@ class KotlinLanguageServer : ILanguageServer { override fun setupWithProject(workspace: Workspace) { logger.info("setupWithProject called, initialized={}", initialized) + (ProjectManagerImpl.getInstance() + .indexingServiceManager + .getService(JvmIndexingService.ID) as? JvmIndexingService?) + ?.refresh() + val jdkHome = Environment.JAVA_HOME.toPath() val jdkRelease = IJdkDistributionProvider.DEFAULT_JAVA_RELEASE val intellijPluginRoot = Paths.get( diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/CompilationEnvironment.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/CompilationEnvironment.kt index f9b20ebc3a..6bb19535d9 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/CompilationEnvironment.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/CompilationEnvironment.kt @@ -1,7 +1,7 @@ package com.itsaky.androidide.lsp.kotlin.compiler -import com.itsaky.androidide.lsp.kotlin.FileEventConsumer import com.itsaky.androidide.lsp.kotlin.KtFileManager +import com.itsaky.androidide.lsp.kotlin.completion.SymbolVisibilityChecker import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.platform.declarations.KotlinAnnotationsResolverFactory import org.jetbrains.kotlin.analysis.api.platform.declarations.KotlinDeclarationProviderFactory @@ -51,7 +51,7 @@ import kotlin.io.path.pathString * @param jdkHome Path to the JDK installation directory. * @param jdkRelease The JDK release version at [jdkHome]. */ -class CompilationEnvironment( +internal class CompilationEnvironment( val projectModel: KotlinProjectModel, val intellijPluginRoot: Path, val jdkHome: Path, @@ -82,6 +82,12 @@ class CompilationEnvironment( val coreApplicationEnvironment: CoreApplicationEnvironment get() = session.coreApplicationEnvironment + val moduleResolver: ModuleResolver? + get() = projectModel.moduleResolver + + val symbolVisibilityChecker: SymbolVisibilityChecker? + get() = projectModel.symbolVisibilityChecker + private val envMessageCollector = object : MessageCollector { override fun clear() { } diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/Compiler.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/Compiler.kt index a02e6ebe44..48bde185b6 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/Compiler.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/Compiler.kt @@ -1,6 +1,5 @@ package com.itsaky.androidide.lsp.kotlin.compiler -import com.itsaky.androidide.lsp.kotlin.FileEventConsumer import com.itsaky.androidide.utils.DocumentUtils import org.jetbrains.kotlin.com.intellij.lang.Language import org.jetbrains.kotlin.com.intellij.openapi.vfs.StandardFileSystems @@ -15,10 +14,9 @@ import org.jetbrains.kotlin.psi.KtPsiFactory import org.slf4j.LoggerFactory import java.nio.file.Path import java.nio.file.Paths -import kotlin.io.path.extension import kotlin.io.path.pathString -class Compiler( +internal class Compiler( projectModel: KotlinProjectModel, intellijPluginRoot: Path, jdkHome: Path, diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/KotlinProjectModel.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/KotlinProjectModel.kt index da57f9f549..90fcae59a2 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/KotlinProjectModel.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/KotlinProjectModel.kt @@ -1,9 +1,11 @@ package com.itsaky.androidide.lsp.kotlin.compiler +import com.itsaky.androidide.lsp.kotlin.completion.SymbolVisibilityChecker import com.itsaky.androidide.projects.api.AndroidModule import com.itsaky.androidide.projects.api.ModuleProject import com.itsaky.androidide.projects.api.Workspace import com.itsaky.androidide.projects.models.bootClassPaths +import org.jetbrains.kotlin.analysis.api.projectStructure.KaLibraryModule import org.jetbrains.kotlin.analysis.api.projectStructure.KaSourceModule import org.jetbrains.kotlin.analysis.project.structure.builder.KtModuleProviderBuilder import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtLibraryModule @@ -11,6 +13,8 @@ import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtSourceModu import org.jetbrains.kotlin.platform.TargetPlatform import org.jetbrains.kotlin.platform.jvm.JvmPlatforms import org.slf4j.LoggerFactory +import java.nio.file.Path +import kotlin.io.path.nameWithoutExtension /** * Holds the project structure derived from a [Workspace]. @@ -23,15 +27,23 @@ import org.slf4j.LoggerFactory * (build complete), it notifies registered listeners so they can * refresh their sessions. */ -class KotlinProjectModel { +internal class KotlinProjectModel { private val logger = LoggerFactory.getLogger(KotlinProjectModel::class.java) private var workspace: Workspace? = null private var platform: TargetPlatform = JvmPlatforms.defaultJvmPlatform + private var _moduleResolver: ModuleResolver? = null + private var _symbolVisibilityChecker: SymbolVisibilityChecker? = null private val listeners = mutableListOf() + val moduleResolver: ModuleResolver? + get() = _moduleResolver + + val symbolVisibilityChecker: SymbolVisibilityChecker? + get() = _symbolVisibilityChecker + /** * The kind of change that occurred. */ @@ -80,49 +92,55 @@ class KotlinProjectModel { this.platform = this@KotlinProjectModel.platform val moduleProjects = workspace.subProjects + .asSequence() .filterIsInstance() .filter { it.path != workspace.rootProject.path } + val jarToModMap = mutableMapOf() + + fun addLibrary(path: Path): KaLibraryModule { + val module = addModule(buildKtLibraryModule { + this.platform = this@KotlinProjectModel.platform + this.libraryName = path.nameWithoutExtension + addBinaryRoot(path) + }) + + jarToModMap[path] = module + return module + } + val bootClassPaths = moduleProjects .filterIsInstance() .flatMap { project -> project.bootClassPaths + .asSequence() .filter { it.exists() } - .map { bootClassPath -> - addModule(buildKtLibraryModule { - this.platform = this@KotlinProjectModel.platform - this.libraryName = bootClassPath.nameWithoutExtension - addBinaryRoot(bootClassPath.toPath()) - }) - } + .map { it.toPath() } + .map(::addLibrary) } val libraryDependencies = moduleProjects .flatMap { it.getCompileClasspaths() } .filter { it.exists() } - .associateWith { library -> - addModule(buildKtLibraryModule { - this.platform = this@KotlinProjectModel.platform - this.libraryName = library.nameWithoutExtension - addBinaryRoot(library.toPath()) - }) - } + .map { it.toPath() } + .associateWith(::addLibrary) val subprojectsAsModules = mutableMapOf() fun getOrCreateModule(project: ModuleProject): KaSourceModule { subprojectsAsModules[project]?.let { return it } + val sourceRoots = project.getSourceDirectories().map { it.toPath() } val module = buildKtSourceModule { this.platform = this@KotlinProjectModel.platform this.moduleName = project.name - addSourceRoots(project.getSourceDirectories().map { it.toPath() }) + addSourceRoots(sourceRoots) bootClassPaths.forEach { addRegularDependency(it) } project.getCompileClasspaths(excludeSourceGeneratedClassPath = true) .forEach { classpath -> - val libDep = libraryDependencies[classpath] + val libDep = libraryDependencies[classpath.toPath()] if (libDep == null) { logger.error( "Skipping non-existent classpath classpath: {}", @@ -143,6 +161,10 @@ class KotlinProjectModel { } moduleProjects.forEach { addModule(getOrCreateModule(it)) } + + val moduleResolver = ModuleResolver(jarMap = jarToModMap) + _moduleResolver = moduleResolver + _symbolVisibilityChecker = SymbolVisibilityChecker(moduleResolver) } } diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/ModuleResolver.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/ModuleResolver.kt new file mode 100644 index 0000000000..704d02978a --- /dev/null +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/ModuleResolver.kt @@ -0,0 +1,25 @@ +package com.itsaky.androidide.lsp.kotlin.compiler + +import org.jetbrains.kotlin.analysis.api.projectStructure.KaLibraryModule +import org.jetbrains.kotlin.analysis.api.projectStructure.KaModule +import org.slf4j.LoggerFactory +import java.nio.file.Path +import java.nio.file.Paths + +internal class ModuleResolver( + private val jarMap: Map, +) { + companion object { + private val logger = LoggerFactory.getLogger(ModuleResolver::class.java) + } + + /** + * Find the module that declares the given source ID (JAR, source file, etc.) + */ + fun findDeclaringModule(sourceId: String): KaModule? { + val path = Paths.get(sourceId) + jarMap[path]?.let { return it } + + return null + } +} \ No newline at end of file diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 51613960b2..ca003ee1c8 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -38,12 +38,10 @@ import org.jetbrains.kotlin.analysis.api.types.KaClassType import org.jetbrains.kotlin.analysis.api.types.KaType import org.jetbrains.kotlin.com.intellij.psi.PsiElement import org.jetbrains.kotlin.name.Name -import org.jetbrains.kotlin.psi.KtDotQualifiedExpression import org.jetbrains.kotlin.psi.KtElement import org.jetbrains.kotlin.psi.KtQualifiedExpression import org.jetbrains.kotlin.psi.KtSafeQualifiedExpression import org.jetbrains.kotlin.psi.psiUtil.getParentOfType -import org.jetbrains.kotlin.psi.psiUtil.startOffset import org.jetbrains.kotlin.types.Variance import org.slf4j.LoggerFactory @@ -58,7 +56,7 @@ private val logger = LoggerFactory.getLogger("KotlinCompletions") * @param params The completion parameters. * @return The completion result. */ -fun CompilationEnvironment.complete(params: CompletionParams): CompletionResult { +internal fun CompilationEnvironment.complete(params: CompletionParams): CompletionResult { val managedFile = fileManager.getOpenFile(params.file) if (managedFile == null) { logger.warn("No managed file for {}", params.file) @@ -91,6 +89,12 @@ fun CompilationEnvironment.complete(params: CompletionParams): CompletionResult useSiteElement = completionKtFile, resolutionMode = KaDanglingFileResolutionMode.PREFER_SELF, ) { + val symbolVisibilityChecker = this@complete.symbolVisibilityChecker + if (symbolVisibilityChecker == null) { + logger.error("No symbol visibility checker available!") + return@analyzeCopy CompletionResult.EMPTY + } + val cursorContext = resolveCursorContext(completionKtFile, completionOffset) if (cursorContext == null) { logger.error( @@ -117,6 +121,7 @@ fun CompilationEnvironment.complete(params: CompletionParams): CompletionResult collectScopeCompletions( scopeContext = scopeContext, scope = compositeScope, + symbolVisibilityChecker = symbolVisibilityChecker, ktElement = ktElement, partial = partial, to = items @@ -240,6 +245,7 @@ private fun KaSession.collectExtensionFunctions( private fun KaSession.collectScopeCompletions( scopeContext: KaScopeContext, scope: KaScope, + symbolVisibilityChecker: SymbolVisibilityChecker, ktElement: KtElement, partial: String, to: MutableList, diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/SymbolVisibilityChecker.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/SymbolVisibilityChecker.kt new file mode 100644 index 0000000000..010b187e41 --- /dev/null +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/SymbolVisibilityChecker.kt @@ -0,0 +1,87 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.itsaky.androidide.lsp.kotlin.compiler.ModuleResolver +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility +import org.jetbrains.kotlin.analysis.api.projectStructure.KaModule +import org.jetbrains.kotlin.analysis.api.projectStructure.allDirectDependencies +import java.util.concurrent.ConcurrentHashMap + +internal class SymbolVisibilityChecker( + private val moduleResolver: ModuleResolver, +) { + // visibility check cache, for memoization + // useSiteModule -> list of modules visible from useSiteModule + private val moduleVisibilityCache = ConcurrentHashMap>() + + fun isVisible( + symbol: JvmSymbol, + useSiteModule: KaModule, + useSitePackage: String? = null, + ): Boolean { + val declaringModule = moduleResolver.findDeclaringModule(symbol.sourceId) + ?: return false + + if (!isReachable(useSiteModule, declaringModule)) return false + if (!arePlatformCompatible(useSiteModule, declaringModule)) return false + if (!isDeclarationVisible(symbol, useSiteModule, declaringModule, useSitePackage)) return false + + return true + } + + fun isReachable(useSiteModule: KaModule, declaringModule: KaModule): Boolean { + if (useSiteModule == declaringModule) return true + if (moduleVisibilityCache[useSiteModule]?.contains(declaringModule) == true) return true + + // walk the dependency graph + val visited = mutableSetOf() + val queue = ArrayDeque() + queue.add(useSiteModule) + + while (queue.isNotEmpty()) { + val current = queue.removeFirst() + if (!visited.add(current)) continue + if (current == declaringModule) return true + + queue.addAll(current.allDirectDependencies()) + } + + return false + } + + fun arePlatformCompatible(useSiteModule: KaModule, declaringModule: KaModule): Boolean { + val usePlatform = useSiteModule.targetPlatform + val declPlatform = declaringModule.targetPlatform + + // the declaring platform must be a superset of, or equal to the use + // site platform + return declPlatform.componentPlatforms.all { declComp -> + usePlatform.componentPlatforms.any { useComp -> + useComp == declComp || useComp.platformName == declComp.platformName + } + } + } + + fun isDeclarationVisible( + symbol: JvmSymbol, + useSiteModule: KaModule, + declaringModule: KaModule, + useSitePackage: String? = null, + ): Boolean { + val isSamePackage = useSitePackage != null && useSitePackage == symbol.packageName + + // TODO(itsaky): this should check whether the use-site element + // is contained in a class that is a descendant of the + // class declaring the given symbol. + // For now, we assume true in all cases. + val isDescendant = true + + return when (symbol.visibility) { + JvmVisibility.PUBLIC -> true + JvmVisibility.PRIVATE -> false + JvmVisibility.INTERNAL -> useSiteModule == declaringModule + JvmVisibility.PROTECTED -> isSamePackage || isDescendant + JvmVisibility.PACKAGE_PRIVATE -> isSamePackage + } + } +} diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/diagnostic/KotlinDiagnosticProvider.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/diagnostic/KotlinDiagnosticProvider.kt index ac2c38672f..4472c1a652 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/diagnostic/KotlinDiagnosticProvider.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/diagnostic/KotlinDiagnosticProvider.kt @@ -9,7 +9,6 @@ import com.itsaky.androidide.models.Range import com.itsaky.androidide.projects.FileManager import kotlinx.coroutines.CancellationException import org.jetbrains.kotlin.analysis.api.KaExperimentalApi -import org.jetbrains.kotlin.analysis.api.analyze import org.jetbrains.kotlin.analysis.api.components.KaDiagnosticCheckerFilter import org.jetbrains.kotlin.analysis.api.diagnostics.KaDiagnosticWithPsi import org.jetbrains.kotlin.analysis.api.diagnostics.KaSeverity @@ -23,7 +22,7 @@ import kotlin.time.toKotlinInstant private val logger = LoggerFactory.getLogger("KotlinDiagnosticProvider") -fun CompilationEnvironment.collectDiagnosticsFor(file: Path): DiagnosticResult = try { +internal fun CompilationEnvironment.collectDiagnosticsFor(file: Path): DiagnosticResult = try { logger.info("Analyzing file: {}", file) return doAnalyze(file) } catch (err: Throwable) { diff --git a/settings.gradle.kts b/settings.gradle.kts index dfb9c6f997..6165b7a722 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -144,7 +144,10 @@ include( ":xml-inflater", ":lsp:api", ":lsp:models", + ":lsp:indexing", ":lsp:java", + ":lsp:jvm-symbol-index", + ":lsp:jvm-symbol-models", ":lsp:kotlin", ":lsp:kotlin-core", ":lsp:kotlin-stdlib-generator", diff --git a/subprojects/projects/build.gradle.kts b/subprojects/projects/build.gradle.kts index 79054954dd..9d2683ae80 100644 --- a/subprojects/projects/build.gradle.kts +++ b/subprojects/projects/build.gradle.kts @@ -23,6 +23,7 @@ dependencies { api(projects.eventbus) api(projects.eventbusEvents) + api(projects.lsp.indexing) api(projects.subprojects.projectModels) api(projects.subprojects.toolingApi) diff --git a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt index 32ab789a01..357fda6b78 100644 --- a/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt +++ b/subprojects/projects/src/main/java/com/itsaky/androidide/projects/ProjectManagerImpl.kt @@ -53,6 +53,8 @@ import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.service.IndexingService +import org.appdevforall.codeonthego.indexing.service.IndexingServiceManager import org.greenrobot.eventbus.EventBus import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode @@ -75,8 +77,19 @@ import kotlin.io.path.pathString class ProjectManagerImpl : IProjectManager, EventReceiver { + + private var _indexingServiceManager: IndexingServiceManager? = null lateinit var projectPath: String + val indexingServiceManager: IndexingServiceManager + get() { + if (_indexingServiceManager == null) { + _indexingServiceManager = IndexingServiceManager() + } + + return _indexingServiceManager!! + } + @Volatile internal var pluginProjectCached: Boolean? = null @@ -89,7 +102,7 @@ class ProjectManagerImpl : override val projectDirPath: String get() = projectPath - override val projectSyncIssues: List? + override val projectSyncIssues: List get() = gradleBuild?.syncIssueList ?: emptyList() companion object { @@ -140,6 +153,10 @@ class ProjectManagerImpl : gradleBuild.syncIssueList, ) + withStopWatch("notify indexing services") { + indexingServiceManager.onProjectSynced() + } + withStopWatch("Setup project") { val indexerScope = CoroutineScope(Dispatchers.Default) val modulesFlow = @@ -232,6 +249,9 @@ class ProjectManagerImpl : this.workspace = null pluginProjectCached = null + _indexingServiceManager?.close() + _indexingServiceManager = null + (this.androidBuildVariants as? MutableMap?)?.clear() }