Implemented LuceneBankFinder

BankingUiCommon/src/main/java/net/dankito/banking/ui/presenter/BankingPresenter.kt

@@ -16,7 +16,7 @@ import net.dankito.banking.ui.model.tan.EnterTanGeneratorAtcResult
 import net.dankito.banking.ui.model.tan.EnterTanResult
 import net.dankito.banking.ui.model.tan.TanChallenge
 import net.dankito.banking.ui.model.tan.TanGeneratorTanMedium
-import net.dankito.fints.banks.BankFinder
+import net.dankito.fints.banks.IBankFinder
 import net.dankito.fints.model.BankInfo
 import net.dankito.utils.IThreadPool
 import net.dankito.utils.ThreadPool
@@ -31,7 +31,8 @@ import kotlin.collections.ArrayList
 
 open class BankingPresenter(
     protected val bankingClientCreator: IBankingClientCreator,
-    protected val dataFolder: File,
+    protected val bankFinder: IBankFinder,
+    protected val databaseFolder: File,
     protected val persister: IBankingPersistence,
     protected val router: IRouter,
     protected val threadPool: IThreadPool = ThreadPool()
@@ -44,9 +45,6 @@ open class BankingPresenter(
     }
 
 
-    protected val bankFinder: BankFinder = BankFinder()
-
-
     protected val clientsForAccounts = mutableMapOf<Account, IBankingClient>()
 
     protected var selectedBankAccountsField = mutableListOf<BankAccount>()
@@ -96,7 +94,7 @@ open class BankingPresenter(
 
     protected open fun readPersistedAccounts() {
         try {
-            dataFolder.mkdirs()
+            databaseFolder.mkdirs()
 
             val deserializedAccounts = persister.readPersistedAccounts()
 
@@ -105,7 +103,7 @@ open class BankingPresenter(
                 val bankInfo = BankInfo(bank.name, bank.bankCode, bank.bic, "", "", "", bank.finTsServerAddress, "FinTS V3.0", null)
 
                 val newClient = bankingClientCreator.createClient(bankInfo, account.customerId, account.password,
-                    dataFolder, threadPool, callback)
+                    databaseFolder, threadPool, callback)
 
                 try {
                     newClient.restoreData()
@@ -133,7 +131,7 @@ open class BankingPresenter(
     // TODO: move BankInfo out of fints4javaLib
     open fun addAccountAsync(bankInfo: BankInfo, customerId: String, pin: String, callback: (AddAccountResponse) -> Unit) {
 
-        val newClient = bankingClientCreator.createClient(bankInfo, customerId, pin, dataFolder, threadPool, this.callback)
+        val newClient = bankingClientCreator.createClient(bankInfo, customerId, pin, databaseFolder, threadPool, this.callback)
 
         newClient.addAccountAsync { response ->
             val account = response.account
@@ -254,10 +252,6 @@ open class BankingPresenter(
     }
 
 
-    open fun preloadBanksAsync() {
-        findUniqueBankForBankCodeAsync("1") { }
-    }
-
     open fun findUniqueBankForIbanAsync(iban: String, callback: (BankInfo?) -> Unit) {
         threadPool.runAsync {
             callback(findUniqueBankForIban(iban))
@@ -278,12 +272,6 @@ open class BankingPresenter(
         return null
     }
 
-    open fun findUniqueBankForBankCodeAsync(bankCode: String, callback: (BankInfo?) -> Unit) {
-        threadPool.runAsync {
-            callback(findUniqueBankForBankCode(bankCode))
-        }
-    }
-
     open fun findUniqueBankForBankCode(bankCode: String): BankInfo? {
         val searchResult = bankFinder.findBankByBankCode(bankCode)
 

build.gradle

@@ -9,6 +9,8 @@ ext {
 
     javaUtilsVersion = '1.0.16-SNAPSHOT'
 
+    luceneUtilsVersion = "0.5.1-SNAPSHOT"
+
 
     hbci4jVersion = '3.1.37'
 

fints4javaAndroidApp/src/main/java/net/dankito/banking/fints4java/android/di/BankingModule.kt

@@ -12,6 +12,8 @@ import net.dankito.banking.persistence.IBankingPersistence
 import net.dankito.banking.ui.IBankingClientCreator
 import net.dankito.banking.ui.IRouter
 import net.dankito.banking.ui.presenter.BankingPresenter
+import net.dankito.fints.banks.IBankFinder
+import net.dankito.fints.banks.LuceneBankFinder
 import net.dankito.utils.IThreadPool
 import net.dankito.utils.ThreadPool
 import net.dankito.utils.serialization.ISerializer
@@ -30,6 +32,10 @@ class BankingModule(internal val mainActivity: AppCompatActivity) {
 
         const val DataFolderKey = "data.folder"
 
+        const val DatabaseFolderKey = "database.folder"
+
+        const val IndexFolderKey = "index.folder"
+
     }
 
 
@@ -46,15 +52,43 @@ class BankingModule(internal val mainActivity: AppCompatActivity) {
     @Singleton
     @Named(DataFolderKey)
     fun provideDataFolder(applicationContext: Context) : File {
-        return File(applicationContext.filesDir, "data/accounts")
+        return File(applicationContext.filesDir, "data")
+    }
+
+    @Provides
+    @Singleton
+    @Named(DatabaseFolderKey)
+    fun provideDatabaseFolder(@Named(DataFolderKey) dataFolder: File) : File {
+        return File(dataFolder, "db")
+    }
+
+    @Provides
+    @Singleton
+    @Named(IndexFolderKey)
+    fun provideIndexFolder(@Named(DataFolderKey) dataFolder: File) : File {
+        return File(dataFolder, "index")
     }
 
 
     @Provides
     @Singleton
-    fun provideBankingPresenter(bankingClientCreator: IBankingClientCreator, @Named(DataFolderKey) dataFolder: File,
+    fun provideBankingPresenter(bankingClientCreator: IBankingClientCreator, bankFinder: IBankFinder,
-                                persister: IBankingPersistence, router: IRouter, threadPool: IThreadPool) : BankingPresenter {
+                                @Named(DatabaseFolderKey) databaseFolder: File, persister: IBankingPersistence,
-        return BankingPresenter(bankingClientCreator, dataFolder, persister, router, threadPool)
+                                router: IRouter, threadPool: IThreadPool) : BankingPresenter {
+        return BankingPresenter(bankingClientCreator, bankFinder, databaseFolder, persister, router, threadPool)
+    }
+
+    @Provides
+    @Singleton
+    fun provideBankFinder(@Named(IndexFolderKey) indexFolder: File, threadPool: IThreadPool) : IBankFinder {
+        val bankFinder = LuceneBankFinder(indexFolder)
+
+        // preloadBankList asynchronously; on Android it takes approximately 18 seconds till banks are indexed for first time -> do it as early as possible
+        threadPool.runAsync {
+            bankFinder.preloadBankList()
+        }
+
+        return bankFinder
     }
 
     @Provides
@@ -65,8 +99,8 @@ class BankingModule(internal val mainActivity: AppCompatActivity) {
 
     @Provides
     @Singleton
-    fun provideBankingPersistence(@Named(DataFolderKey) dataFolder: File, serializer: ISerializer) : IBankingPersistence {
+    fun provideBankingPersistence(@Named(DatabaseFolderKey) databaseFolder: File, serializer: ISerializer) : IBankingPersistence {
-        return BankingPersistenceJson(File(dataFolder, "accounts.json"), serializer)
+        return BankingPersistenceJson(File(databaseFolder, "accounts.json"), serializer)
     }
 
     @Provides

fints4javaAndroidApp/src/main/java/net/dankito/banking/fints4java/android/ui/dialogs/AddAccountDialog.kt

@@ -44,8 +44,6 @@ open class AddAccountDialog : DialogFragment() {
 
     init {
         BankingComponent.component.inject(this)
-
-        presenter.preloadBanksAsync()
     }
 
 

fints4javaLib/build.gradle

@@ -19,6 +19,8 @@ dependencies {
 
     api "net.dankito.utils:java-utils:$javaUtilsVersion"
 
+    implementation "net.dankito.search:lucene-4-utils:$luceneUtilsVersion"
+
 
     testCompile "junit:junit:$junitVersion"
     testCompile "org.assertj:assertj-core:$assertJVersion"

fints4javaLib/src/main/kotlin/net/dankito/fints/banks/BankFinderBase.kt

@@ -0,0 +1,37 @@
+package net.dankito.fints.banks
+
+import net.dankito.fints.model.BankInfo
+import net.dankito.utils.serialization.JacksonJsonSerializer
+import org.slf4j.LoggerFactory
+
+
+abstract class BankFinderBase : IBankFinder {
+
+    companion object {
+        const val BankListFileName = "BankList.json"
+
+        private val log = LoggerFactory.getLogger(InMemoryBankFinder::class.java)
+    }
+
+
+    protected open fun loadBankListFile(): List<BankInfo> {
+        try {
+            val bankListString = readBankListFile()
+
+            JacksonJsonSerializer().deserializeList(bankListString, BankInfo::class.java)?.let {
+                return it
+            }
+        } catch (e: Exception) {
+            log.error("Could not load bank list", e)
+        }
+
+        return listOf()
+    }
+
+    protected open fun readBankListFile(): String {
+        val inputStream = BankFinderBase::class.java.classLoader.getResourceAsStream(BankListFileName)
+
+        return inputStream.bufferedReader().readText()
+    }
+
+}

fints4javaLib/src/main/kotlin/net/dankito/fints/banks/IBankFinder.kt

@@ -0,0 +1,16 @@
+package net.dankito.fints.banks
+
+import net.dankito.fints.model.BankInfo
+
+
+interface IBankFinder {
+
+    fun getBankList(): List<BankInfo>
+
+    fun findBankByBankCode(query: String): List<BankInfo>
+
+    fun findBankByNameBankCodeOrCity(query: String?): List<BankInfo>
+
+    fun preloadBankList()
+
+}

fints4javaLib/src/main/kotlin/net/dankito/fints/banks/InMemoryBankFinder.kt

@@ -1,21 +1,15 @@
 package net.dankito.fints.banks
 
 import net.dankito.fints.model.BankInfo
-import net.dankito.utils.serialization.JacksonJsonSerializer
-import org.slf4j.LoggerFactory
 
 
-open class BankFinder {
+open class InMemoryBankFinder : BankFinderBase(), IBankFinder {
-
-    companion object {
-        private val log = LoggerFactory.getLogger(BankFinder::class.java)
-    }
 
 
     protected var bankListField: List<BankInfo>? = null
 
 
-    open fun findBankByBankCode(query: String): List<BankInfo> {
+    override fun findBankByBankCode(query: String): List<BankInfo> {
         if (query.isEmpty()) {
             return getBankList()
         }
@@ -23,7 +17,7 @@ open class BankFinder {
         return getBankList().filter { it.bankCode.startsWith(query) }
     }
 
-    open fun findBankByNameBankCodeOrCity(query: String?): List<BankInfo> {
+    override fun findBankByNameBankCodeOrCity(query: String?): List<BankInfo> {
         if (query.isNullOrEmpty()) {
             return getBankList()
         }
@@ -53,32 +47,21 @@ open class BankFinder {
     }
 
 
-    open fun getBankList(): List<BankInfo> {
+    override fun preloadBankList() {
+        findBankByBankCode("1")
+    }
+
+
+    override fun getBankList(): List<BankInfo> {
         bankListField?.let {
             return it
         }
 
-        val bankList = loadBankList()
+        val bankList = loadBankListFile()
 
         this.bankListField = bankList
 
         return bankList
     }
 
-    protected open fun loadBankList(): List<BankInfo> {
-        try {
-            val inputStream = BankFinder::class.java.classLoader.getResourceAsStream("BankList.json")
-
-            val bankListString = inputStream.bufferedReader().readText()
-
-            JacksonJsonSerializer().deserializeList(bankListString, BankInfo::class.java)?.let {
-                return it
-            }
-        } catch (e: Exception) {
-            log.error("Could not load bank list", e)
-        }
-
-        return listOf()
-    }
-
 }

fints4javaLib/src/main/kotlin/net/dankito/fints/banks/LuceneBankFinder.kt

@@ -0,0 +1,172 @@
+package net.dankito.fints.banks
+
+import net.dankito.fints.model.BankInfo
+import net.dankito.utils.hashing.HashAlgorithm
+import net.dankito.utils.hashing.HashService
+import net.dankito.utils.io.FileUtils
+import net.dankito.utils.lucene.index.DocumentsWriter
+import net.dankito.utils.lucene.index.FieldBuilder
+import net.dankito.utils.lucene.search.FieldMapper
+import net.dankito.utils.lucene.search.QueryBuilder
+import net.dankito.utils.lucene.search.Searcher
+import org.apache.lucene.document.Document
+import org.apache.lucene.search.Query
+import java.io.File
+
+
+open class LuceneBankFinder(indexFolder: File) : BankFinderBase(), IBankFinder {
+
+    companion object {
+
+        const val IndexedBankListFileHashIdFieldName = "IndexedBankListFileHashId"
+        const val IndexedBankListFileHashIdFieldValue = "IndexedBankListFileHashValue"
+        const val IndexedBankListFileHashFieldName = "IndexedBankListFileHash"
+
+        const val BankInfoNameFieldName = "name"
+        const val BankInfoBankCodeFieldName = "bank_code"
+        const val BankInfoBicFieldName = "bic"
+        const val BankInfoCityIndexedFieldName = "city_indexed"
+        const val BankInfoCityStoredFieldName = "city_stored"
+        const val BankInfoPostalCodeFieldName = "postal_code"
+        const val BankInfoChecksumMethodFieldName = "checksum_method"
+        const val BankInfoPinTanServerAddressFieldName = "pin_tan_server_address"
+        const val BankInfoPinTanVersionFieldName = "pin_tan_version"
+        const val BankInfoOldBankCodeFieldName = "old_bank_code"
+
+    }
+
+
+    protected val indexDir = File(indexFolder, "banklist")
+
+
+    protected val fileUtils = FileUtils()
+
+    protected val hashService = HashService(fileUtils)
+
+
+    protected val fields = FieldBuilder()
+
+
+    protected val queries = QueryBuilder()
+
+    protected val mapper = FieldMapper()
+
+    protected val searcher = Searcher(indexDir)
+
+
+    override fun findBankByBankCode(query: String): List<BankInfo> {
+        if (query.isBlank()) {
+            return getBankList()
+        }
+
+        val luceneQuery = queries.startsWith(BankInfoBankCodeFieldName, query)
+
+        return getBanksFromQuery(luceneQuery)
+    }
+
+    override fun findBankByNameBankCodeOrCity(query: String?): List<BankInfo> {
+        if (query.isNullOrBlank()) {
+            return getBankList()
+        }
+
+        val luceneQuery = queries.createQueriesForSingleTerms(query.toLowerCase()) { singleTerm ->
+            listOf(
+                queries.fulltextQuery(BankInfoNameFieldName, singleTerm),
+                queries.startsWith(BankInfoBankCodeFieldName, singleTerm),
+                queries.contains(BankInfoCityIndexedFieldName, singleTerm)
+            )
+        }
+
+        return getBanksFromQuery(luceneQuery)
+    }
+
+    override fun getBankList(): List<BankInfo> {
+        return getBanksFromQuery(queries.allDocumentsThatHaveField(BankInfoNameFieldName))
+    }
+
+    protected fun getBanksFromQuery(query: Query): List<BankInfo> {
+        val results = searcher.search(query, 100_000) // there are more than 16.000 banks in bank list -> 10.000 is too few
+
+        return results.hits.map { result ->
+            BankInfo(
+                mapper.string(result, BankInfoNameFieldName),
+                mapper.string(result, BankInfoBankCodeFieldName),
+                mapper.string(result, BankInfoBicFieldName),
+                mapper.string(result, BankInfoPostalCodeFieldName),
+                mapper.string(result, BankInfoCityStoredFieldName),
+                mapper.string(result, BankInfoChecksumMethodFieldName),
+                mapper.nullableString(result, BankInfoPinTanServerAddressFieldName),
+                mapper.nullableString(result, BankInfoPinTanVersionFieldName),
+                mapper.nullableString(result, BankInfoOldBankCodeFieldName)
+            )
+        }
+    }
+
+
+    override fun preloadBankList() {
+        val hashSearchResult = searcher.search(
+            queries.exact(IndexedBankListFileHashIdFieldName, IndexedBankListFileHashIdFieldValue, false))
+
+        val lastIndexedBankListFileHash = hashSearchResult.hits.firstOrNull()?.let {
+            mapper.string(it, IndexedBankListFileHashFieldName)
+        }
+
+        if (lastIndexedBankListFileHash == null) {
+            updateIndex()
+        }
+        else {
+            val currentBankListFileHash = calculateCurrentBankListFileHash()
+
+            if (currentBankListFileHash != lastIndexedBankListFileHash) {
+                updateIndex(currentBankListFileHash)
+            }
+        }
+    }
+
+    protected open fun updateIndex() {
+        updateIndex(calculateCurrentBankListFileHash())
+    }
+
+    protected open fun updateIndex(bankListFileHash: String) {
+        fileUtils.deleteFolderRecursively(indexDir)
+        indexDir.mkdirs()
+
+        DocumentsWriter(indexDir).use { writer ->
+            val banks = loadBankListFile()
+
+            writer.saveDocuments(banks.map {
+                createDocumentForBank(it, writer)
+            } )
+
+            writer.updateDocument(IndexedBankListFileHashIdFieldName, IndexedBankListFileHashIdFieldValue,
+                fields.storedField(IndexedBankListFileHashFieldName, bankListFileHash)
+            )
+        }
+    }
+
+    protected open fun createDocumentForBank(bank: BankInfo, writer: DocumentsWriter): Document {
+        return writer.createDocumentForNonNullFields(
+            fields.fullTextSearchField(BankInfoNameFieldName, bank.name, true),
+            fields.keywordField(BankInfoBankCodeFieldName, bank.bankCode, true),
+            fields.fullTextSearchField(BankInfoCityIndexedFieldName, bank.city, true),
+
+            fields.storedField(BankInfoCityStoredFieldName, bank.city),
+            fields.storedField(BankInfoBicFieldName, bank.bic),
+            fields.storedField(BankInfoPostalCodeFieldName, bank.postalCode),
+            fields.storedField(BankInfoChecksumMethodFieldName, bank.checksumMethod),
+            fields.nullableStoredField(BankInfoPinTanServerAddressFieldName, bank.pinTanAddress),
+            fields.nullableStoredField(BankInfoPinTanVersionFieldName, bank.pinTanVersion),
+            fields.nullableStoredField(BankInfoOldBankCodeFieldName, bank.oldBankCode)
+        )
+    }
+
+
+    protected open fun calculateCurrentBankListFileHash(): String {
+        return calculateHash(readBankListFile())
+    }
+
+    protected open fun calculateHash(stringToHash: String): String {
+        return hashService.hashString(HashAlgorithm.SHA512, stringToHash)
+    }
+
+}

fints4javaLib/src/test/java/net/dankito/fints/java/JavaShowcase.java

@@ -2,7 +2,8 @@ package net.dankito.fints.java;
 
 import net.dankito.fints.FinTsClient;
 import net.dankito.fints.FinTsClientCallback;
-import net.dankito.fints.banks.BankFinder;
+import net.dankito.fints.banks.IBankFinder;
+import net.dankito.fints.banks.InMemoryBankFinder;
 import net.dankito.fints.messages.datenelemente.implementierte.tan.TanGeneratorTanMedium;
 import net.dankito.fints.model.*;
 import net.dankito.fints.model.mapper.BankDataMapper;
@@ -19,7 +20,7 @@ import java.util.List;
 public class JavaShowcase {
 
     public static void main(String[] args) {
-        BankFinder bankFinder = new BankFinder();
+        IBankFinder bankFinder = new InMemoryBankFinder();
 
         // set your bank code (Bankleitzahl) here. Or create BankData manually. Required fields are:
         // bankCode, bankCountryCode (Germany = 280), finTs3ServerAddress and for bank transfers bic

fints4javaLib/src/test/kotlin/net/dankito/fints/FinTsClientTest.kt

@@ -1,6 +1,6 @@
 package net.dankito.fints
 
-import net.dankito.fints.banks.BankFinder
+import net.dankito.fints.banks.InMemoryBankFinder
 import net.dankito.fints.messages.datenelemente.abgeleiteteformate.Laenderkennzeichen
 import net.dankito.fints.messages.datenelemente.implementierte.Dialogsprache
 import net.dankito.fints.messages.datenelemente.implementierte.KundensystemStatus
@@ -62,7 +62,7 @@ class FinTsClientTest {
     private val BankDataAnonymous = BankData("10070000", Laenderkennzeichen.Germany, "https://fints.deutsche-bank.de/", "DEUTDEBBXXX")
 
     // TODO: add your settings here:
-    private val bankInfo = BankFinder().findBankByBankCode("<your bank code (BLZ) here>").first()
+    private val bankInfo = InMemoryBankFinder().findBankByBankCode("<your bank code (BLZ) here>").first()
     private val Bank = BankDataMapper().mapFromBankInfo(bankInfo)
     private val Customer = CustomerData("<your customer id (Kontonummer) here>", "<your PIN here>")
 

fints4javaLib/src/test/kotlin/net/dankito/fints/banks/BankFinderTestBase.kt

@@ -3,9 +3,12 @@ package net.dankito.fints.banks
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.Test
 
-class BankFinderTest {
+abstract class BankFinderTestBase {
 
-    private val underTest = BankFinder()
+    protected abstract fun createBankFinder(): IBankFinder
+
+
+    protected val underTest = createBankFinder()
 
 
     @Test

fints4javaLib/src/test/kotlin/net/dankito/fints/banks/InMemoryBankFinderTest.kt

@@ -0,0 +1,10 @@
+package net.dankito.fints.banks
+
+
+class InMemoryBankFinderTest : BankFinderTestBase() {
+
+    override fun createBankFinder(): IBankFinder {
+        return InMemoryBankFinder()
+    }
+
+}

fints4javaLib/src/test/kotlin/net/dankito/fints/banks/LuceneBankFinderTest.kt

@@ -0,0 +1,31 @@
+package net.dankito.fints.banks
+
+import net.dankito.utils.io.FileUtils
+import org.junit.AfterClass
+import java.io.File
+
+
+class LuceneBankFinderTest : BankFinderTestBase() {
+
+    companion object {
+        private val IndexFolder = File("testData", "index")
+
+
+        @AfterClass
+        @JvmStatic
+        fun deleteIndex() {
+            FileUtils().deleteFolderRecursively(IndexFolder.parentFile)
+        }
+    }
+
+
+    override fun createBankFinder(): IBankFinder {
+        return LuceneBankFinder(IndexFolder)
+    }
+
+
+    init {
+        underTest.preloadBankList()
+    }
+
+}