Implemented BankListCreator to parse German banks file from Deutsche Kreditwirtschaft

BankListCreator/build.gradle

@@ -0,0 +1,27 @@
+apply plugin: 'java-library'
+apply plugin: 'kotlin'
+
+
+sourceCompatibility = 1.8
+
+compileKotlin {
+    kotlinOptions.jvmTarget = "1.8"
+}
+compileTestKotlin {
+    kotlinOptions.jvmTarget = "1.8"
+}
+
+
+
+dependencies {
+    compile project(':fints4javaLib')
+
+    implementation 'org.docx4j:docx4j-JAXB-ReferenceImpl:8.1.3'
+
+
+    testCompile "junit:junit:$junitVersion"
+    testCompile "org.assertj:assertj-core:$assertJVersion"
+
+    testCompile "ch.qos.logback:logback-core:$logbackVersion"
+    testCompile "ch.qos.logback:logback-classic:$logbackVersion"
+}

BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/BankListCreator.kt

@@ -0,0 +1,20 @@
+package net.dankito.banking.banklistcreator
+
+import net.dankito.banking.banklistcreator.parser.DeutscheKreditwirtschaftBankListParser
+import net.dankito.utils.serialization.JacksonJsonSerializer
+import java.io.File
+
+
+open class BankListCreator @JvmOverloads constructor(
+    protected val parser: DeutscheKreditwirtschaftBankListParser = DeutscheKreditwirtschaftBankListParser()
+) {
+
+    fun createBankListFromDeutscheKreditwirtschaftXlsxFile(bankFileOutputFile: File,
+                                                           deutscheKreditwirtschaftXlsxFile: File) {
+
+        val banks = parser.parse(deutscheKreditwirtschaftXlsxFile)
+
+        JacksonJsonSerializer().serializeObject(banks, bankFileOutputFile)
+    }
+
+}

BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParser.kt

@@ -0,0 +1,276 @@
+package net.dankito.banking.banklistcreator.parser
+
+import net.dankito.banking.banklistcreator.parser.model.BankCodeListEntry
+import net.dankito.banking.banklistcreator.parser.model.ServerAddressesListEntry
+import net.dankito.fints.model.BankInfo
+import org.docx4j.openpackaging.packages.SpreadsheetMLPackage
+import org.slf4j.LoggerFactory
+import org.xlsx4j.org.apache.poi.ss.usermodel.DataFormatter
+import org.xlsx4j.sml.Cell
+import org.xlsx4j.sml.Row
+import org.xlsx4j.sml.SheetData
+import java.io.File
+
+
+/**
+ * Parses the list of German banks from Deutsche Kreditwirtschaft you can retrieve by registering here:
+ * https://www.hbci-zka.de/register/hersteller.htm
+ */
+open class DeutscheKreditwirtschaftBankListParser {
+
+    companion object {
+        private val log = LoggerFactory.getLogger(DeutscheKreditwirtschaftBankListParser::class.java)
+    }
+
+
+    fun parse(bankListFile: File): List<BankInfo> {
+        val xlsxPkg = SpreadsheetMLPackage.load(bankListFile)
+
+        val workbookPart = xlsxPkg.getWorkbookPart()
+        val sheets = workbookPart.contents.sheets.sheet
+        val formatter = DataFormatter()
+
+        var serverAddressesList = listOf<ServerAddressesListEntry>()
+        var bankCodesList = listOf<BankCodeListEntry>()
+
+        for (index in 0 until sheets.size) {
+            log.info("\nParsing sheet ${sheets[index].name}:\n")
+            val sheet = workbookPart.getWorksheet(index)
+            val workSheetData = sheet.contents.sheetData
+
+            if (isListWithFinTsServerAddresses(workSheetData, formatter)) {
+                serverAddressesList = parseListWithFinTsServerAddresses(workSheetData, formatter)
+            }
+            else if (isBankCodeList(workSheetData, formatter)) {
+                bankCodesList = parseBankCodeList(workSheetData, formatter)
+            }
+        }
+
+        return mapBankCodeAndServerAddressesList(bankCodesList, serverAddressesList)
+    }
+
+    private fun mapBankCodeAndServerAddressesList(banks: List<BankCodeListEntry>,
+                                                  serverAddresses: List<ServerAddressesListEntry>): List<BankInfo> {
+
+        val serverAddressesByBankCode = mutableMapOf<String, MutableList<ServerAddressesListEntry>>()
+        serverAddresses.forEach { serverAddress ->
+            if (serverAddressesByBankCode.containsKey(serverAddress.bankCode) == false) {
+                serverAddressesByBankCode.put(serverAddress.bankCode, mutableListOf(serverAddress))
+            }
+            else {
+                serverAddressesByBankCode[serverAddress.bankCode]!!.add(serverAddress)
+            }
+        }
+
+        return banks.map { mapToBankInfo(it, serverAddressesByBankCode as Map<String, List<ServerAddressesListEntry>>) }
+    }
+
+    private fun mapToBankInfo(bank: BankCodeListEntry,
+                              serverAddressesByBankCode: Map<String, List<ServerAddressesListEntry>>): BankInfo {
+
+        val serverAddress = findServerAddress(bank, serverAddressesByBankCode)
+
+        return BankInfo(
+            bank.bankName,
+            bank.bankCode,
+            bank.bic,
+            bank.postalCode,
+            bank.city,
+            bank.checksumMethod,
+            serverAddress?.pinTanAddress,
+            serverAddress?.pinTanVersion,
+            bank.oldBankCode
+        )
+    }
+
+    private fun findServerAddress(bankCode: BankCodeListEntry,
+                                  serverAddressesByBankCode: Map<String, List<ServerAddressesListEntry>>
+    ): ServerAddressesListEntry? {
+
+        serverAddressesByBankCode[bankCode.bankCode]?.let { serverAddresses ->
+            serverAddresses.firstOrNull { it.city == bankCode.city }?.let {
+                return it
+            }
+
+            return serverAddresses[0]
+        }
+
+        return null
+    }
+
+
+    private fun isListWithFinTsServerAddresses(workSheetData: SheetData, formatter: DataFormatter): Boolean {
+        return hasHeaders(workSheetData, formatter, listOf("BLZ", "PIN/TAN-Zugang URL"))
+    }
+
+    private fun parseListWithFinTsServerAddresses(workSheetData: SheetData, formatter: DataFormatter):
+            List<ServerAddressesListEntry> {
+
+        val entries = mutableListOf<ServerAddressesListEntry>()
+
+        val headerRow = workSheetData.row[0]
+        val headerNames = headerRow.c.map { getCellText(it, formatter) }
+
+        val bankNameColumnIndex = headerNames.indexOf("Institut")
+        val bankCodeColumnIndex = headerNames.indexOf("BLZ")
+        val bicColumnIndex = headerNames.indexOf("BIC")
+        val cityColumnIndex = headerNames.indexOf("Ort")
+        val pinTanAddressColumnIndex = headerNames.indexOf("PIN/TAN-Zugang URL")
+        val pinTanVersionColumnIndex = headerNames.indexOf("Version")
+
+        for (row in workSheetData.row.subList(1, workSheetData.row.size)) {
+            parseToServerAddressesListEntry(row, formatter, bankNameColumnIndex, bankCodeColumnIndex, bicColumnIndex,
+                cityColumnIndex, pinTanAddressColumnIndex, pinTanVersionColumnIndex)?.let { entry ->
+                entries.add(entry)
+            }
+        }
+
+        return entries
+    }
+
+    private fun parseToServerAddressesListEntry(row: Row, formatter: DataFormatter, bankNameColumnIndex: Int,
+                                                bankCodeColumnIndex: Int, bicColumnIndex: Int, cityColumnIndex: Int,
+                                                pinTanAddressColumnIndex: Int, pinTanVersionColumnIndex: Int):
+            ServerAddressesListEntry? {
+
+        try {
+            val bankCode = getCellText(row, bankCodeColumnIndex, formatter)
+
+            if (bankCode.isNotEmpty()) { // filter out empty rows
+
+                return ServerAddressesListEntry(
+                    getCellText(row, bankNameColumnIndex, formatter),
+                    bankCode,
+                    getCellText(row, bicColumnIndex, formatter),
+                    getCellText(row, cityColumnIndex, formatter),
+                    getCellText(row, pinTanAddressColumnIndex, formatter),
+                    getCellText(row, pinTanVersionColumnIndex, formatter)
+                )
+            }
+        } catch (e: Exception) {
+            log.error("Could not parse row ${getRowAsString(row, formatter)} to BankCodeListEntry", e)
+        }
+
+        return null
+    }
+
+
+    private fun isBankCodeList(workSheetData: SheetData, formatter: DataFormatter): Boolean {
+        return hasHeaders(workSheetData, formatter, listOf("Bankleitzahl", "Merkmal"))
+    }
+
+    private fun parseBankCodeList(workSheetData: SheetData, formatter: DataFormatter): List<BankCodeListEntry> {
+        val entries = mutableListOf<BankCodeListEntry>()
+
+        val headerRow = workSheetData.row[0]
+        val headerNames = headerRow.c.map { getCellText(it, formatter) }
+
+        val bankNameColumnIndex = headerNames.indexOf("Bezeichnung")
+        val bankCodeColumnIndex = headerNames.indexOf("Bankleitzahl")
+        val bicColumnIndex = headerNames.indexOf("BIC")
+        val postalCodeColumnIndex = headerNames.indexOf("PLZ")
+        val cityColumnIndex = headerNames.indexOf("Ort")
+        val checksumMethodColumnIndex = headerNames.indexOf("Prüfziffer-berechnungs-methode")
+        val bankCodeDeletedColumnIndex = headerNames.indexOf("Bankleitzahl-löschung")
+        val newBankCodeColumnIndex = headerNames.indexOf("Nachfolge-Bankleitzahl")
+
+        var lastParsedEntry: BankCodeListEntry? = null
+
+        for (row in workSheetData.row.subList(1, workSheetData.row.size)) {
+            parseToBankCodeListEntry(row, formatter, bankNameColumnIndex, bankCodeColumnIndex, bicColumnIndex,
+                postalCodeColumnIndex, cityColumnIndex, checksumMethodColumnIndex, bankCodeDeletedColumnIndex,
+                newBankCodeColumnIndex)?.let { entry ->
+                // if the following banks have the same BIC, the BIC is only given for the first bank -> get BIC from previous bank
+                if (entry.bic.isEmpty() &&
+                        (entry.bankCode == lastParsedEntry?.bankCode || entry.bankCode == lastParsedEntry?.oldBankCode)) {
+                    entry.bic = lastParsedEntry?.bic!!
+                }
+
+                entries.add(entry)
+
+                lastParsedEntry = entry
+            }
+        }
+
+        updateDeletedBanks(entries)
+
+        return entries
+    }
+
+    private fun parseToBankCodeListEntry(row: Row, formatter: DataFormatter, bankNameColumnIndex: Int,
+                                         bankCodeColumnIndex: Int, bicColumnIndex: Int, postalCodeColumnIndex: Int,
+                                         cityColumnIndex: Int, checksumMethodColumnIndex: Int,
+                                         bankCodeDeletedColumnIndex: Int, newBankCodeColumnIndex: Int): BankCodeListEntry? {
+
+        try {
+            val bankCode = getCellText(row, bankCodeColumnIndex, formatter)
+
+            if (bankCode.isNotEmpty()) { // filter out empty rows
+                var newBankCode: String? = null
+                val isBankCodeDeleted = getCellText(row, bankCodeDeletedColumnIndex, formatter) == "1"
+                val newBankCodeCellText = getCellText(row, newBankCodeColumnIndex, formatter)
+                if (isBankCodeDeleted && newBankCodeCellText.isNotEmpty() && newBankCodeCellText != "00000000") {
+                    newBankCode = newBankCodeCellText
+                }
+
+                return BankCodeListEntry(
+                    getCellText(row, bankNameColumnIndex, formatter),
+                    newBankCode ?: bankCode,
+                    getCellText(row, bicColumnIndex, formatter),
+                    getCellText(row, postalCodeColumnIndex, formatter),
+                    getCellText(row, cityColumnIndex, formatter),
+                    getCellText(row, checksumMethodColumnIndex, formatter),
+                    if (newBankCode != null) bankCode else newBankCode
+                )
+            }
+        } catch (e: Exception) {
+            log.error("Could not parse row ${getRowAsString(row, formatter)} to BankCodeListEntry", e)
+        }
+
+        return null
+    }
+
+    /**
+     * Deleted banks may not have a BIC. This method fixes this
+     */
+    private fun updateDeletedBanks(banks: MutableList<BankCodeListEntry>) {
+        val banksByCode = banks.associateBy { it.bankCode }
+
+        val deletedBanks = banks.filter { it.isBankCodeDeleted }
+
+        for (deletedBank in deletedBanks) {
+            banksByCode[deletedBank.bankCode]?.let { newBank ->
+                deletedBank.bic = newBank.bic
+            }
+        }
+    }
+
+
+    private fun hasHeaders(workSheetData: SheetData, formatter: DataFormatter, headerNames: List<String>): Boolean {
+        if (workSheetData.row.isNotEmpty()) {
+            val headerRow = workSheetData.row[0]
+
+            val rowHeaderNames = headerRow.c.map { getCellText(it, formatter) }
+
+            return rowHeaderNames.containsAll(headerNames)
+        }
+
+        return false
+    }
+
+    private fun getCellText(row: Row, columnIndex: Int, formatter: DataFormatter): String {
+        return getCellText(row.c[columnIndex], formatter)
+    }
+
+    private fun getCellText(cell: Cell, formatter: DataFormatter): String {
+        if (cell.f != null) { // cell with formular
+            return cell.v
+        }
+        return formatter.formatCellValue(cell)
+    }
+
+    private fun getRowAsString(row: Row, formatter: DataFormatter): String {
+        return row.c.joinToString("\t|", "|\t", "\t|") { getCellText(it, formatter) }
+    }
+
+}

BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/BankCodeListEntry.kt

@@ -0,0 +1,22 @@
+package net.dankito.banking.banklistcreator.parser.model
+
+
+open class BankCodeListEntry(
+    val bankName: String,
+    val bankCode: String,
+    var bic: String, // TODO: make val again
+    val postalCode: String,
+    val city: String,
+    val checksumMethod: String,
+    val oldBankCode: String?
+) {
+
+    val isBankCodeDeleted: Boolean
+        get() = oldBankCode != null
+
+
+    override fun toString(): String {
+        return "$bankCode $bankName ($bic, $city)"
+    }
+
+}

BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/ServerAddressesListEntry.kt

@@ -0,0 +1,17 @@
+package net.dankito.banking.banklistcreator.parser.model
+
+
+open class ServerAddressesListEntry(
+    val bankName: String,
+    val bankCode: String,
+    val bic: String,
+    val city: String,
+    val pinTanAddress: String?,
+    val pinTanVersion: String?
+) {
+
+    override fun toString(): String {
+        return "$bankCode $bankName ($city, $pinTanAddress)"
+    }
+
+}

BankListCreator/src/test/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParserTest.kt

@@ -0,0 +1,35 @@
+package net.dankito.banking.banklistcreator.parser
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Ignore
+import org.junit.Test
+import java.io.File
+
+
+@Ignore
+class DeutscheKreditwirtschaftBankListParserTest {
+
+    private val underTest = DeutscheKreditwirtschaftBankListParser()
+
+
+    @Test
+    fun parse() {
+
+        // when
+        // TODO: set path to bank list file from Deutsche Kreditwirtschaft here
+        val result = underTest.parse(File(""))
+
+        // then
+        assertThat(result).hasSize(16282)
+
+        result.forEach { bankInfo ->
+            assertThat(bankInfo.name).isNotEmpty()
+            assertThat(bankInfo.bankCode).isNotEmpty()
+//            assertThat(bankInfo.bic).isNotEmpty() // TODO: is there a way to find BICs for all banks?
+            assertThat(bankInfo.postalCode).isNotEmpty()
+            assertThat(bankInfo.city).isNotEmpty()
+            assertThat(bankInfo.checksumMethod).isNotEmpty()
+        }
+    }
+
+}

BankListCreator/src/test/resources/logback-test.xml

@@ -0,0 +1,24 @@
+<configuration>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>DEBUG</level>
+        </filter>
+    </appender>
+
+    <!-- Insert the current time formatted as "yyyyMMdd'T'HHmmss" under
+         the key "bySecond" into the logger context. This value will be
+         available to all subsequent configuration elements. -->
+    <timestamp key="bySecond" datePattern="yyyyMMdd'T'HHmmss"/>
+
+    <!-- Raise log level here if you don't want to see noisy log output -->
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+</configuration>

fints4javaLib/build.gradle

@@ -15,7 +15,9 @@ compileTestKotlin {
 
 
 dependencies {
-    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
+    api "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
+
+    api "net.dankito.utils:java-utils:$javaUtilsVersion"
 
     compile "net.dankito.utils:java-utils:$javaUtilsVersion"
 

fints4javaLib/src/main/kotlin/net/dankito/fints/model/BankInfo.kt

@@ -0,0 +1,33 @@
+package net.dankito.fints.model
+
+
+open class BankInfo(
+    val name: String,
+    val bankCode: String,
+    val bic: String,
+    val postalCode: String,
+    val city: String,
+    val checksumMethod: String,
+    val pinTanAddress: String?,
+    val pinTanVersion: String?,
+    val oldBankCode: String?
+) {
+
+
+    protected constructor() : this("", "", "", "", "", "", null, null, null) // for object deserializers
+
+    val supportsPinTan: Boolean
+        get() = pinTanAddress.isNullOrEmpty() == false
+
+    val supportsFinTs3_0: Boolean
+        get() = pinTanVersion == "FinTS V3.0"
+
+    val isBankCodeDeleted: Boolean
+        get() = oldBankCode != null // TODO: this is not in all cases true, there are banks with new bank code which haven't been deleted
+
+
+    override fun toString(): String {
+        return "$bankCode $name $city"
+    }
+
+}

settings.gradle

@@ -1,3 +1,5 @@
 rootProject.name = 'fints4java'
 
 include ':fints4javaLib'
+
+include ':BankListCreator'