From a07b6b115e64509ba288b8d046920dc6a9061383 Mon Sep 17 00:00:00 2001 From: dankl Date: Sat, 12 Oct 2019 20:54:02 +0200 Subject: [PATCH] Implemented BankListCreator to parse German banks file from Deutsche Kreditwirtschaft --- BankListCreator/build.gradle | 27 ++ .../banklistcreator/BankListCreator.kt | 20 ++ .../DeutscheKreditwirtschaftBankListParser.kt | 276 ++++++++++++++++++ .../parser/model/BankCodeListEntry.kt | 22 ++ .../parser/model/ServerAddressesListEntry.kt | 17 ++ ...tscheKreditwirtschaftBankListParserTest.kt | 35 +++ .../src/test/resources/logback-test.xml | 24 ++ fints4javaLib/build.gradle | 4 +- .../net/dankito/fints/model/BankInfo.kt | 33 +++ settings.gradle | 4 +- 10 files changed, 460 insertions(+), 2 deletions(-) create mode 100644 BankListCreator/build.gradle create mode 100644 BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/BankListCreator.kt create mode 100644 BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParser.kt create mode 100644 BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/BankCodeListEntry.kt create mode 100644 BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/ServerAddressesListEntry.kt create mode 100644 BankListCreator/src/test/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParserTest.kt create mode 100755 BankListCreator/src/test/resources/logback-test.xml create mode 100644 fints4javaLib/src/main/kotlin/net/dankito/fints/model/BankInfo.kt diff --git a/BankListCreator/build.gradle b/BankListCreator/build.gradle new file mode 100644 index 00000000..ec99c8a7 --- /dev/null +++ b/BankListCreator/build.gradle @@ -0,0 +1,27 @@ +apply plugin: 'java-library' +apply plugin: 'kotlin' + + +sourceCompatibility = 1.8 + +compileKotlin { + kotlinOptions.jvmTarget = "1.8" +} +compileTestKotlin { + kotlinOptions.jvmTarget = "1.8" +} + + + +dependencies { + compile project(':fints4javaLib') + + implementation 'org.docx4j:docx4j-JAXB-ReferenceImpl:8.1.3' + + + testCompile "junit:junit:$junitVersion" + testCompile "org.assertj:assertj-core:$assertJVersion" + + testCompile "ch.qos.logback:logback-core:$logbackVersion" + testCompile "ch.qos.logback:logback-classic:$logbackVersion" +} \ No newline at end of file diff --git a/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/BankListCreator.kt b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/BankListCreator.kt new file mode 100644 index 00000000..684b1c70 --- /dev/null +++ b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/BankListCreator.kt @@ -0,0 +1,20 @@ +package net.dankito.banking.banklistcreator + +import net.dankito.banking.banklistcreator.parser.DeutscheKreditwirtschaftBankListParser +import net.dankito.utils.serialization.JacksonJsonSerializer +import java.io.File + + +open class BankListCreator @JvmOverloads constructor( + protected val parser: DeutscheKreditwirtschaftBankListParser = DeutscheKreditwirtschaftBankListParser() +) { + + fun createBankListFromDeutscheKreditwirtschaftXlsxFile(bankFileOutputFile: File, + deutscheKreditwirtschaftXlsxFile: File) { + + val banks = parser.parse(deutscheKreditwirtschaftXlsxFile) + + JacksonJsonSerializer().serializeObject(banks, bankFileOutputFile) + } + +} \ No newline at end of file diff --git a/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParser.kt b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParser.kt new file mode 100644 index 00000000..a1703e14 --- /dev/null +++ b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParser.kt @@ -0,0 +1,276 @@ +package net.dankito.banking.banklistcreator.parser + +import net.dankito.banking.banklistcreator.parser.model.BankCodeListEntry +import net.dankito.banking.banklistcreator.parser.model.ServerAddressesListEntry +import net.dankito.fints.model.BankInfo +import org.docx4j.openpackaging.packages.SpreadsheetMLPackage +import org.slf4j.LoggerFactory +import org.xlsx4j.org.apache.poi.ss.usermodel.DataFormatter +import org.xlsx4j.sml.Cell +import org.xlsx4j.sml.Row +import org.xlsx4j.sml.SheetData +import java.io.File + + +/** + * Parses the list of German banks from Deutsche Kreditwirtschaft you can retrieve by registering here: + * https://www.hbci-zka.de/register/hersteller.htm + */ +open class DeutscheKreditwirtschaftBankListParser { + + companion object { + private val log = LoggerFactory.getLogger(DeutscheKreditwirtschaftBankListParser::class.java) + } + + + fun parse(bankListFile: File): List { + val xlsxPkg = SpreadsheetMLPackage.load(bankListFile) + + val workbookPart = xlsxPkg.getWorkbookPart() + val sheets = workbookPart.contents.sheets.sheet + val formatter = DataFormatter() + + var serverAddressesList = listOf() + var bankCodesList = listOf() + + for (index in 0 until sheets.size) { + log.info("\nParsing sheet ${sheets[index].name}:\n") + val sheet = workbookPart.getWorksheet(index) + val workSheetData = sheet.contents.sheetData + + if (isListWithFinTsServerAddresses(workSheetData, formatter)) { + serverAddressesList = parseListWithFinTsServerAddresses(workSheetData, formatter) + } + else if (isBankCodeList(workSheetData, formatter)) { + bankCodesList = parseBankCodeList(workSheetData, formatter) + } + } + + return mapBankCodeAndServerAddressesList(bankCodesList, serverAddressesList) + } + + private fun mapBankCodeAndServerAddressesList(banks: List, + serverAddresses: List): List { + + val serverAddressesByBankCode = mutableMapOf>() + serverAddresses.forEach { serverAddress -> + if (serverAddressesByBankCode.containsKey(serverAddress.bankCode) == false) { + serverAddressesByBankCode.put(serverAddress.bankCode, mutableListOf(serverAddress)) + } + else { + serverAddressesByBankCode[serverAddress.bankCode]!!.add(serverAddress) + } + } + + return banks.map { mapToBankInfo(it, serverAddressesByBankCode as Map>) } + } + + private fun mapToBankInfo(bank: BankCodeListEntry, + serverAddressesByBankCode: Map>): BankInfo { + + val serverAddress = findServerAddress(bank, serverAddressesByBankCode) + + return BankInfo( + bank.bankName, + bank.bankCode, + bank.bic, + bank.postalCode, + bank.city, + bank.checksumMethod, + serverAddress?.pinTanAddress, + serverAddress?.pinTanVersion, + bank.oldBankCode + ) + } + + private fun findServerAddress(bankCode: BankCodeListEntry, + serverAddressesByBankCode: Map> + ): ServerAddressesListEntry? { + + serverAddressesByBankCode[bankCode.bankCode]?.let { serverAddresses -> + serverAddresses.firstOrNull { it.city == bankCode.city }?.let { + return it + } + + return serverAddresses[0] + } + + return null + } + + + private fun isListWithFinTsServerAddresses(workSheetData: SheetData, formatter: DataFormatter): Boolean { + return hasHeaders(workSheetData, formatter, listOf("BLZ", "PIN/TAN-Zugang URL")) + } + + private fun parseListWithFinTsServerAddresses(workSheetData: SheetData, formatter: DataFormatter): + List { + + val entries = mutableListOf() + + val headerRow = workSheetData.row[0] + val headerNames = headerRow.c.map { getCellText(it, formatter) } + + val bankNameColumnIndex = headerNames.indexOf("Institut") + val bankCodeColumnIndex = headerNames.indexOf("BLZ") + val bicColumnIndex = headerNames.indexOf("BIC") + val cityColumnIndex = headerNames.indexOf("Ort") + val pinTanAddressColumnIndex = headerNames.indexOf("PIN/TAN-Zugang URL") + val pinTanVersionColumnIndex = headerNames.indexOf("Version") + + for (row in workSheetData.row.subList(1, workSheetData.row.size)) { + parseToServerAddressesListEntry(row, formatter, bankNameColumnIndex, bankCodeColumnIndex, bicColumnIndex, + cityColumnIndex, pinTanAddressColumnIndex, pinTanVersionColumnIndex)?.let { entry -> + entries.add(entry) + } + } + + return entries + } + + private fun parseToServerAddressesListEntry(row: Row, formatter: DataFormatter, bankNameColumnIndex: Int, + bankCodeColumnIndex: Int, bicColumnIndex: Int, cityColumnIndex: Int, + pinTanAddressColumnIndex: Int, pinTanVersionColumnIndex: Int): + ServerAddressesListEntry? { + + try { + val bankCode = getCellText(row, bankCodeColumnIndex, formatter) + + if (bankCode.isNotEmpty()) { // filter out empty rows + + return ServerAddressesListEntry( + getCellText(row, bankNameColumnIndex, formatter), + bankCode, + getCellText(row, bicColumnIndex, formatter), + getCellText(row, cityColumnIndex, formatter), + getCellText(row, pinTanAddressColumnIndex, formatter), + getCellText(row, pinTanVersionColumnIndex, formatter) + ) + } + } catch (e: Exception) { + log.error("Could not parse row ${getRowAsString(row, formatter)} to BankCodeListEntry", e) + } + + return null + } + + + private fun isBankCodeList(workSheetData: SheetData, formatter: DataFormatter): Boolean { + return hasHeaders(workSheetData, formatter, listOf("Bankleitzahl", "Merkmal")) + } + + private fun parseBankCodeList(workSheetData: SheetData, formatter: DataFormatter): List { + val entries = mutableListOf() + + val headerRow = workSheetData.row[0] + val headerNames = headerRow.c.map { getCellText(it, formatter) } + + val bankNameColumnIndex = headerNames.indexOf("Bezeichnung") + val bankCodeColumnIndex = headerNames.indexOf("Bankleitzahl") + val bicColumnIndex = headerNames.indexOf("BIC") + val postalCodeColumnIndex = headerNames.indexOf("PLZ") + val cityColumnIndex = headerNames.indexOf("Ort") + val checksumMethodColumnIndex = headerNames.indexOf("Prüfziffer-berechnungs-methode") + val bankCodeDeletedColumnIndex = headerNames.indexOf("Bankleitzahl-löschung") + val newBankCodeColumnIndex = headerNames.indexOf("Nachfolge-Bankleitzahl") + + var lastParsedEntry: BankCodeListEntry? = null + + for (row in workSheetData.row.subList(1, workSheetData.row.size)) { + parseToBankCodeListEntry(row, formatter, bankNameColumnIndex, bankCodeColumnIndex, bicColumnIndex, + postalCodeColumnIndex, cityColumnIndex, checksumMethodColumnIndex, bankCodeDeletedColumnIndex, + newBankCodeColumnIndex)?.let { entry -> + // if the following banks have the same BIC, the BIC is only given for the first bank -> get BIC from previous bank + if (entry.bic.isEmpty() && + (entry.bankCode == lastParsedEntry?.bankCode || entry.bankCode == lastParsedEntry?.oldBankCode)) { + entry.bic = lastParsedEntry?.bic!! + } + + entries.add(entry) + + lastParsedEntry = entry + } + } + + updateDeletedBanks(entries) + + return entries + } + + private fun parseToBankCodeListEntry(row: Row, formatter: DataFormatter, bankNameColumnIndex: Int, + bankCodeColumnIndex: Int, bicColumnIndex: Int, postalCodeColumnIndex: Int, + cityColumnIndex: Int, checksumMethodColumnIndex: Int, + bankCodeDeletedColumnIndex: Int, newBankCodeColumnIndex: Int): BankCodeListEntry? { + + try { + val bankCode = getCellText(row, bankCodeColumnIndex, formatter) + + if (bankCode.isNotEmpty()) { // filter out empty rows + var newBankCode: String? = null + val isBankCodeDeleted = getCellText(row, bankCodeDeletedColumnIndex, formatter) == "1" + val newBankCodeCellText = getCellText(row, newBankCodeColumnIndex, formatter) + if (isBankCodeDeleted && newBankCodeCellText.isNotEmpty() && newBankCodeCellText != "00000000") { + newBankCode = newBankCodeCellText + } + + return BankCodeListEntry( + getCellText(row, bankNameColumnIndex, formatter), + newBankCode ?: bankCode, + getCellText(row, bicColumnIndex, formatter), + getCellText(row, postalCodeColumnIndex, formatter), + getCellText(row, cityColumnIndex, formatter), + getCellText(row, checksumMethodColumnIndex, formatter), + if (newBankCode != null) bankCode else newBankCode + ) + } + } catch (e: Exception) { + log.error("Could not parse row ${getRowAsString(row, formatter)} to BankCodeListEntry", e) + } + + return null + } + + /** + * Deleted banks may not have a BIC. This method fixes this + */ + private fun updateDeletedBanks(banks: MutableList) { + val banksByCode = banks.associateBy { it.bankCode } + + val deletedBanks = banks.filter { it.isBankCodeDeleted } + + for (deletedBank in deletedBanks) { + banksByCode[deletedBank.bankCode]?.let { newBank -> + deletedBank.bic = newBank.bic + } + } + } + + + private fun hasHeaders(workSheetData: SheetData, formatter: DataFormatter, headerNames: List): Boolean { + if (workSheetData.row.isNotEmpty()) { + val headerRow = workSheetData.row[0] + + val rowHeaderNames = headerRow.c.map { getCellText(it, formatter) } + + return rowHeaderNames.containsAll(headerNames) + } + + return false + } + + private fun getCellText(row: Row, columnIndex: Int, formatter: DataFormatter): String { + return getCellText(row.c[columnIndex], formatter) + } + + private fun getCellText(cell: Cell, formatter: DataFormatter): String { + if (cell.f != null) { // cell with formular + return cell.v + } + return formatter.formatCellValue(cell) + } + + private fun getRowAsString(row: Row, formatter: DataFormatter): String { + return row.c.joinToString("\t|", "|\t", "\t|") { getCellText(it, formatter) } + } + +} \ No newline at end of file diff --git a/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/BankCodeListEntry.kt b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/BankCodeListEntry.kt new file mode 100644 index 00000000..11b39616 --- /dev/null +++ b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/BankCodeListEntry.kt @@ -0,0 +1,22 @@ +package net.dankito.banking.banklistcreator.parser.model + + +open class BankCodeListEntry( + val bankName: String, + val bankCode: String, + var bic: String, // TODO: make val again + val postalCode: String, + val city: String, + val checksumMethod: String, + val oldBankCode: String? +) { + + val isBankCodeDeleted: Boolean + get() = oldBankCode != null + + + override fun toString(): String { + return "$bankCode $bankName ($bic, $city)" + } + +} \ No newline at end of file diff --git a/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/ServerAddressesListEntry.kt b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/ServerAddressesListEntry.kt new file mode 100644 index 00000000..5c7c47fa --- /dev/null +++ b/BankListCreator/src/main/kotlin/net/dankito/banking/banklistcreator/parser/model/ServerAddressesListEntry.kt @@ -0,0 +1,17 @@ +package net.dankito.banking.banklistcreator.parser.model + + +open class ServerAddressesListEntry( + val bankName: String, + val bankCode: String, + val bic: String, + val city: String, + val pinTanAddress: String?, + val pinTanVersion: String? +) { + + override fun toString(): String { + return "$bankCode $bankName ($city, $pinTanAddress)" + } + +} \ No newline at end of file diff --git a/BankListCreator/src/test/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParserTest.kt b/BankListCreator/src/test/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParserTest.kt new file mode 100644 index 00000000..2d05a1dd --- /dev/null +++ b/BankListCreator/src/test/kotlin/net/dankito/banking/banklistcreator/parser/DeutscheKreditwirtschaftBankListParserTest.kt @@ -0,0 +1,35 @@ +package net.dankito.banking.banklistcreator.parser + +import org.assertj.core.api.Assertions.assertThat +import org.junit.Ignore +import org.junit.Test +import java.io.File + + +@Ignore +class DeutscheKreditwirtschaftBankListParserTest { + + private val underTest = DeutscheKreditwirtschaftBankListParser() + + + @Test + fun parse() { + + // when + // TODO: set path to bank list file from Deutsche Kreditwirtschaft here + val result = underTest.parse(File("")) + + // then + assertThat(result).hasSize(16282) + + result.forEach { bankInfo -> + assertThat(bankInfo.name).isNotEmpty() + assertThat(bankInfo.bankCode).isNotEmpty() +// assertThat(bankInfo.bic).isNotEmpty() // TODO: is there a way to find BICs for all banks? + assertThat(bankInfo.postalCode).isNotEmpty() + assertThat(bankInfo.city).isNotEmpty() + assertThat(bankInfo.checksumMethod).isNotEmpty() + } + } + +} \ No newline at end of file diff --git a/BankListCreator/src/test/resources/logback-test.xml b/BankListCreator/src/test/resources/logback-test.xml new file mode 100755 index 00000000..aaa0bc32 --- /dev/null +++ b/BankListCreator/src/test/resources/logback-test.xml @@ -0,0 +1,24 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + DEBUG + + + + + + + + + + + \ No newline at end of file diff --git a/fints4javaLib/build.gradle b/fints4javaLib/build.gradle index 443736ac..2e4b4e68 100644 --- a/fints4javaLib/build.gradle +++ b/fints4javaLib/build.gradle @@ -15,7 +15,9 @@ compileTestKotlin { dependencies { - compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion" + api "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion" + + api "net.dankito.utils:java-utils:$javaUtilsVersion" compile "net.dankito.utils:java-utils:$javaUtilsVersion" diff --git a/fints4javaLib/src/main/kotlin/net/dankito/fints/model/BankInfo.kt b/fints4javaLib/src/main/kotlin/net/dankito/fints/model/BankInfo.kt new file mode 100644 index 00000000..f8ad0c6e --- /dev/null +++ b/fints4javaLib/src/main/kotlin/net/dankito/fints/model/BankInfo.kt @@ -0,0 +1,33 @@ +package net.dankito.fints.model + + +open class BankInfo( + val name: String, + val bankCode: String, + val bic: String, + val postalCode: String, + val city: String, + val checksumMethod: String, + val pinTanAddress: String?, + val pinTanVersion: String?, + val oldBankCode: String? +) { + + + protected constructor() : this("", "", "", "", "", "", null, null, null) // for object deserializers + + val supportsPinTan: Boolean + get() = pinTanAddress.isNullOrEmpty() == false + + val supportsFinTs3_0: Boolean + get() = pinTanVersion == "FinTS V3.0" + + val isBankCodeDeleted: Boolean + get() = oldBankCode != null // TODO: this is not in all cases true, there are banks with new bank code which haven't been deleted + + + override fun toString(): String { + return "$bankCode $name $city" + } + +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index cd9dc977..310bf533 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,5 @@ rootProject.name = 'fints4java' -include ':fints4javaLib' \ No newline at end of file +include ':fints4javaLib' + +include ':BankListCreator' \ No newline at end of file