Implemented BanksFinTsDetailsRetriever to retrieve banks FinTS information from all banks in bank list from Deutsche Kreditwirtschaft

docs/GetBankInfo.md

@@ -1,21 +1,21 @@
 
 - Welche Geschäftsvorfälle in welchen Versionen werden unterstützt?
-- Welche pain Nachrichten werden unterstützt (unterstützen alle "pain.001.001.03")?
+- :heavy_check_mark:Welche pain Nachrichten werden unterstützt (unterstützen alle "pain.001.001.03")?
-- Welche TAN Arten werden unterstützt? Wie heißen diese?
+- :heavy_check_mark:Welche TAN Arten werden unterstützt? Wie heißen diese?
 - Ist BIC je abweichend von der der Bankparameterliste? (wohl nur mit einem echten Nutzer herausfindbar)
 - Ist BLZ je abweichend von der der Bankparameterliste?
 - UPD Version
-- HHD 1.3 oder HHD 1.4?
+- :heavy_check_mark:HHD 1.3 oder HHD 1.4? -> Die allermeisten unterstützen HHD 1.4, nur noch knapp 500 HHD 1.3
-- Wird MT940 unterstützt oder nur CAMT?
+- :heavy_check_mark:Wird MT940 unterstützt oder nur CAMT? -> Alle unterstützen HKKAZ, ganz wenige jedoch nur in Version 4
-- Hat irgendeine Bank Geschäftsvorfallparameter zu HKVVB, HKIDN oder HKSYN?
+- :heavy_check_mark:Hat irgendeine Bank Geschäftsvorfallparameter zu HKVVB, HKIDN oder HKSYN? -> Bei allen drei: Nein
-- Unterstützen alle HKTAN in Version 6?
+- :heavy_check_mark:Unterstützen alle HKTAN in Version 6? -> Ja, fast alle.
-- Unterstützen alle HKSAL in Version 5?
+- :heavy_check_mark:Unterstützen alle HKSAL in Version 5? -> Nein, viele unterstützen nur HKSAL 7 (und 4)
-- Unterstützen Banken HKSAL in Versionen größer oder kleiner 5?
+- :heavy_check_mark:Unterstützen Banken HKSAL in Versionen größer oder kleiner 5?
-- Unterstützen alle HKKAZ in Version 5-7?
+- :heavy_check_mark:Unterstützen alle HKKAZ in Version 5-7? -> Ja, nur ganz, ganz wenige HKKAZ 4
-- Unterstützen Banken HKKAZ in Versionen größer oder kleiner 5-7?
+- :heavy_check_mark:Unterstützen Banken HKKAZ in Versionen größer oder kleiner 5-7?
-- Unterstützen alle HKCCS in Version 1?
+- :heavy_check_mark:Unterstützen alle HKCCS in Version 1? -> Ja, fast alle.
-- Unterstützen Banken HKCCS in Versionen größer 1?
+- :heavy_check_mark:Unterstützen Banken HKCCS in Versionen größer 1? -> Nein.
-- Erlaubt es irgendeine Bank HKTAN abzubrechen?
+- :heavy_check_mark:Erlaubt es irgendeine Bank HKTAN abzubrechen? -> Ja, aber nur ganz, ganz wenige (32) und ziemlich unbekannte
-- Hat maxAllowedJobs je einen Wert > 0?
+- :heavy_check_mark:Hat maxAllowedJobs je einen Wert > 0? -> Ja, bei 439
-- Unterstützte Sprachen
+- :heavy_check_mark:Unterstützte Sprachen -> Gibt auch welche mit Englisch oder Französisch
-- Standardsprache
+- :heavy_check_mark:Standardsprache -> Ausschließlich Deutsch

fints4javaLib/build.gradle

@@ -9,7 +9,7 @@ compileKotlin {
     kotlinOptions.jvmTarget = "1.6"
 }
 compileTestKotlin {
-    kotlinOptions.jvmTarget = "1.6"
+    kotlinOptions.jvmTarget = "1.8"
 }
 
 
@@ -22,15 +22,18 @@ dependencies {
     implementation "net.dankito.search:lucene-4-utils:$luceneUtilsVersion"
 
 
-    testCompile "junit:junit:$junitVersion"
+    testImplementation "junit:junit:$junitVersion"
-    testCompile "org.assertj:assertj-core:$assertJVersion"
+    testImplementation "org.assertj:assertj-core:$assertJVersion"
 
-    testCompile "org.mockito:mockito-core:$mockitoVersion"
+    testImplementation "org.mockito:mockito-core:$mockitoVersion"
-    testCompile "com.nhaarman:mockito-kotlin:$mockitoKotlinVersion" // so that Mockito.any() doesn't return null which null-safe Kotlin parameter don't like
+    testImplementation "com.nhaarman:mockito-kotlin:$mockitoKotlinVersion" // so that Mockito.any() doesn't return null which null-safe Kotlin parameter don't like
     // for how to enable mocking final class (which is standard in Kotlin) with Mockito see https://github.com/mockito/mockito/wiki/What's-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods
 
-    testCompile "ch.qos.logback:logback-core:$logbackVersion"
+    testImplementation "ch.qos.logback:logback-classic:$logbackVersion"
-    testCompile "ch.qos.logback:logback-classic:$logbackVersion"
 
-    testCompile "net.dankito.utils:java-fx-utils:$javaFxUtilsVersion"
+    testImplementation("net.dankito.utils:java-fx-utils:$javaFxUtilsVersion") {
+        exclude group: "org.controlsfx"
+    }
+
+    testImplementation "org.apache.commons:commons-csv:1.8"
 }

fints4javaLib/src/test/kotlin/net/dankito/fints/bankdetails/BanksFinTsDetailsRetriever.kt

@@ -0,0 +1,246 @@
+package net.dankito.fints.bankdetails
+
+import net.dankito.fints.FinTsClient
+import net.dankito.fints.banks.InMemoryBankFinder
+import net.dankito.fints.callback.NoOpFinTsClientCallback
+import net.dankito.fints.messages.MessageBuilder
+import net.dankito.fints.messages.Separators
+import net.dankito.fints.messages.datenelemente.implementierte.Dialogsprache
+import net.dankito.fints.model.BankData
+import net.dankito.fints.model.BankInfo
+import net.dankito.fints.model.DialogData
+import net.dankito.fints.model.ProductData
+import net.dankito.fints.model.mapper.BankDataMapper
+import net.dankito.fints.response.Response
+import net.dankito.fints.response.ResponseParser
+import net.dankito.fints.response.segments.SepaAccountInfoParameters
+import net.dankito.fints.response.segments.TanInfo
+import net.dankito.fints.response.segments.TanProcedureParameters
+import net.dankito.fints.util.Java8Base64Service
+import org.apache.commons.csv.CSVFormat
+import org.apache.commons.csv.CSVPrinter
+import org.junit.Ignore
+import org.junit.Test
+import org.slf4j.LoggerFactory
+import java.io.File
+import java.io.FileWriter
+import java.text.SimpleDateFormat
+import java.util.*
+
+
+@Ignore // not a real test, run manually to retrieve FinTS information from all banks
+class BanksFinTsDetailsRetriever {
+
+    companion object {
+        private val OutputFolderDateFormat = SimpleDateFormat("yyyy_MM_dd_HH_mm_ss")
+
+        private val log = LoggerFactory.getLogger(BanksFinTsDetailsRetriever::class.java)
+    }
+
+
+    private val bankFinder = InMemoryBankFinder()
+
+    private val bankDataMapper = BankDataMapper()
+
+    private val product: ProductData = ProductData("15E53C26816138699C7B6A3E8", "0.1") // TODO: get version dynamically
+
+    private val messageBuilder = MessageBuilder()
+
+    private val finTsClient = object : FinTsClient(NoOpFinTsClientCallback(), Java8Base64Service()) {
+
+        fun getAndHandleResponseForMessagePublic(requestBody: String, bank: BankData): Response {
+            return getAndHandleResponseForMessage(requestBody, bank)
+        }
+
+        fun updateBankDataPublic(bank: BankData, response: Response) {
+            super.updateBankData(bank, response)
+        }
+    }
+
+
+    private val requestNotSuccessful = mutableListOf<BankInfo>()
+
+    private val tanProcedureParameter = mutableMapOf<String, MutableSet<TanProcedureParameters>>()
+
+    private val doesNotSupportHKTAN6 = mutableListOf<BankInfo>()
+    private val doesNotSupportHKSAL5or7 = mutableListOf<BankInfo>()
+    private val doesNotSupportHKKAZ5to7 = mutableListOf<BankInfo>()
+    private val doesNotSupportHKCCS1 = mutableListOf<BankInfo>()
+
+    private val supportsHhd13 = mutableListOf<BankInfo>()
+    private val supportsHhd14 = mutableListOf<BankInfo>()
+
+    private val doesNotSupportPain_001_001_or_003_03 = mutableListOf<BankInfo>()
+
+
+    @Test
+    fun retrieveAllBanksFinTsDetails() {
+
+        val allBanks = bankFinder.getBankList()
+        val banksSupportingFinTs = allBanks.filter { it.supportsFinTs3_0 }
+
+        val outputFolder = File("bankData", OutputFolderDateFormat.format(Date()))
+        val responsesFolder = File(outputFolder, "responses")
+        responsesFolder.mkdirs()
+
+        val csvFile = FileWriter(File(outputFolder, "bank_details.csv"))
+        val csvPrinter = CSVPrinter(csvFile, CSVFormat.DEFAULT.withHeader(
+            "BLZ", "Name", "Ort", "BPD", "Tanverfahren", "Technische Tanverfahrennamen", "HHD 1.3?", "HHD 1.4?",
+            "HKTAN 6?", "HKTAN", "HKSAL 5?", "HKSAL", "HKKAZ 5-7?", "HKKAZ", "HKCAZ", "HKCCS 1?", "HKCCS",
+            "pain.001.001.03?", "SEPA Formate", "Sprachen", "Untersstützte Geschäftsvorfälle"
+        ))
+
+
+        val uniqueBanks = banksSupportingFinTs.associateBy { "${it.bankCode}_${it.name}" }
+        var bankIndex = 0
+
+        uniqueBanks.forEach { bankName, bankInfo ->
+            log.info("[${++bankIndex}] Getting details for $bankName ...")
+
+            getAndSaveBankDetails(bankInfo, responsesFolder, csvPrinter)
+        }
+
+        printStatistics()
+
+        csvPrinter.close()
+        csvFile.close()
+    }
+
+
+    private fun getAnonymousBankInfo(bank: BankData): Response {
+        val dialogData = DialogData()
+        val requestBody = messageBuilder.createAnonymousDialogInitMessage(bank, product, dialogData)
+
+        val anonymousBankInfoResponse =
+            finTsClient.getAndHandleResponseForMessagePublic(requestBody, bank)
+
+        finTsClient.updateBankDataPublic(bank, anonymousBankInfoResponse)
+        return anonymousBankInfoResponse
+    }
+
+    private fun getAndSaveBankDetails(bankInfo: BankInfo, responsesFolder: File, csvPrinter: CSVPrinter) {
+        val bank = bankDataMapper.mapFromBankInfo(bankInfo)
+
+        val anonymousBankInfoResponse = getAnonymousBankInfo(bank)
+
+        File(responsesFolder, "${bankInfo.bankCode}_${bankInfo.name.replace('/', '-')}").writeText(
+            anonymousBankInfoResponse.receivedSegments.joinToString(System.lineSeparator()) { it.segmentString + Separators.SegmentSeparator })
+
+        if (anonymousBankInfoResponse.successful == false) {
+            requestNotSuccessful.add(bankInfo)
+            log.warn("Did not receive response from bank $bankInfo: ${anonymousBankInfoResponse.receivedSegments.joinToString(System.lineSeparator()) { it.segmentString + Separators.SegmentSeparator }}")
+
+            return
+        }
+
+
+        val supportsHKTAN6 = supportsJobInVersion(bank, "HKTAN", 6)
+        val supportsHKSAL5or7 = supportsJobInVersion(bank, "HKSAL", listOf(5, 7))
+        val supportsHKKAZ5to7 = supportsJobInVersion(bank, "HKKAZ", listOf(5, 6, 7))
+        val supportsHKCCS1 = supportsJobInVersion(bank, "HKCCS", 1)
+
+        val tanInfo = anonymousBankInfoResponse.receivedSegments.filterIsInstance(TanInfo::class.java)
+        val supportedTanProcedures = tanInfo.flatMap { it.tanProcedureParameters.procedureParameters }.map { it.technicalTanProcedureIdentification }
+        val hhd13Supported = supportedTanProcedures.firstOrNull { it.startsWith("hhd1.3", true) } != null
+        val hhd14Supported = supportedTanProcedures.firstOrNull { it.startsWith("hhd1.4", true) } != null
+
+        val supportedHKTANVersions = tanInfo.map { it.segmentVersion }
+        val supportedHKSALVersions = getSupportedVersions(bank, "HKSAL")
+        val supportedHKKAZVersions = getSupportedVersions(bank, "HKKAZ")
+        val supportedHKCAZVersions = getSupportedVersions(bank, "HKCAZ")
+        val supportedHKCCSVersions = getSupportedVersions(bank, "HKCCS")
+
+        val sepaAccountInfoParameters = anonymousBankInfoResponse.receivedSegments.filterIsInstance<SepaAccountInfoParameters>()
+        val supportedSepaFormats = sepaAccountInfoParameters.flatMap { it.supportedSepaFormats }.map { it.substring(it.indexOf(":xsd:") + ":xsd:".length) }
+        val supportsPain_001_001_or_003_03 = supportedSepaFormats.firstOrNull { it.contains("pain.001.001.03") or it.contains("pain.001.003.03") } != null
+
+        csvPrinter.printRecord(bankInfo.bankCode, bankInfo.name, bankInfo.city,
+            bank.bpdVersion,
+            bank.supportedTanProcedures.joinToString(", ") { it.securityFunction.code + ": " + it.displayName + " (" + it.type + ")" },
+            supportedTanProcedures.joinToString(", "),
+            hhd13Supported,
+            hhd14Supported,
+            supportsHKTAN6,
+            supportedHKTANVersions.joinToString(", "),
+            supportsHKSAL5or7,
+            supportedHKSALVersions.joinToString(", "),
+            supportsHKKAZ5to7,
+            supportedHKKAZVersions.joinToString(", "),
+            supportedHKCAZVersions.joinToString(", "),
+            supportsHKCCS1,
+            supportedHKCCSVersions.joinToString(", "),
+            supportsPain_001_001_or_003_03,
+            supportedSepaFormats.joinToString(", "),
+            bank.supportedLanguages.filter { it != Dialogsprache.German }.joinToString(", ") { it.name },
+            bank.supportedJobs.joinToString(", ") { it.jobName + " " + it.segmentVersion }
+        )
+
+        tanInfo.flatMap { it.tanProcedureParameters.procedureParameters }.forEach { procedureParameter ->
+            if (tanProcedureParameter.containsKey(procedureParameter.procedureName) == false) {
+                tanProcedureParameter.put(procedureParameter.procedureName, mutableSetOf(procedureParameter))
+            }
+            else {
+                tanProcedureParameter[procedureParameter.procedureName]?.add(procedureParameter)
+            }
+        }
+
+        if (supportsHKTAN6 == false) {
+            doesNotSupportHKTAN6.add(bankInfo)
+        }
+        if (supportsHKSAL5or7 == false) {
+            doesNotSupportHKSAL5or7.add(bankInfo)
+        }
+        if (supportsHKKAZ5to7 == false) {
+            doesNotSupportHKKAZ5to7.add(bankInfo)
+        }
+        if (supportsHKCCS1 == false) {
+            doesNotSupportHKCCS1.add(bankInfo)
+        }
+
+        if (hhd13Supported) {
+            supportsHhd13.add(bankInfo)
+        }
+        if (hhd14Supported) {
+            supportsHhd14.add(bankInfo)
+        }
+
+        if (supportsPain_001_001_or_003_03 == false) {
+            doesNotSupportPain_001_001_or_003_03.add(bankInfo)
+        }
+    }
+
+    private fun getSupportedVersions(bank: BankData, jobName: String): List<Int> {
+        return bank.supportedJobs.filter { it.jobName == jobName }.map { it.segmentVersion }
+    }
+
+    private fun supportsJobInVersion(bank: BankData, jobName: String, version: Int): Boolean {
+        return supportsJobInVersion(bank, jobName, listOf(version))
+    }
+
+    private fun supportsJobInVersion(bank: BankData, jobName: String, versions: List<Int>): Boolean {
+        return bank.supportedJobs.firstOrNull { it.jobName == jobName && versions.contains(it.segmentVersion) } != null
+    }
+
+
+    private fun printStatistics() {
+        log.info("Did not receive response from Banks ${printBanks(requestNotSuccessful)}")
+
+        log.info("ZkaTanProcedures: ${ResponseParser.ZkaTanProcedures.joinToString()}")
+        log.info("TanProcedureParameters:${tanProcedureParameter.map { System.lineSeparator() + it.key + ": " + it.value.joinToString(", ") { it.securityFunction.code + " " + it.zkaTanProcedure + " " + it.technicalTanProcedureIdentification } } }")
+
+        log.info("Banks supporting HHD 1.3 (${supportsHhd13.size}):${printBanks(supportsHhd13)}")
+        log.info("Banks supporting HHD 1.4 (${supportsHhd14.size}):${printBanks(supportsHhd14)}")
+
+        log.info("Banks not supporting HKTAN 6 ${printBanks(doesNotSupportHKTAN6)}")
+        log.info("Banks not supporting HKSAL 5 or 7 ${printBanks(doesNotSupportHKSAL5or7)}")
+        log.info("Banks not supporting HKKAZ 5-7 ${printBanks(doesNotSupportHKKAZ5to7)}")
+        log.info("Banks not supporting HKCCS 1 ${printBanks(doesNotSupportHKCCS1)}")
+
+        log.info("Banks not supporting pain.001.001.03 or pain.001.003.03 ${printBanks(doesNotSupportPain_001_001_or_003_03)}")
+    }
+
+    private fun printBanks(banks: List<BankInfo>): String {
+        return "(${banks.size}):${ banks.joinToString { System.lineSeparator() + it } }\n\n\n"
+    }
+
+}