Configured publishing to Maven repository

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "gradle/scripts"]
+	path = gradle/scripts
+	url = git@github.com:dankito/GradleScripts.git

build.gradle.kts

@@ -15,7 +15,25 @@ allprojects {
     group = "net.codinux.invoicing"
     version = "0.5.0-SNAPSHOT"
 
+    ext["sourceCodeRepositoryBaseUrl"] = "git.dankito.net/codinux/eInvoicing"
+
+    ext["projectDescription"] = "Tools to work with eInvoices according to EU standard EN 16931"
+
+
     repositories {
         mavenCentral()
     }
+}
+
+
+tasks.register("publishAllToMavenLocal") {
+    dependsOn(
+        ":e-invoice-domain:publishToMavenLocal"
+    )
+}
+
+tasks.register("publishAll") {
+    dependsOn(
+        ":e-invoice-domain:publish"
+    )
 }

e-invoice-api/build.gradle.kts

@@ -28,7 +28,7 @@ dependencies {
     implementation("io.quarkus:quarkus-micrometer")
     implementation("io.quarkus:quarkus-micrometer-registry-prometheus")
 
-    implementation(project(":e-invoicing-domain"))
+    implementation(project(":e-invoice-domain"))
 
     implementation("net.codinux.log:klf:$klfVersion")
     implementation("net.codinux.log:quarkus-loki-log-appender:$lokiLogAppenderVersion")

e-invoice-api/src/main/kotlin/net/codinux/invoicing/api/InvoicingResource.kt

@@ -67,20 +67,29 @@ class InvoicingResource(
 
     @Path("extract")
     @POST
-    @Consumes(MediaType.APPLICATION_OCTET_STREAM)
+    @Consumes(MediaTypePdf, MediaType.APPLICATION_OCTET_STREAM)
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(summary = "Extract invoice data from a Factur-X / ZUGFeRD or XRechnung file")
     @Tag(name = "Extract")
-    fun extractInvoiceData(invoice: FileUpload) =
-        service.extractInvoiceData(invoice.uploadedFile())
+    fun extractInvoiceDataFromPdf(invoice: java.nio.file.Path) =
+        service.extractInvoiceDataFromPdf(invoice)
+
+    @Path("extract")
+    @POST
+    @Consumes(MediaType.APPLICATION_XML)
+    @Produces(MediaType.APPLICATION_JSON)
+    @Operation(summary = "Extract invoice data from a Factur-X / ZUGFeRD or XRechnung file")
+    @Tag(name = "Extract")
+    fun extractInvoiceDataFromXml(invoice: java.nio.file.Path) =
+        service.extractInvoiceDataFromXml(invoice)
 
     @Path("validate")
     @POST
     @Consumes(MediaType.APPLICATION_OCTET_STREAM)
     @Operation(summary = "Validate a Factur-X / ZUGFeRD or XRechnung file")
     @Tag(name = "Validate")
-    fun validateInvoiceXml(invoice: FileUpload) =
-        service.validateInvoice(invoice.uploadedFile()).reportAsXml
+    fun validateInvoiceXml(invoice: java.nio.file.Path) =
+        service.validateInvoice(invoice).reportAsXml
 
 
     private fun createPdfFileResponse(pdfFile: java.nio.file.Path, invoice: Invoice): Response =

e-invoice-api/src/main/kotlin/net/codinux/invoicing/service/InvoicingService.kt

@@ -7,7 +7,6 @@ import net.codinux.invoicing.reader.EInvoiceReader
 import net.codinux.invoicing.validation.EInvoiceValidator
 import java.io.File
 import java.nio.file.Path
-import kotlin.io.path.extension
 
 @Singleton
 class InvoicingService {
@@ -42,17 +41,15 @@ class InvoicingService {
     }
 
 
-    fun extractInvoiceData(invoiceFile: Path) = when (invoiceFile.extension.lowercase()) {
-        "xml" -> reader.extractFromXml(invoiceFile.toFile())
-        "pdf" -> reader.extractFromPdf(invoiceFile.toFile())
-        else -> throw IllegalArgumentException("We can only extract eInvoice data from .xml and .pdf files")
-    }
+    fun extractInvoiceDataFromPdf(invoiceFile: Path) =
+        reader.extractFromPdf(invoiceFile.toFile())
+
+    fun extractInvoiceDataFromXml(invoiceFile: Path) =
+        reader.extractFromXml(invoiceFile.toFile())
 
 
-    fun validateInvoice(invoiceFile: Path) =when (invoiceFile.extension.lowercase()) {
-        "xml", "pdf" -> validator.validate(invoiceFile.toFile())
-        else -> throw IllegalArgumentException("We can only validate .xml and .pdf eInvoice files")
-    }
+    fun validateInvoice(invoiceFile: Path) =
+        validator.validate(invoiceFile.toFile())
 
 
     private fun createTempPdfFile(): Path =

e-invoice-domain/build.gradle.kts

@@ -8,6 +8,8 @@ kotlin {
 }
 
 
+val kotlinCoroutinesVersion: String by project
+
 val mustangVersion: String by project
 
 val angusMailVersion: String by project
@@ -19,6 +21,8 @@ val xunitVersion: String by project
 val logbackVersion: String by project
 
 dependencies {
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinCoroutinesVersion")
+
     implementation("org.mustangproject:library:$mustangVersion")
     implementation("org.mustangproject:validator:$mustangVersion")
 
@@ -38,4 +42,10 @@ dependencies {
 
 tasks.test {
     useJUnitPlatform()
-}
+}
+
+
+
+ext["customArtifactId"] = "e-invoice"
+
+apply(from = "../gradle/scripts/publish-codinux-repo.gradle.kts")

e-invoice-domain/src/main/kotlin/net/codinux/invoicing/mail/MailAttachmentWithEInvoice.kt

@@ -5,7 +5,12 @@ import java.io.File
 
 class MailAttachmentWithEInvoice(
     val filename: String,
-    val contentType: String,
+    /**
+     * Attachment's media type like "application/xml", "application/pdf", ...
+     *
+     * Should always be non-null, but can theoretically be null.
+     */
+    val mediaType: String?,
     val invoice: Invoice,
     val file: File
 ) {

e-invoice-domain/src/main/kotlin/net/codinux/invoicing/mail/MailReader.kt

@@ -0,0 +1,268 @@
+package net.codinux.invoicing.mail
+
+import jakarta.mail.Folder
+import jakarta.mail.Message
+import jakarta.mail.Multipart
+import jakarta.mail.Part
+import jakarta.mail.Session
+import jakarta.mail.Store
+import jakarta.mail.event.MessageCountAdapter
+import jakarta.mail.event.MessageCountEvent
+import kotlinx.coroutines.*
+import net.codinux.invoicing.model.Invoice
+import net.codinux.invoicing.reader.EInvoiceReader
+import net.codinux.log.logger
+import org.eclipse.angus.mail.imap.IMAPFolder
+import java.io.File
+import java.time.Instant
+import java.util.*
+import java.util.concurrent.Executors
+import kotlin.math.max
+
+class MailReader(
+    private val eInvoiceReader: EInvoiceReader = EInvoiceReader()
+) {
+
+    private data class MessagePart(
+        val mediaType: String,
+        val part: Part
+    )
+
+
+    private val mailDispatcher = Executors.newFixedThreadPool(max(24, Runtime.getRuntime().availableProcessors() * 4)).asCoroutineDispatcher()
+
+    private val log by logger()
+
+
+    fun listenForNewReceivedEInvoices(account: MailAccount, downloadMessageBody: Boolean = false, emailFolderName: String = "INBOX", eInvoiceReceived: (MailWithInvoice) -> Unit) = runBlocking {
+        try {
+            connect(account) { store ->
+                val folder = store.getFolder(emailFolderName)
+                folder.open(Folder.READ_ONLY)
+
+                folder.addMessageCountListener(object : MessageCountAdapter() {
+                    override fun messagesAdded(event: MessageCountEvent) {
+                        event.messages.forEach { message ->
+                            findEInvoice(message, downloadMessageBody)?.let {
+                                eInvoiceReceived(it)
+                            }
+                        }
+                    }
+                })
+
+                launch(mailDispatcher) {
+                    keepConnectionOpen(account, folder)
+                }
+            }
+        } catch (e: Throwable) {
+            log.error(e) { "Listening to new received eInvoices of '${account.username}' failed" }
+        }
+
+        log.info { "Stopped listening to new received eInvoices of '${account.username}'" }
+    }
+
+    private suspend fun keepConnectionOpen(account: MailAccount, folder: Folder) {
+        log.info { "Listening to new mails of ${account.username}" }
+
+        // Use IMAP IDLE to keep the connection alive
+        while (true) {
+            if (!folder.isOpen) {
+                log.info { "Reopening inbox of ${account.username} ..." }
+                folder.open(Folder.READ_ONLY)
+            }
+
+            (folder as IMAPFolder).idle()
+
+            delay(250)
+        }
+    }
+
+
+    fun listAllMessagesWithEInvoice(account: MailAccount, downloadMessageBody: Boolean = false, emailFolderName: String = "INBOX"): List<MailWithInvoice> {
+        try {
+            connect(account) { store ->
+                val inbox = store.getFolder(emailFolderName)
+                inbox.open(Folder.READ_ONLY)
+
+                listAllMessagesWithEInvoiceInFolder(inbox, downloadMessageBody).also {
+                    inbox.close(false)
+                }
+            }
+        } catch (e: Throwable) {
+            log.error(e) { "Could not read mails of account $account" }
+        }
+
+        return emptyList()
+    }
+
+    private fun listAllMessagesWithEInvoiceInFolder(folder: Folder, downloadMessageBody: Boolean): List<MailWithInvoice> = runBlocking {
+        val messageCount = folder.messageCount
+        if (messageCount <= 0) {
+            return@runBlocking emptyList()
+        }
+
+        IntRange(1, messageCount).mapNotNull { messageNumber -> // message numbers start at 1
+            async(mailDispatcher) {
+                try {
+                    findEInvoice(folder.getMessage(messageNumber), downloadMessageBody)
+                } catch (e: Throwable) {
+                    log.error(e) { "Could not get message with messageNumber $messageNumber" }
+                    null
+                }
+            }
+        }
+            .awaitAll()
+            .filterNotNull()
+    }
+
+    private fun findEInvoice(message: Message, downloadMessageBody: Boolean): MailWithInvoice? {
+        try {
+            val parts = getAllMessageParts(message)
+
+            val attachmentsWithEInvoice = parts.mapNotNull { part ->
+                findEInvoice(part)
+            }
+
+            if (attachmentsWithEInvoice.isNotEmpty()) {
+                return MailWithInvoice(
+                    message.from?.joinToString(), message.subject ?: "",
+                    message.sentDate?.let { map(it) }, map(message.receivedDate), message.messageNumber,
+                    parts.any { it.mediaType == "application/pgp-encrypted" },
+                    if (downloadMessageBody) getPlainTextBody(parts) else null, if (downloadMessageBody) getHtmlBody(parts) else null,
+                    attachmentsWithEInvoice
+                )
+            }
+        } catch (e: Throwable) {
+            log.error(e) { "Could not read message $message" }
+        }
+
+        return null
+    }
+
+    private fun findEInvoice(messagePart: MessagePart): MailAttachmentWithEInvoice? {
+        try {
+            val part = messagePart.part
+            val invoice = tryToReadEInvoice(part, messagePart.mediaType)
+
+            if (invoice != null) {
+                val filename = File(part.fileName)
+                val file = File.createTempFile(filename.nameWithoutExtension, "." + filename.extension).also { file ->
+                    part.inputStream.use { it.copyTo(file.outputStream()) }
+                    file.deleteOnExit()
+                }
+
+                return MailAttachmentWithEInvoice(part.fileName, messagePart.mediaType, invoice, file)
+            }
+        } catch (e: Throwable) {
+            log.error(e) { "Could not check attachment '${messagePart.part.fileName}' (${messagePart.mediaType}) for eInvoice" }
+        }
+
+        return null
+    }
+
+    private fun tryToReadEInvoice(part: Part, mediaType: String?): Invoice? = try {
+        val filename = part.fileName?.lowercase() ?: ""
+
+        if (filename.endsWith(".pdf") || mediaType == "application/pdf" || mediaType == "application/octet-stream") {
+            eInvoiceReader.extractFromPdf(part.inputStream)
+        } else if (filename.endsWith(".xml") || mediaType == "application/xml" || mediaType == "text/xml") {
+            eInvoiceReader.extractFromXml(part.inputStream)
+        } else {
+            null
+        }
+    } catch (e: Throwable) {
+        log.debug(e) { "Could not extract invoices from ${part.fileName}" }
+        null
+    }
+
+
+    private fun getAllMessageParts(part: Part): List<MessagePart> {
+        return if (part.isMimeType("multipart/*")) {
+            val multipart = part.content as Multipart
+            val parts = IntRange(0, multipart.count - 1).map { multipart.getBodyPart(it) }
+
+            parts.flatMap { subPart ->
+                getAllMessageParts(subPart)
+            }
+        } else {
+            val mediaType = getMediaType(part)
+            if (mediaType == null) {
+                log.warn { "Could not determine media type of message part $part" }
+                emptyList()
+            } else {
+                listOf(MessagePart(mediaType, part))
+            }
+        }
+    }
+
+    /**
+     * In most cases parameters are added to content-type's media type, e.g.
+     * - text/html; charset=utf-8
+     * - multipart/related; boundary="boundary-related"; type="text/html"
+     *
+     * -> This method removes parameters and return media type (first part) only
+     */
+    private fun getMediaType(part: Part): String? = part.contentType?.lowercase()?.let { contentType ->
+        val indexOfSeparator = contentType.indexOf(';')
+
+        if (indexOfSeparator > -1) {
+            contentType.substring(0, indexOfSeparator)
+        } else {
+            contentType
+        }
+    }
+
+    private fun getPlainTextBody(parts: Collection<MessagePart>) = getBodyWithMediaType(parts, "text/plain")
+
+    private fun getHtmlBody(parts: Collection<MessagePart>) = getBodyWithMediaType(parts, "text/html")
+
+    private fun getBodyWithMediaType(parts: Collection<MessagePart>, mediaType: String): String? = try {
+        val partsForMediaType = parts.filter { it.mediaType == mediaType }
+
+        if (partsForMediaType.size == 1) {
+            partsForMediaType.first().part.content as? String
+        } else if (partsForMediaType.isEmpty()) {
+            null
+        } else {
+            val partsForMediaTypeWithoutFilename = partsForMediaType.filter { it.part.fileName == null }
+            if (partsForMediaTypeWithoutFilename.size == 1) {
+                partsForMediaTypeWithoutFilename.first().part.content as? String
+            } else if (partsForMediaTypeWithoutFilename.isEmpty()) {
+                log.warn { "Multiple message parts with media type '$mediaType' found, but all have a filename" }
+                null
+            } else { // if there are multiple parts without filename, then the second one is in most cases quoted previous message(s)
+                partsForMediaTypeWithoutFilename.mapNotNull { it.part.content as? String }.joinToString("\r\n")
+            }
+        }
+    } catch (e: Throwable) {
+        log.error(e) { "Could not get message body for media type '$mediaType'" }
+        null
+    }
+
+    private fun map(date: Date): Instant =
+        date.toInstant()
+
+
+    private fun <T> connect(account: MailAccount, connected: (Store) -> T?): T? {
+        val properties = mapAccountToJavaMailProperties(account)
+
+        val session = Session.getInstance(properties)
+        session.getStore("imap").use { store ->
+            store.connect(account.serverAddress, account.username, account.password)
+
+            return connected(store)
+        }
+    }
+
+    private fun mapAccountToJavaMailProperties(account: MailAccount) = Properties().apply {
+        put("mail.store.protocol", "imap")
+
+        put("mail.imap.host", account.serverAddress)
+        put("mail.imap.port", account.port?.toString() ?: "993")  // Default IMAP over SSL
+        put("mail.imap.ssl.enable", "true")
+
+        put("mail.imap.connectiontimeout", "5000")
+        put("mail.imap.timeout", "5000")
+    }
+
+}

e-invoice-domain/src/main/kotlin/net/codinux/invoicing/mail/MailWithInvoice.kt

@@ -0,0 +1,27 @@
+package net.codinux.invoicing.mail
+
+import java.time.Instant
+import java.time.ZoneId
+
+class MailWithInvoice(
+    val sender: String?,
+    val subject: String,
+    val sent: Instant?,
+    val received: Instant,
+    /**
+     * From documentation of underlying mail library:
+     * "Since message numbers can change within a session if the folder is expunged, clients are advised not to use
+     * message numbers as references to messages."
+     *
+     * -> use with care. Message numbers are not valid / the same anymore after expunge.
+     */
+    val messageNumber: Int,
+    val isEncrypted: Boolean = false,
+    val plainTextBody: String?,
+    val htmlBody: String?,
+    val attachmentsWithEInvoice: List<MailAttachmentWithEInvoice>
+) {
+    val plainTextOrHtmlBody: String? by lazy { plainTextBody ?: htmlBody }
+
+    override fun toString() = "${(sent ?: received).atZone(ZoneId.systemDefault()).toLocalDate()} $sender: $subject, ${attachmentsWithEInvoice.size} invoice(s)"
+}

e-invoice-domain/src/test/kotlin/net/codinux/invoicing/mail/MailReaderTest.kt

@@ -1,6 +1,7 @@
 package net.codinux.invoicing.mail
 
 import assertk.assertThat
+import assertk.assertions.isEmpty
 import assertk.assertions.isNotEmpty
 import org.junit.jupiter.api.Test
 import kotlin.test.Ignore
@@ -27,6 +28,9 @@ class MailReaderTest {
         val result = underTest.listAllMessagesWithEInvoice(mailAccount)
 
         assertThat(result).isNotEmpty()
+
+        val messagesWithoutBody = result.filter { it.plainTextOrHtmlBody == null }
+        assertThat(messagesWithoutBody).isEmpty()
     }
 
 }

e-invoicing-domain/src/main/kotlin/net/codinux/invoicing/mail/MailReader.kt

@@ -1,124 +0,0 @@
-package net.codinux.invoicing.mail
-
-import jakarta.mail.BodyPart
-import jakarta.mail.Folder
-import jakarta.mail.Part
-import jakarta.mail.Session
-import jakarta.mail.internet.MimeMultipart
-import net.codinux.invoicing.model.Invoice
-import net.codinux.invoicing.reader.EInvoiceReader
-import net.codinux.log.logger
-import java.io.File
-import java.time.LocalDate
-import java.time.ZoneId
-import java.util.*
-
-class MailReader(
-    private val eInvoiceReader: EInvoiceReader = EInvoiceReader()
-) {
-
-    private val log by logger()
-
-
-    fun listAllMessagesWithEInvoice(account: MailAccount): List<MailWithInvoice> {
-        val properties = mapAccountToJavaMailProperties(account)
-
-        try {
-            val session = Session.getInstance(properties)
-            session.getStore("imap").use { store ->
-                store.connect(account.serverAddress, account.username, account.password)
-
-                val inbox = store.getFolder("INBOX")
-                inbox.open(Folder.READ_ONLY)
-
-                return listAllMessagesWithEInvoiceInFolder(inbox).also {
-                    inbox.close(false)
-                }
-            }
-        } catch (e: Throwable) {
-            log.error(e) { "Could not read mails of account $account" }
-        }
-
-        return emptyList()
-    }
-
-    private fun mapAccountToJavaMailProperties(account: MailAccount) = Properties().apply {
-        put("mail.store.protocol", "imap")
-
-        put("mail.imap.host", account.serverAddress)
-        put("mail.imap.port", account.port?.toString() ?: "993")  // Default IMAP over SSL
-        put("mail.imap.ssl.enable", "true")
-
-        put("mail.imap.connectiontimeout", "5000")
-        put("mail.imap.timeout", "5000")
-    }
-
-
-    private fun listAllMessagesWithEInvoiceInFolder(folder: Folder): List<MailWithInvoice> = folder.messages.mapNotNull { message ->
-        try {
-            if (message.isMimeType("multipart/*")) {
-                val multipart = message.content as MimeMultipart
-                val parts = IntRange(0, multipart.count - 1).map { multipart.getBodyPart(it) }
-
-                val attachmentsWithEInvoice = parts.mapNotNull { part ->
-                    findEInvoice(part)
-                }
-
-                if (attachmentsWithEInvoice.isNotEmpty()) {
-                    return@mapNotNull MailWithInvoice(message.from.joinToString(), message.subject, map(message.sentDate), attachmentsWithEInvoice)
-                }
-            }
-        } catch (e: Throwable) {
-            log.error(e) { "Could not read mail $message" }
-        }
-
-        null
-    }
-
-    private fun findEInvoice(part: BodyPart): MailAttachmentWithEInvoice? {
-        try {
-            if (part.disposition == Part.ATTACHMENT) {
-                val invoice = tryToReadEInvoice(part)
-                if (invoice != null) {
-                    var contentType = part.contentType
-                    val indexOfSeparator = contentType.indexOf(';')
-                    if (indexOfSeparator > -1) {
-                        contentType = contentType.substring(0, indexOfSeparator)
-                    }
-
-                    val filename = File(part.fileName)
-                    val file = File.createTempFile(filename.nameWithoutExtension, filename.extension).also { file ->
-                        part.inputStream.use { it.copyTo(file.outputStream()) }
-                    }
-
-                    return MailAttachmentWithEInvoice(part.fileName, contentType, invoice, file)
-                }
-            }
-        } catch (e: Throwable) {
-            log.error(e) { "Could not check attachment '${part.fileName}' for eInvoice" }
-        }
-
-        return null
-    }
-
-    private fun tryToReadEInvoice(part: BodyPart): Invoice? = try {
-        val filename = part.fileName.lowercase()
-        val contentType = part.contentType.lowercase()
-
-        if (filename.endsWith(".pdf") || contentType.startsWith("application/pdf") || contentType.startsWith("application/octet-stream")) {
-            eInvoiceReader.extractFromPdf(part.inputStream)
-        } else if (filename.endsWith(".xml") || contentType.startsWith("application/xml") || contentType.startsWith("text/xml")) {
-            eInvoiceReader.extractFromXml(part.inputStream)
-        } else {
-            null
-        }
-    } catch (e: Throwable) {
-        log.debug(e) { "Could not extract invoices from ${part.fileName}" }
-        null
-    }
-
-    // TODO: same code as in MustangMapper
-    private fun map(date: Date): LocalDate =
-        date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate()
-
-}

e-invoicing-domain/src/main/kotlin/net/codinux/invoicing/mail/MailWithInvoice.kt

@@ -1,12 +0,0 @@
-package net.codinux.invoicing.mail
-
-import java.time.LocalDate
-
-class MailWithInvoice(
-    val sender: String,
-    val subject: String,
-    val date: LocalDate,
-    val attachmentsWithEInvoice: List<MailAttachmentWithEInvoice>
-) {
-    override fun toString() = "$date $sender: $subject, ${attachmentsWithEInvoice.size} invoices"
-}

gradle.properties

@@ -5,7 +5,9 @@ org.gradle.parallel=true
 
 # Quarkus 3.12 requires Kotlin 2.0
 #kotlinVersion=1.9.25
+#kotlinCoroutinesVersion=1.8.1
 kotlinVersion=2.0.20
+kotlinCoroutinesVersion=1.9.0
 
 quarkusVersion=3.16.3
 

gradle/scripts

@@ -0,0 +1 @@
+Subproject commit 88f1b01167e6a34b5b91f8797845bca0b7e4d3ab

settings.gradle.kts

@@ -26,6 +26,6 @@ plugins {
 
 rootProject.name = "eInvoicing"
 
-include("e-invoicing-domain")
+include("e-invoice-domain")
 
 include("e-invoice-api")