Compare commits

..

No commits in common. "e43a8a72396310542ec860a17123745cff24aba5" and "f9db232fbccbe3ccd7077efeb803362fefb5d1ce" have entirely different histories.

45 changed files with 155 additions and 363 deletions

3
.gitmodules vendored
View File

@ -1,3 +0,0 @@
[submodule "gradle/scripts"]
path = gradle/scripts
url = git@github.com:dankito/GradleScripts.git

View File

@ -15,25 +15,7 @@ allprojects {
group = "net.codinux.invoicing"
version = "0.5.0-SNAPSHOT"
ext["sourceCodeRepositoryBaseUrl"] = "git.dankito.net/codinux/eInvoicing"
ext["projectDescription"] = "Tools to work with eInvoices according to EU standard EN 16931"
repositories {
mavenCentral()
}
}
tasks.register("publishAllToMavenLocal") {
dependsOn(
":e-invoice-domain:publishToMavenLocal"
)
}
tasks.register("publishAll") {
dependsOn(
":e-invoice-domain:publish"
)
}

View File

@ -28,7 +28,7 @@ dependencies {
implementation("io.quarkus:quarkus-micrometer")
implementation("io.quarkus:quarkus-micrometer-registry-prometheus")
implementation(project(":e-invoice-domain"))
implementation(project(":e-invoicing-domain"))
implementation("net.codinux.log:klf:$klfVersion")
implementation("net.codinux.log:quarkus-loki-log-appender:$lokiLogAppenderVersion")

View File

@ -67,29 +67,20 @@ class InvoicingResource(
@Path("extract")
@POST
@Consumes(MediaTypePdf, MediaType.APPLICATION_OCTET_STREAM)
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@Produces(MediaType.APPLICATION_JSON)
@Operation(summary = "Extract invoice data from a Factur-X / ZUGFeRD or XRechnung file")
@Tag(name = "Extract")
fun extractInvoiceDataFromPdf(invoice: java.nio.file.Path) =
service.extractInvoiceDataFromPdf(invoice)
@Path("extract")
@POST
@Consumes(MediaType.APPLICATION_XML)
@Produces(MediaType.APPLICATION_JSON)
@Operation(summary = "Extract invoice data from a Factur-X / ZUGFeRD or XRechnung file")
@Tag(name = "Extract")
fun extractInvoiceDataFromXml(invoice: java.nio.file.Path) =
service.extractInvoiceDataFromXml(invoice)
fun extractInvoiceData(invoice: FileUpload) =
service.extractInvoiceData(invoice.uploadedFile())
@Path("validate")
@POST
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@Operation(summary = "Validate a Factur-X / ZUGFeRD or XRechnung file")
@Tag(name = "Validate")
fun validateInvoiceXml(invoice: java.nio.file.Path) =
service.validateInvoice(invoice).reportAsXml
fun validateInvoiceXml(invoice: FileUpload) =
service.validateInvoice(invoice.uploadedFile()).reportAsXml
private fun createPdfFileResponse(pdfFile: java.nio.file.Path, invoice: Invoice): Response =

View File

@ -7,6 +7,7 @@ import net.codinux.invoicing.reader.EInvoiceReader
import net.codinux.invoicing.validation.EInvoiceValidator
import java.io.File
import java.nio.file.Path
import kotlin.io.path.extension
@Singleton
class InvoicingService {
@ -41,15 +42,17 @@ class InvoicingService {
}
fun extractInvoiceDataFromPdf(invoiceFile: Path) =
reader.extractFromPdf(invoiceFile.toFile())
fun extractInvoiceDataFromXml(invoiceFile: Path) =
reader.extractFromXml(invoiceFile.toFile())
fun extractInvoiceData(invoiceFile: Path) = when (invoiceFile.extension.lowercase()) {
"xml" -> reader.extractFromXml(invoiceFile.toFile())
"pdf" -> reader.extractFromPdf(invoiceFile.toFile())
else -> throw IllegalArgumentException("We can only extract eInvoice data from .xml and .pdf files")
}
fun validateInvoice(invoiceFile: Path) =
validator.validate(invoiceFile.toFile())
fun validateInvoice(invoiceFile: Path) =when (invoiceFile.extension.lowercase()) {
"xml", "pdf" -> validator.validate(invoiceFile.toFile())
else -> throw IllegalArgumentException("We can only validate .xml and .pdf eInvoice files")
}
private fun createTempPdfFile(): Path =

View File

@ -1,268 +0,0 @@
package net.codinux.invoicing.mail
import jakarta.mail.Folder
import jakarta.mail.Message
import jakarta.mail.Multipart
import jakarta.mail.Part
import jakarta.mail.Session
import jakarta.mail.Store
import jakarta.mail.event.MessageCountAdapter
import jakarta.mail.event.MessageCountEvent
import kotlinx.coroutines.*
import net.codinux.invoicing.model.Invoice
import net.codinux.invoicing.reader.EInvoiceReader
import net.codinux.log.logger
import org.eclipse.angus.mail.imap.IMAPFolder
import java.io.File
import java.time.Instant
import java.util.*
import java.util.concurrent.Executors
import kotlin.math.max
class MailReader(
private val eInvoiceReader: EInvoiceReader = EInvoiceReader()
) {
private data class MessagePart(
val mediaType: String,
val part: Part
)
private val mailDispatcher = Executors.newFixedThreadPool(max(24, Runtime.getRuntime().availableProcessors() * 4)).asCoroutineDispatcher()
private val log by logger()
fun listenForNewReceivedEInvoices(account: MailAccount, downloadMessageBody: Boolean = false, emailFolderName: String = "INBOX", eInvoiceReceived: (MailWithInvoice) -> Unit) = runBlocking {
try {
connect(account) { store ->
val folder = store.getFolder(emailFolderName)
folder.open(Folder.READ_ONLY)
folder.addMessageCountListener(object : MessageCountAdapter() {
override fun messagesAdded(event: MessageCountEvent) {
event.messages.forEach { message ->
findEInvoice(message, downloadMessageBody)?.let {
eInvoiceReceived(it)
}
}
}
})
launch(mailDispatcher) {
keepConnectionOpen(account, folder)
}
}
} catch (e: Throwable) {
log.error(e) { "Listening to new received eInvoices of '${account.username}' failed" }
}
log.info { "Stopped listening to new received eInvoices of '${account.username}'" }
}
private suspend fun keepConnectionOpen(account: MailAccount, folder: Folder) {
log.info { "Listening to new mails of ${account.username}" }
// Use IMAP IDLE to keep the connection alive
while (true) {
if (!folder.isOpen) {
log.info { "Reopening inbox of ${account.username} ..." }
folder.open(Folder.READ_ONLY)
}
(folder as IMAPFolder).idle()
delay(250)
}
}
fun listAllMessagesWithEInvoice(account: MailAccount, downloadMessageBody: Boolean = false, emailFolderName: String = "INBOX"): List<MailWithInvoice> {
try {
connect(account) { store ->
val inbox = store.getFolder(emailFolderName)
inbox.open(Folder.READ_ONLY)
listAllMessagesWithEInvoiceInFolder(inbox, downloadMessageBody).also {
inbox.close(false)
}
}
} catch (e: Throwable) {
log.error(e) { "Could not read mails of account $account" }
}
return emptyList()
}
private fun listAllMessagesWithEInvoiceInFolder(folder: Folder, downloadMessageBody: Boolean): List<MailWithInvoice> = runBlocking {
val messageCount = folder.messageCount
if (messageCount <= 0) {
return@runBlocking emptyList()
}
IntRange(1, messageCount).mapNotNull { messageNumber -> // message numbers start at 1
async(mailDispatcher) {
try {
findEInvoice(folder.getMessage(messageNumber), downloadMessageBody)
} catch (e: Throwable) {
log.error(e) { "Could not get message with messageNumber $messageNumber" }
null
}
}
}
.awaitAll()
.filterNotNull()
}
private fun findEInvoice(message: Message, downloadMessageBody: Boolean): MailWithInvoice? {
try {
val parts = getAllMessageParts(message)
val attachmentsWithEInvoice = parts.mapNotNull { part ->
findEInvoice(part)
}
if (attachmentsWithEInvoice.isNotEmpty()) {
return MailWithInvoice(
message.from?.joinToString(), message.subject ?: "",
message.sentDate?.let { map(it) }, map(message.receivedDate), message.messageNumber,
parts.any { it.mediaType == "application/pgp-encrypted" },
if (downloadMessageBody) getPlainTextBody(parts) else null, if (downloadMessageBody) getHtmlBody(parts) else null,
attachmentsWithEInvoice
)
}
} catch (e: Throwable) {
log.error(e) { "Could not read message $message" }
}
return null
}
private fun findEInvoice(messagePart: MessagePart): MailAttachmentWithEInvoice? {
try {
val part = messagePart.part
val invoice = tryToReadEInvoice(part, messagePart.mediaType)
if (invoice != null) {
val filename = File(part.fileName)
val file = File.createTempFile(filename.nameWithoutExtension, "." + filename.extension).also { file ->
part.inputStream.use { it.copyTo(file.outputStream()) }
file.deleteOnExit()
}
return MailAttachmentWithEInvoice(part.fileName, messagePart.mediaType, invoice, file)
}
} catch (e: Throwable) {
log.error(e) { "Could not check attachment '${messagePart.part.fileName}' (${messagePart.mediaType}) for eInvoice" }
}
return null
}
private fun tryToReadEInvoice(part: Part, mediaType: String?): Invoice? = try {
val filename = part.fileName?.lowercase() ?: ""
if (filename.endsWith(".pdf") || mediaType == "application/pdf" || mediaType == "application/octet-stream") {
eInvoiceReader.extractFromPdf(part.inputStream)
} else if (filename.endsWith(".xml") || mediaType == "application/xml" || mediaType == "text/xml") {
eInvoiceReader.extractFromXml(part.inputStream)
} else {
null
}
} catch (e: Throwable) {
log.debug(e) { "Could not extract invoices from ${part.fileName}" }
null
}
private fun getAllMessageParts(part: Part): List<MessagePart> {
return if (part.isMimeType("multipart/*")) {
val multipart = part.content as Multipart
val parts = IntRange(0, multipart.count - 1).map { multipart.getBodyPart(it) }
parts.flatMap { subPart ->
getAllMessageParts(subPart)
}
} else {
val mediaType = getMediaType(part)
if (mediaType == null) {
log.warn { "Could not determine media type of message part $part" }
emptyList()
} else {
listOf(MessagePart(mediaType, part))
}
}
}
/**
* In most cases parameters are added to content-type's media type, e.g.
* - text/html; charset=utf-8
* - multipart/related; boundary="boundary-related"; type="text/html"
*
* -> This method removes parameters and return media type (first part) only
*/
private fun getMediaType(part: Part): String? = part.contentType?.lowercase()?.let { contentType ->
val indexOfSeparator = contentType.indexOf(';')
if (indexOfSeparator > -1) {
contentType.substring(0, indexOfSeparator)
} else {
contentType
}
}
private fun getPlainTextBody(parts: Collection<MessagePart>) = getBodyWithMediaType(parts, "text/plain")
private fun getHtmlBody(parts: Collection<MessagePart>) = getBodyWithMediaType(parts, "text/html")
private fun getBodyWithMediaType(parts: Collection<MessagePart>, mediaType: String): String? = try {
val partsForMediaType = parts.filter { it.mediaType == mediaType }
if (partsForMediaType.size == 1) {
partsForMediaType.first().part.content as? String
} else if (partsForMediaType.isEmpty()) {
null
} else {
val partsForMediaTypeWithoutFilename = partsForMediaType.filter { it.part.fileName == null }
if (partsForMediaTypeWithoutFilename.size == 1) {
partsForMediaTypeWithoutFilename.first().part.content as? String
} else if (partsForMediaTypeWithoutFilename.isEmpty()) {
log.warn { "Multiple message parts with media type '$mediaType' found, but all have a filename" }
null
} else { // if there are multiple parts without filename, then the second one is in most cases quoted previous message(s)
partsForMediaTypeWithoutFilename.mapNotNull { it.part.content as? String }.joinToString("\r\n")
}
}
} catch (e: Throwable) {
log.error(e) { "Could not get message body for media type '$mediaType'" }
null
}
private fun map(date: Date): Instant =
date.toInstant()
private fun <T> connect(account: MailAccount, connected: (Store) -> T?): T? {
val properties = mapAccountToJavaMailProperties(account)
val session = Session.getInstance(properties)
session.getStore("imap").use { store ->
store.connect(account.serverAddress, account.username, account.password)
return connected(store)
}
}
private fun mapAccountToJavaMailProperties(account: MailAccount) = Properties().apply {
put("mail.store.protocol", "imap")
put("mail.imap.host", account.serverAddress)
put("mail.imap.port", account.port?.toString() ?: "993") // Default IMAP over SSL
put("mail.imap.ssl.enable", "true")
put("mail.imap.connectiontimeout", "5000")
put("mail.imap.timeout", "5000")
}
}

View File

@ -1,27 +0,0 @@
package net.codinux.invoicing.mail
import java.time.Instant
import java.time.ZoneId
class MailWithInvoice(
val sender: String?,
val subject: String,
val sent: Instant?,
val received: Instant,
/**
* From documentation of underlying mail library:
* "Since message numbers can change within a session if the folder is expunged, clients are advised not to use
* message numbers as references to messages."
*
* -> use with care. Message numbers are not valid / the same anymore after expunge.
*/
val messageNumber: Int,
val isEncrypted: Boolean = false,
val plainTextBody: String?,
val htmlBody: String?,
val attachmentsWithEInvoice: List<MailAttachmentWithEInvoice>
) {
val plainTextOrHtmlBody: String? by lazy { plainTextBody ?: htmlBody }
override fun toString() = "${(sent ?: received).atZone(ZoneId.systemDefault()).toLocalDate()} $sender: $subject, ${attachmentsWithEInvoice.size} invoice(s)"
}

View File

@ -8,8 +8,6 @@ kotlin {
}
val kotlinCoroutinesVersion: String by project
val mustangVersion: String by project
val angusMailVersion: String by project
@ -21,8 +19,6 @@ val xunitVersion: String by project
val logbackVersion: String by project
dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinCoroutinesVersion")
implementation("org.mustangproject:library:$mustangVersion")
implementation("org.mustangproject:validator:$mustangVersion")
@ -42,10 +38,4 @@ dependencies {
tasks.test {
useJUnitPlatform()
}
ext["customArtifactId"] = "e-invoice"
apply(from = "../gradle/scripts/publish-codinux-repo.gradle.kts")
}

View File

@ -5,12 +5,7 @@ import java.io.File
class MailAttachmentWithEInvoice(
val filename: String,
/**
* Attachment's media type like "application/xml", "application/pdf", ...
*
* Should always be non-null, but can theoretically be null.
*/
val mediaType: String?,
val contentType: String,
val invoice: Invoice,
val file: File
) {

View File

@ -0,0 +1,124 @@
package net.codinux.invoicing.mail
import jakarta.mail.BodyPart
import jakarta.mail.Folder
import jakarta.mail.Part
import jakarta.mail.Session
import jakarta.mail.internet.MimeMultipart
import net.codinux.invoicing.model.Invoice
import net.codinux.invoicing.reader.EInvoiceReader
import net.codinux.log.logger
import java.io.File
import java.time.LocalDate
import java.time.ZoneId
import java.util.*
class MailReader(
private val eInvoiceReader: EInvoiceReader = EInvoiceReader()
) {
private val log by logger()
fun listAllMessagesWithEInvoice(account: MailAccount): List<MailWithInvoice> {
val properties = mapAccountToJavaMailProperties(account)
try {
val session = Session.getInstance(properties)
session.getStore("imap").use { store ->
store.connect(account.serverAddress, account.username, account.password)
val inbox = store.getFolder("INBOX")
inbox.open(Folder.READ_ONLY)
return listAllMessagesWithEInvoiceInFolder(inbox).also {
inbox.close(false)
}
}
} catch (e: Throwable) {
log.error(e) { "Could not read mails of account $account" }
}
return emptyList()
}
private fun mapAccountToJavaMailProperties(account: MailAccount) = Properties().apply {
put("mail.store.protocol", "imap")
put("mail.imap.host", account.serverAddress)
put("mail.imap.port", account.port?.toString() ?: "993") // Default IMAP over SSL
put("mail.imap.ssl.enable", "true")
put("mail.imap.connectiontimeout", "5000")
put("mail.imap.timeout", "5000")
}
private fun listAllMessagesWithEInvoiceInFolder(folder: Folder): List<MailWithInvoice> = folder.messages.mapNotNull { message ->
try {
if (message.isMimeType("multipart/*")) {
val multipart = message.content as MimeMultipart
val parts = IntRange(0, multipart.count - 1).map { multipart.getBodyPart(it) }
val attachmentsWithEInvoice = parts.mapNotNull { part ->
findEInvoice(part)
}
if (attachmentsWithEInvoice.isNotEmpty()) {
return@mapNotNull MailWithInvoice(message.from.joinToString(), message.subject, map(message.sentDate), attachmentsWithEInvoice)
}
}
} catch (e: Throwable) {
log.error(e) { "Could not read mail $message" }
}
null
}
private fun findEInvoice(part: BodyPart): MailAttachmentWithEInvoice? {
try {
if (part.disposition == Part.ATTACHMENT) {
val invoice = tryToReadEInvoice(part)
if (invoice != null) {
var contentType = part.contentType
val indexOfSeparator = contentType.indexOf(';')
if (indexOfSeparator > -1) {
contentType = contentType.substring(0, indexOfSeparator)
}
val filename = File(part.fileName)
val file = File.createTempFile(filename.nameWithoutExtension, filename.extension).also { file ->
part.inputStream.use { it.copyTo(file.outputStream()) }
}
return MailAttachmentWithEInvoice(part.fileName, contentType, invoice, file)
}
}
} catch (e: Throwable) {
log.error(e) { "Could not check attachment '${part.fileName}' for eInvoice" }
}
return null
}
private fun tryToReadEInvoice(part: BodyPart): Invoice? = try {
val filename = part.fileName.lowercase()
val contentType = part.contentType.lowercase()
if (filename.endsWith(".pdf") || contentType.startsWith("application/pdf") || contentType.startsWith("application/octet-stream")) {
eInvoiceReader.extractFromPdf(part.inputStream)
} else if (filename.endsWith(".xml") || contentType.startsWith("application/xml") || contentType.startsWith("text/xml")) {
eInvoiceReader.extractFromXml(part.inputStream)
} else {
null
}
} catch (e: Throwable) {
log.debug(e) { "Could not extract invoices from ${part.fileName}" }
null
}
// TODO: same code as in MustangMapper
private fun map(date: Date): LocalDate =
date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate()
}

View File

@ -0,0 +1,12 @@
package net.codinux.invoicing.mail
import java.time.LocalDate
class MailWithInvoice(
val sender: String,
val subject: String,
val date: LocalDate,
val attachmentsWithEInvoice: List<MailAttachmentWithEInvoice>
) {
override fun toString() = "$date $sender: $subject, ${attachmentsWithEInvoice.size} invoices"
}

View File

@ -1,7 +1,6 @@
package net.codinux.invoicing.mail
import assertk.assertThat
import assertk.assertions.isEmpty
import assertk.assertions.isNotEmpty
import org.junit.jupiter.api.Test
import kotlin.test.Ignore
@ -28,9 +27,6 @@ class MailReaderTest {
val result = underTest.listAllMessagesWithEInvoice(mailAccount)
assertThat(result).isNotEmpty()
val messagesWithoutBody = result.filter { it.plainTextOrHtmlBody == null }
assertThat(messagesWithoutBody).isEmpty()
}
}

View File

@ -5,9 +5,7 @@ org.gradle.parallel=true
# Quarkus 3.12 requires Kotlin 2.0
#kotlinVersion=1.9.25
#kotlinCoroutinesVersion=1.8.1
kotlinVersion=2.0.20
kotlinCoroutinesVersion=1.9.0
quarkusVersion=3.16.3

@ -1 +0,0 @@
Subproject commit 88f1b01167e6a34b5b91f8797845bca0b7e4d3ab

View File

@ -26,6 +26,6 @@ plugins {
rootProject.name = "eInvoicing"
include("e-invoice-domain")
include("e-invoicing-domain")
include("e-invoice-api")