[libeufin] branch master updated: More structure when downloading transactions.

[gnunet]

This is an automated email from the git hooks/post-receive script.

ms pushed a commit to branch master
in repository libeufin.

The following commit(s) were added to refs/heads/master by this push:
     new 955000a  More structure when downloading transactions.
955000a is described below

commit 955000a5dc291627e2a96e9bbd6e14b567726305
Author: MS <ms@taler.net>
AuthorDate: Thu Sep 2 09:10:07 2021 +0000

    More structure when downloading transactions.
    
    In detail, Nexus informs the requester about both
    downloaded and new transactions.
---
 .../tech/libeufin/nexus/bankaccount/BankAccount.kt | 53 ++++++++++++++++++----
 .../tech/libeufin/nexus/ebics/EbicsClient.kt       |  3 +-
 .../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt | 25 +++++++---
 .../tech/libeufin/nexus/server/NexusServer.kt      |  6 +--
 util/src/main/kotlin/Ebics.kt                      |  9 +++-
 5 files changed, 74 insertions(+), 22 deletions(-)

diff --git 
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index c911733..2b7c3bd 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -123,9 +123,30 @@ private fun findDuplicate(bankAccountId: String, 
acctSvcrRef: String): NexusBank
     }
 }
 
-fun processCamtMessage(bankAccountId: String, camtDoc: Document, code: 
String): Int {
+/**
+ * NOTE: this type can be used BOTH for one Camt document OR
+ * for a set of those.
+ */
+data class CamtProcessingResult(
+    /**
+     * Number of transactions that are new to the database.
+     * Note that transaction T can be downloaded multiple times;
+     * for example, once in a C52 and once - maybe a day later -
+     * in a C53.  The second time, the transaction is not considered
+     * 'new'.
+     */
+    val newTransactions: Int,
+
+    /**
+     * Total number of transactions that were included in a report
+     * or a statement.
+     */
+    val downloadedTransactions: Int
+)
+fun processCamtMessage(bankAccountId: String, camtDoc: Document, code: 
String): CamtProcessingResult {
     logger.info("processing CAMT message")
     var newTransactions = 0
+    var downloadedTransactions = 0
     transaction {
         val acct = NexusBankAccountEntity.findByName(bankAccountId)
         if (acct == null) {
@@ -156,6 +177,7 @@ fun processCamtMessage(bankAccountId: String, camtDoc: 
Document, code: String):
         }
         val entries = res.reports.map { it.entries }.flatten()
         logger.info("found ${entries.size} money movements")
+        downloadedTransactions = entries.size
         txloop@ for (entry in entries) {
             val singletonBatchedTransaction = 
entry.batches?.get(0)?.batchTransactions?.get(0)
                 ?: throw NexusError(
@@ -204,15 +226,19 @@ fun processCamtMessage(bankAccountId: String, camtDoc: 
Document, code: String):
             }
         }
     }
-    return newTransactions
+    return CamtProcessingResult(
+        newTransactions = newTransactions,
+        downloadedTransactions = downloadedTransactions
+    )
 }
 
 /**
  * Create new transactions for an account based on bank messages it
  * did not see before.
  */
-fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: 
String): Int {
+fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId: 
String): CamtProcessingResult {
     var totalNew = 0
+    var downloadedTransactions = 0
     transaction {
         val conn =
             NexusBankConnectionEntity.find { 
NexusBankConnectionsTable.connectionId eq bankConnectionId }.firstOrNull()
@@ -229,17 +255,22 @@ fun ingestBankMessagesIntoAccount(bankConnectionId: 
String, bankAccountId: Strin
                     (NexusBankMessagesTable.id greater 
acct.highestSeenBankMessageSerialId)
         }.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
             val doc = 
XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
-            val newTransactions = processCamtMessage(bankAccountId, doc, 
it.code)
-            if (newTransactions == -1) {
+            val processingResult = processCamtMessage(bankAccountId, doc, 
it.code)
+            if (processingResult.newTransactions == -1) {
                 it.errors = true
                 return@forEach
             }
             lastId = it.id.value
-            totalNew += newTransactions
+            totalNew += processingResult.newTransactions
+            downloadedTransactions += processingResult.downloadedTransactions
         }
         acct.highestSeenBankMessageSerialId = lastId
     }
-    return totalNew
+    // return totalNew
+    return CamtProcessingResult(
+        newTransactions = totalNew,
+        downloadedTransactions = downloadedTransactions
+    )
 }
 
 /**
@@ -285,7 +316,9 @@ fun addPaymentInitiation(paymentData: Pain001Data, 
debtorAccount: NexusBankAccou
     }
 }
 
-suspend fun fetchBankAccountTransactions(client: HttpClient, fetchSpec: 
FetchSpecJson, accountId: String): Int {
+suspend fun fetchBankAccountTransactions(
+    client: HttpClient, fetchSpec: FetchSpecJson, accountId: String
+): CamtProcessingResult {
     val res = transaction {
         val acct = NexusBankAccountEntity.findByName(accountId)
         if (acct == null) {
@@ -314,11 +347,11 @@ suspend fun fetchBankAccountTransactions(client: 
HttpClient, fetchSpec: FetchSpe
         accountId
     )
 
-    val newTransactions = ingestBankMessagesIntoAccount(res.connectionName, 
accountId)
+    val ingestionResult = ingestBankMessagesIntoAccount(res.connectionName, 
accountId)
     ingestFacadeTransactions(accountId, "taler-wire-gateway", ::talerFilter, 
::maybeTalerRefunds)
     ingestFacadeTransactions(accountId, "anastasis", ::anastasisFilter, null)
 
-    return newTransactions
+    return ingestionResult
 }
 
 fun importBankAccount(call: ApplicationCall, offeredBankAccountId: String, 
nexusBankAccountId: String) {
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
index 65caf51..b04c388 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt
@@ -90,7 +90,8 @@ suspend fun doEbicsDownloadTransaction(
         else -> {
             throw EbicsProtocolError(
                 HttpStatusCode.InternalServerError,
-                "unexpected return code ${initResponse.technicalReturnCode}"
+                "unexpected return code ${initResponse.technicalReturnCode}",
+                initResponse.technicalReturnCode
             )
         }
     }
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index d312d59..23abfd5 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -104,12 +104,25 @@ private suspend fun fetchEbicsC5x(
     subscriberDetails: EbicsClientSubscriberDetails
 ) {
     logger.debug("Requesting $historyType")
-    val response = doEbicsDownloadTransaction(
-        client,
-        subscriberDetails,
-        historyType,
-        orderParams
-    )
+    val response = try {
+        doEbicsDownloadTransaction(
+            client,
+            subscriberDetails,
+            historyType,
+            orderParams
+        )
+    } catch (e: EbicsProtocolError) {
+        /**
+         * This error type is not an actual error in this handler.
+         */
+        if (e.ebicsTechnicalCode == 
EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) {
+            logger.info("Could not find new transactions to download")
+            return
+        }
+        // re-throw in any other error case.
+        throw e
+    }
+
     when (historyType) {
         "C52" -> {
         }
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt 
b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
index a4410fd..5bfa62c 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/server/NexusServer.kt
@@ -720,10 +720,8 @@ fun serverMain(host: String, port: Int) {
                         null
                     )
                 }
-                val newTransactions = fetchBankAccountTransactions(client, 
fetchSpec, accountid)
-                call.respond(makeJsonObject {
-                    prop("newTransactions", newTransactions)
-                })
+                val ingestionResult = fetchBankAccountTransactions(client, 
fetchSpec, accountid)
+                call.respond(ingestionResult)
                 return@post
             }
 
diff --git a/util/src/main/kotlin/Ebics.kt b/util/src/main/kotlin/Ebics.kt
index 7926bad..e2df0fd 100644
--- a/util/src/main/kotlin/Ebics.kt
+++ b/util/src/main/kotlin/Ebics.kt
@@ -45,7 +45,14 @@ private val logger: Logger = 
LoggerFactory.getLogger("tech.libeufin.util")
 
 data class EbicsProtocolError(
     val httpStatusCode: HttpStatusCode,
-    val reason: String
+    val reason: String,
+    /**
+     * This error type is also used when Nexus finds itself
+     * in an inconsistent state, without interacting with the
+     * bank.  In this case, the EBICS code below can be left
+     * null.
+     */
+    val ebicsTechnicalCode: EbicsReturnCode? = null
 ) : Exception(reason)
 
 data class EbicsDateRange(val start: ZonedDateTime, val end: ZonedDateTime)

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.