diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala
index be27454b847..2f57eada0ac 100644
--- a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala
+++ b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala
@@ -58,8 +58,8 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
/**
* There is a list of variables that MUST be set at that point:
- * - an id for the node (it is ok that it may be change latter on the process, for ex. for unusal changes)
- * - an hostname
+ * - an id for the node (it is ok that it may be change later on the process, for ex. for unusual changes)
+ * - a hostname
* - an admin (root user on the node)
* - a policy server id
* - an OS name
@@ -85,7 +85,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
ZIO.foldLeft(checks)(inventory)((currentInventory, check) => check(currentInventory))
}
- // Utilitary method to get only once the RUDDER and the AGENT
+ // Utility method to get only once the RUDDER and the AGENT
private def getInTags(xml: NodeSeq, tag: String): NodeSeq = {
xml \\ tag
}
@@ -200,7 +200,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
/**
* Kernel version: either
- * - non empty OPERATINGSYSTEM > KERNEL_VERSION
+ * - non-empty OPERATINGSYSTEM > KERNEL_VERSION
* or
* - (on AIX and non empty HARDWARE > OSVERSION )
* Other cases are failure (missing required info)
diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala
new file mode 100644
index 00000000000..58f48e8fec6
--- /dev/null
+++ b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala
@@ -0,0 +1,131 @@
+/*
+ *************************************************************************************
+ * Copyright 2024 Normation SAS
+ *************************************************************************************
+ *
+ * This file is part of Rudder.
+ *
+ * Rudder is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * In accordance with the terms of section 7 (7. Additional Terms.) of
+ * the GNU General Public License version 3, the copyright holders add
+ * the following Additional permissions:
+ * Notwithstanding to the terms of section 5 (5. Conveying Modified Source
+ * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General
+ * Public License version 3, when you create a Related Module, this
+ * Related Module is not considered as a part of the work and may be
+ * distributed under the license agreement of your choice.
+ * A "Related Module" means a set of sources files including their
+ * documentation that, without modification of the Source Code, enables
+ * supplementary functions or services in addition to those offered by
+ * the Software.
+ *
+ * Rudder is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rudder. If not, see .
+
+ *
+ *************************************************************************************
+ */
+
+package com.normation.inventory.provisioning.fusion
+
+import com.normation.errors.*
+import com.normation.inventory.provisioning.fusion.OptText.optText
+import com.normation.inventory.services.provisioning.*
+import java.time.*
+import java.time.format.DateTimeFormatter
+import scala.xml.NodeSeq
+import zio.*
+
+class PreInventoryParserCheckInventoryAge(maxBeforeNow: Duration, maxAfterNow: Duration) extends PreInventoryParser {
+ override val name = "post_process_inventory:check_inventory_age"
+
+ /**
+ * We don't want to accept inventories;
+ * - older than maxBeforeNow
+ * - more in the future than maxAfterNow
+ *
+ * Age is deducted from the attribute:
+ * - ACCESSLOG > LOGDATE, which is a datetime in local server time, without timezone.
+ * - OPERATINGSYSTEM > TIMEZONE > TIMEZONE that may not be there, for more fun.
+ *
+ * If we are not able to parse the timezone, then assume UTC because it's at least a known error.
+ * If logdate is missing or not parsable, assume the inventory is out of range (it will likely
+ * be refused further one).
+ */
+ override def apply(inventory: NodeSeq): IOResult[NodeSeq] = {
+ val now = OffsetDateTime.now()
+ val date = PreInventoryParserCheckInventoryAge.extracDateValue(inventory)
+
+ (for {
+ d <- PreInventoryParserCheckInventoryAge.parseInventoryDate(date)
+ _ <- PreInventoryParserCheckInventoryAge.checkDate(d, now, maxBeforeNow, maxAfterNow)
+ } yield inventory).toIO
+ }
+
+}
+
+object PreInventoryParserCheckInventoryAge {
+
+ val dateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+ val offsetDateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssZ")
+
+ def extracDateValue(inventory: NodeSeq): String = {
+ val logdate = optText(inventory \\ "ACCESSLOG" \ "LOGDATE").getOrElse("")
+ val timezone = optText(inventory \\ "OPERATINGSYSTEM" \ "TIMEZONE" \ "OFFSET").getOrElse("")
+
+ logdate + timezone
+ }
+
+ // try to parse the inventory date in format YYYY-MM-dd HH:mm:ssZ
+ // ex UTC: 2023-11-16 10:46:35+0000
+ // ex Europe/Paris: 2023-11-16 11:46:12+0100
+ def parseOffsetDateTime(date: String): PureResult[OffsetDateTime] = {
+ PureResult.attempt(s"Error when parsing date '${date}' as a time-zoned date with offset")(
+ OffsetDateTime.parse(date, offsetDateTimeFormat)
+ )
+ }
+
+ // try to parse the inventory date in format: YYYY-MM-dd HH:mm:ss
+ // Since in that case, we don't have timezone information, we need to chose arbitrary,
+ // and we chose UTC so that it's easier to identify problem when they arise.
+ def parseLocalDateTime(date: String): PureResult[OffsetDateTime] = {
+ PureResult
+ .attempt(s"Error when parsing date '${date}' as a local date time")(
+ LocalDateTime.parse(date, dateTimeFormat)
+ )
+ .map(OffsetDateTime.of(_, ZoneOffset.UTC))
+ }
+
+ def parseInventoryDate(date: String): PureResult[OffsetDateTime] = {
+ parseOffsetDateTime(date) match {
+ case Left(_) => parseLocalDateTime(date)
+ case Right(d) => Right(d)
+ }
+ }
+
+ // check that date is in window between [now-maxBeforeNow, now+maxAfterNow]
+ def checkDate(date: OffsetDateTime, now: OffsetDateTime, maxBeforeNow: Duration, maxAfterNow: Duration): PureResult[Unit] = {
+ val pastLimit = now.minus(maxBeforeNow)
+ val futureLimit = now.plus(maxAfterNow)
+
+ val beforePastLimit = Inconsistency(s"Inventory is too old, refusing (inventory date is before '${pastLimit.toString}')")
+ val afterFutureLimit = Inconsistency(
+ s"Inventory is too far in the future, refusing (inventory date is after '${futureLimit.toString}')"
+ )
+
+ (date.isBefore(pastLimit), date.isAfter(futureLimit)) match {
+ case (false, false) => Right(())
+ case (true, _) => Left(beforePastLimit)
+ case (_, true) => Left(afterFutureLimit)
+ }
+ }
+}
diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala
new file mode 100644
index 00000000000..56e382d37ee
--- /dev/null
+++ b/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala
@@ -0,0 +1,97 @@
+/*
+ *************************************************************************************
+ * Copyright 2024 Normation SAS
+ *************************************************************************************
+ *
+ * This file is part of Rudder.
+ *
+ * Rudder is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * In accordance with the terms of section 7 (7. Additional Terms.) of
+ * the GNU General Public License version 3, the copyright holders add
+ * the following Additional permissions:
+ * Notwithstanding to the terms of section 5 (5. Conveying Modified Source
+ * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General
+ * Public License version 3, when you create a Related Module, this
+ * Related Module is not considered as a part of the work and may be
+ * distributed under the license agreement of your choice.
+ * A "Related Module" means a set of sources files including their
+ * documentation that, without modification of the Source Code, enables
+ * supplementary functions or services in addition to those offered by
+ * the Software.
+ *
+ * Rudder is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rudder. If not, see .
+ *
+ *************************************************************************************
+ */
+
+package com.normation.inventory.provisioning.fusion
+
+import better.files.Resource
+import com.normation.inventory.provisioning.fusion.PreInventoryParserCheckInventoryAge.*
+import com.normation.zio.*
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
+import org.junit.runner.*
+import org.specs2.mutable.*
+import org.specs2.runner.*
+import scala.xml.XML
+import zio.*
+
+@RunWith(classOf[JUnitRunner])
+class TestPreParsingDate extends Specification {
+
+ val now = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC)
+ val centos8 = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC)
+ val maxBefore = 1.day
+ val maxAfter = 2.hours
+ val okDate = centos8
+ val before = now.minus(maxBefore).minusSeconds(1)
+ val after = now.plus(maxAfter).plusSeconds(1)
+
+ "Pre inventory check age" should {
+ "be able to extract a date without timezone info in inventory" in {
+ val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs"))
+ val v = extracDateValue(xml)
+ val date = parseInventoryDate(v)
+ (v === "2023-11-16 11:26:18") and
+ (date must beRight[OffsetDateTime](beEqualTo(centos8)))
+ }
+ "be able to extract a date timezone info in inventory" in {
+ val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/sles15sp4.ocs"))
+ val v = extracDateValue(xml)
+ val date = parseInventoryDate(v)
+ (v === "2023-11-16 11:46:12+0100") and
+ (date must beRight[OffsetDateTime](beEqualTo(OffsetDateTime.of(2023, 11, 16, 11, 46, 12, 0, ZoneOffset.ofHours(1)))))
+ }
+ "accept date in range" in {
+ checkDate(okDate, now, maxBefore, maxAfter) === Right(())
+ }
+ "reject date before" in {
+ checkDate(before, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft(
+ matching(s".*Inventory is too old.*")
+ )
+ }
+ "reject date after" in {
+ checkDate(after, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft(
+ matching(s".*Inventory is too far in the future.*")
+ )
+ }
+ "reject centos8 inventory from 2023 (testing full process)" in {
+ val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs"))
+ val checker = new PreInventoryParserCheckInventoryAge(maxBefore, maxAfter)
+ ZioRuntime.unsafeRun(checker(xml).either).left.map(_.fullMsg) must beLeft(
+ matching(s".*Inventory is too old.*")
+ )
+ }
+ }
+}
diff --git a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample
index a5bd6c7eacb..8f08e9b9652 100644
--- a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample
+++ b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample
@@ -243,7 +243,16 @@ inventories.watcher.period.garbage.old=5 minutes
# trying to add them. This can be important if Rudder was down for
# a long time and inventories accumulated, no need to keep the older.
#
-inventories.watcher.max.age.before.deletion=3 days
+inventories.watcher.max.age.before.deletion=2 days
+
+#
+# To prevent data corruption from server sending inventories with unexpected date,
+# either in the past (for example because an old inventory was blocked in some pipe
+# somewhere), or in the future (for example because server's clock is incorrectly set),
+# Rudder refuse inventories out of following range:
+#
+inventories.reject.maxAgeBeforeNow=2 days
+inventories.reject.maxAgeAfterNow=12 hours
#
# You may want to limit the number of inventory files parsed and save in parallel.
@@ -262,6 +271,7 @@ inventory.parse.parallelization=2
# In that case, node's processes will appear to be empty in node details.
# Boolean, default to "false" (ie: processes are parsed and displayed)
inventory.parse.ignore.processes=false
+
#
# In some case, cause performance problems in rudder web-app
# and they are used so you can't ignore them. In that case, Rudder may need to
@@ -269,15 +279,6 @@ inventory.parse.ignore.processes=false
# above which one dedicated request will be done just for processes.
inventory.threshold.processes.isolatedWrite=1000
-#
-# You can keep exhaustive information about LDAP base modification
-# happening in relation to inventory processing. It is mostly used
-# for debug.
-# You can enable that log by setting the following logger in
-# /opt/rudder/etc/logback.xml file to "trace" level (default "off"):
-#
-#
-ldif.tracelog.rootdir=/var/rudder/inventories/debug
#
# Automatic inventories cleaning
diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala
index f4523d8285b..9fd76cff043 100644
--- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala
+++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala
@@ -859,24 +859,53 @@ object RudderParsedProperties {
)
}
- val LDIF_TRACELOG_ROOT_DIR: String = {
+ // don't parse some elements in inventories: processes
+ val INVENTORIES_IGNORE_PROCESSES: Boolean = {
try {
- config.getString("ldif.tracelog.rootdir")
+ config.getBoolean("inventory.parse.ignore.processes")
} catch {
- case ex: ConfigException => "/var/rudder/inventories/debug"
+ case ex: ConfigException => false
}
}
- // don't parse some elements in inventories: processes
- val INVENTORIES_IGNORE_PROCESSES: Boolean = {
+ val INVENTORIES_MAX_BEFORE_NOW: Duration = {
try {
- config.getBoolean("inventory.parse.ignore.processes")
+ Duration.fromScala(
+ scala.concurrent.duration.Duration.apply(
+ config.getString(
+ "inventories.reject.maxAgeBeforeNow"
+ )
+ )
+ )
} catch {
- case ex: ConfigException => false
+ case ex: Exception =>
+ ApplicationLogger.info(
+ s"Error when reading key: 'inventories.reject.maxAgeBeforeNow', defaulting to 2 days: ${ex.getMessage}"
+ )
+ 2.days
}
}
+
+ val INVENTORIES_MAX_AFTER_NOW: Duration = {
+ try {
+ Duration.fromScala(
+ scala.concurrent.duration.Duration.apply(
+ config.getString(
+ "inventories.reject.maxAgeAfterNow"
+ )
+ )
+ )
+ } catch {
+ case ex: Exception =>
+ ApplicationLogger.info(
+ s"Error when reading key: 'inventories.reject.maxAgeAfterNow', defaulting to 12h: ${ex.getMessage}"
+ )
+ 12.hours
+ }
+ }
+
// the limit above which processes need an individual LDAP write request
- val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int = {
+ val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int = {
try {
config.getInt("inventory.threshold.processes.isolatedWrite")
} catch {
@@ -1878,7 +1907,8 @@ object RudderConfigInit {
new DefaultInventoryParser(
fusionReportParser,
Seq(
- new PreInventoryParserCheckConsistency
+ new PreInventoryParserCheckConsistency,
+ new PreInventoryParserCheckInventoryAge(INVENTORIES_MAX_BEFORE_NOW, INVENTORIES_MAX_AFTER_NOW)
)
)
}