Fixes #25996: Refuse inventory too far from \"now\"

webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala

@@ -58,8 +58,8 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
 
   /**
    * There is a list of variables that MUST be set at that point:
-   * - an id for the node (it is ok that it may be change latter on the process, for ex. for unusal changes)
-   * - an hostname
+   * - an id for the node (it is ok that it may be change later on the process, for ex. for unusual changes)
+   * - a hostname
    * - an admin (root user on the node)
    * - a policy server id
    * - an OS name
@@ -85,7 +85,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
     ZIO.foldLeft(checks)(inventory)((currentInventory, check) => check(currentInventory))
   }
 
-  // Utilitary method to get only once the RUDDER and the AGENT
+  // Utility method to get only once the RUDDER and the AGENT
   private def getInTags(xml: NodeSeq, tag: String): NodeSeq = {
     xml \\ tag
   }
@@ -200,7 +200,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser {
 
   /**
    * Kernel version: either
-   * - non empty OPERATINGSYSTEM > KERNEL_VERSION
+   * - non-empty OPERATINGSYSTEM > KERNEL_VERSION
    * or
    * - (on AIX and non empty HARDWARE > OSVERSION )
    * Other cases are failure (missing required info)

webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala

@@ -0,0 +1,131 @@
+/*
+ *************************************************************************************
+ * Copyright 2024 Normation SAS
+ *************************************************************************************
+ *
+ * This file is part of Rudder.
+ *
+ * Rudder is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * In accordance with the terms of section 7 (7. Additional Terms.) of
+ * the GNU General Public License version 3, the copyright holders add
+ * the following Additional permissions:
+ * Notwithstanding to the terms of section 5 (5. Conveying Modified Source
+ * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General
+ * Public License version 3, when you create a Related Module, this
+ * Related Module is not considered as a part of the work and may be
+ * distributed under the license agreement of your choice.
+ * A "Related Module" means a set of sources files including their
+ * documentation that, without modification of the Source Code, enables
+ * supplementary functions or services in addition to those offered by
+ * the Software.
+ *
+ * Rudder is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rudder.  If not, see <http://www.gnu.org/licenses/>.
+
+ *
+ *************************************************************************************
+ */
+
+package com.normation.inventory.provisioning.fusion
+
+import com.normation.errors.*
+import com.normation.inventory.provisioning.fusion.OptText.optText
+import com.normation.inventory.services.provisioning.*
+import java.time.*
+import java.time.format.DateTimeFormatter
+import scala.xml.NodeSeq
+import zio.*
+
+class PreInventoryParserCheckInventoryAge(maxBeforeNow: Duration, maxAfterNow: Duration) extends PreInventoryParser {
+  override val name = "post_process_inventory:check_inventory_age"
+
+  /**
+   * We don't want to accept inventories;
+   * - older than maxBeforeNow
+   * - more in the future than maxAfterNow
+   *
+   * Age is deducted from the attribute:
+   * - ACCESSLOG > LOGDATE, which is a datetime in local server time, without timezone.
+   * - OPERATINGSYSTEM > TIMEZONE > TIMEZONE that may not be there, for more fun.
+   *
+   * If we are not able to parse the timezone, then assume UTC because it's at least a known error.
+   * If logdate is missing or not parsable, assume the inventory is out of range (it will likely
+   * be refused further one).
+   */
+  override def apply(inventory: NodeSeq): IOResult[NodeSeq] = {
+    val now  = OffsetDateTime.now()
+    val date = PreInventoryParserCheckInventoryAge.extracDateValue(inventory)
+
+    (for {
+      d <- PreInventoryParserCheckInventoryAge.parseInventoryDate(date)
+      _ <- PreInventoryParserCheckInventoryAge.checkDate(d, now, maxBeforeNow, maxAfterNow)
+    } yield inventory).toIO
+  }
+
+}
+
+object PreInventoryParserCheckInventoryAge {
+
+  val dateTimeFormat       = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+  val offsetDateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssZ")
+
+  def extracDateValue(inventory: NodeSeq): String = {
+    val logdate  = optText(inventory \\ "ACCESSLOG" \ "LOGDATE").getOrElse("")
+    val timezone = optText(inventory \\ "OPERATINGSYSTEM" \ "TIMEZONE" \ "OFFSET").getOrElse("")
+
+    logdate + timezone
+  }
+
+  // try to parse the inventory date in format YYYY-MM-dd HH:mm:ssZ
+  // ex UTC: 2023-11-16 10:46:35+0000
+  // ex Europe/Paris: 2023-11-16 11:46:12+0100
+  def parseOffsetDateTime(date: String): PureResult[OffsetDateTime] = {
+    PureResult.attempt(s"Error when parsing date '${date}' as a time-zoned date with offset")(
+      OffsetDateTime.parse(date, offsetDateTimeFormat)
+    )
+  }
+
+  // try to parse the inventory date in format: YYYY-MM-dd HH:mm:ss
+  // Since in that case, we don't have timezone information, we need to chose arbitrary,
+  // and we chose UTC so that it's easier to identify problem when they arise.
+  def parseLocalDateTime(date: String): PureResult[OffsetDateTime] = {
+    PureResult
+      .attempt(s"Error when parsing date '${date}' as a local date time")(
+        LocalDateTime.parse(date, dateTimeFormat)
+      )
+      .map(OffsetDateTime.of(_, ZoneOffset.UTC))
+  }
+
+  def parseInventoryDate(date: String): PureResult[OffsetDateTime] = {
+    parseOffsetDateTime(date) match {
+      case Left(_)  => parseLocalDateTime(date)
+      case Right(d) => Right(d)
+    }
+  }
+
+  // check that date is in window between [now-maxBeforeNow, now+maxAfterNow]
+  def checkDate(date: OffsetDateTime, now: OffsetDateTime, maxBeforeNow: Duration, maxAfterNow: Duration): PureResult[Unit] = {
+    val pastLimit   = now.minus(maxBeforeNow)
+    val futureLimit = now.plus(maxAfterNow)
+
+    val beforePastLimit  = Inconsistency(s"Inventory is too old, refusing (inventory date is before '${pastLimit.toString}')")
+    val afterFutureLimit = Inconsistency(
+      s"Inventory is too far in the future, refusing (inventory date is after '${futureLimit.toString}')"
+    )
+
+    (date.isBefore(pastLimit), date.isAfter(futureLimit)) match {
+      case (false, false) => Right(())
+      case (true, _)      => Left(beforePastLimit)
+      case (_, true)      => Left(afterFutureLimit)
+    }
+  }
+}

webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala

@@ -0,0 +1,97 @@
+/*
+ *************************************************************************************
+ * Copyright 2024 Normation SAS
+ *************************************************************************************
+ *
+ * This file is part of Rudder.
+ *
+ * Rudder is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * In accordance with the terms of section 7 (7. Additional Terms.) of
+ * the GNU General Public License version 3, the copyright holders add
+ * the following Additional permissions:
+ * Notwithstanding to the terms of section 5 (5. Conveying Modified Source
+ * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General
+ * Public License version 3, when you create a Related Module, this
+ * Related Module is not considered as a part of the work and may be
+ * distributed under the license agreement of your choice.
+ * A "Related Module" means a set of sources files including their
+ * documentation that, without modification of the Source Code, enables
+ * supplementary functions or services in addition to those offered by
+ * the Software.
+ *
+ * Rudder is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rudder.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *************************************************************************************
+ */
+
+package com.normation.inventory.provisioning.fusion
+
+import better.files.Resource
+import com.normation.inventory.provisioning.fusion.PreInventoryParserCheckInventoryAge.*
+import com.normation.zio.*
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
+import org.junit.runner.*
+import org.specs2.mutable.*
+import org.specs2.runner.*
+import scala.xml.XML
+import zio.*
+
+@RunWith(classOf[JUnitRunner])
+class TestPreParsingDate extends Specification {
+
+  val now       = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC)
+  val centos8   = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC)
+  val maxBefore = 1.day
+  val maxAfter  = 2.hours
+  val okDate    = centos8
+  val before    = now.minus(maxBefore).minusSeconds(1)
+  val after     = now.plus(maxAfter).plusSeconds(1)
+
+  "Pre inventory check age" should {
+    "be able to extract a date without timezone info in inventory" in {
+      val xml  = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs"))
+      val v    = extracDateValue(xml)
+      val date = parseInventoryDate(v)
+      (v === "2023-11-16 11:26:18") and
+      (date must beRight[OffsetDateTime](beEqualTo(centos8)))
+    }
+    "be able to extract a date  timezone info in inventory" in {
+      val xml  = XML.load(Resource.getAsStream("fusion-inventories/8.0/sles15sp4.ocs"))
+      val v    = extracDateValue(xml)
+      val date = parseInventoryDate(v)
+      (v === "2023-11-16 11:46:12+0100") and
+      (date must beRight[OffsetDateTime](beEqualTo(OffsetDateTime.of(2023, 11, 16, 11, 46, 12, 0, ZoneOffset.ofHours(1)))))
+    }
+    "accept date in range" in {
+      checkDate(okDate, now, maxBefore, maxAfter) === Right(())
+    }
+    "reject date before" in {
+      checkDate(before, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft(
+        matching(s".*Inventory is too old.*")
+      )
+    }
+    "reject date after" in {
+      checkDate(after, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft(
+        matching(s".*Inventory is too far in the future.*")
+      )
+    }
+    "reject centos8 inventory from 2023 (testing full process)" in {
+      val xml     = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs"))
+      val checker = new PreInventoryParserCheckInventoryAge(maxBefore, maxAfter)
+      ZioRuntime.unsafeRun(checker(xml).either).left.map(_.fullMsg) must beLeft(
+        matching(s".*Inventory is too old.*")
+      )
+    }
+  }
+}

webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample

@@ -243,7 +243,16 @@ inventories.watcher.period.garbage.old=5 minutes
 # trying to add them. This can be important if Rudder was down for
 # a long time and inventories accumulated, no need to keep the older.
 #
-inventories.watcher.max.age.before.deletion=3 days
+inventories.watcher.max.age.before.deletion=2 days
+
+#
+# To prevent data corruption from server sending inventories with unexpected date,
+# either in the past (for example because an old inventory was blocked in some pipe
+# somewhere), or in the future (for example because server's clock is incorrectly set),
+# Rudder refuse inventories out of following range:
+#
+inventories.reject.maxAgeBeforeNow=2 days
+inventories.reject.maxAgeAfterNow=12 hours
 
 #
 # You may want to limit the number of inventory files parsed and save in parallel.
@@ -262,22 +271,14 @@ inventory.parse.parallelization=2
 # In that case, node's processes will appear to be empty in node details.
 # Boolean, default to "false" (ie: processes are parsed and displayed)
 inventory.parse.ignore.processes=false
+
 #
 # In some case, <PROCESSES> cause performance problems in rudder web-app
 # and they are used so you can't ignore them. In that case, Rudder may need to
 # use a different LDAP write request just for them. This is the threshold number of processed
 # above which one dedicated request will be done just for processes.
 inventory.threshold.processes.isolatedWrite=1000
 
-#
-# You can keep exhaustive information about LDAP base modification
-# happening in relation to inventory processing. It is mostly used
-# for debug.
-# You can enable that log by setting the following logger in
-# /opt/rudder/etc/logback.xml file to "trace" level (default "off"):
-# <logger name="trace.ldif.in.file" level="trace" />
-#
-ldif.tracelog.rootdir=/var/rudder/inventories/debug
 
 #
 # Automatic inventories cleaning

webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala

@@ -859,24 +859,53 @@ object RudderParsedProperties {
     )
   }
 
-  val LDIF_TRACELOG_ROOT_DIR: String = {
+  // don't parse some elements in inventories: processes
+  val INVENTORIES_IGNORE_PROCESSES: Boolean = {
     try {
-      config.getString("ldif.tracelog.rootdir")
+      config.getBoolean("inventory.parse.ignore.processes")
     } catch {
-      case ex: ConfigException => "/var/rudder/inventories/debug"
+      case ex: ConfigException => false
     }
   }
 
-  // don't parse some elements in inventories: processes
-  val INVENTORIES_IGNORE_PROCESSES:                   Boolean = {
+  val INVENTORIES_MAX_BEFORE_NOW: Duration = {
     try {
-      config.getBoolean("inventory.parse.ignore.processes")
+      Duration.fromScala(
+        scala.concurrent.duration.Duration.apply(
+          config.getString(
+            "inventories.reject.maxAgeBeforeNow"
+          )
+        )
+      )
     } catch {
-      case ex: ConfigException => false
+      case ex: Exception =>
+        ApplicationLogger.info(
+          s"Error when reading key: 'inventories.reject.maxAgeBeforeNow', defaulting to 2 days: ${ex.getMessage}"
+        )
+        2.days
     }
   }
+
+  val INVENTORIES_MAX_AFTER_NOW: Duration = {
+    try {
+      Duration.fromScala(
+        scala.concurrent.duration.Duration.apply(
+          config.getString(
+            "inventories.reject.maxAgeAfterNow"
+          )
+        )
+      )
+    } catch {
+      case ex: Exception =>
+        ApplicationLogger.info(
+          s"Error when reading key: 'inventories.reject.maxAgeAfterNow', defaulting to 12h: ${ex.getMessage}"
+        )
+        12.hours
+    }
+  }
+
   // the limit above which processes need an individual LDAP write request
-  val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int     = {
+  val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int = {
     try {
       config.getInt("inventory.threshold.processes.isolatedWrite")
     } catch {
@@ -1878,7 +1907,8 @@ object RudderConfigInit {
       new DefaultInventoryParser(
         fusionReportParser,
         Seq(
-          new PreInventoryParserCheckConsistency
+          new PreInventoryParserCheckConsistency,
+          new PreInventoryParserCheckInventoryAge(INVENTORIES_MAX_BEFORE_NOW, INVENTORIES_MAX_AFTER_NOW)
         )
       )
     }