diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala index be27454b847..2f57eada0ac 100644 --- a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala +++ b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckConsistency.scala @@ -58,8 +58,8 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser { /** * There is a list of variables that MUST be set at that point: - * - an id for the node (it is ok that it may be change latter on the process, for ex. for unusal changes) - * - an hostname + * - an id for the node (it is ok that it may be change later on the process, for ex. for unusual changes) + * - a hostname * - an admin (root user on the node) * - a policy server id * - an OS name @@ -85,7 +85,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser { ZIO.foldLeft(checks)(inventory)((currentInventory, check) => check(currentInventory)) } - // Utilitary method to get only once the RUDDER and the AGENT + // Utility method to get only once the RUDDER and the AGENT private def getInTags(xml: NodeSeq, tag: String): NodeSeq = { xml \\ tag } @@ -200,7 +200,7 @@ class PreInventoryParserCheckConsistency extends PreInventoryParser { /** * Kernel version: either - * - non empty OPERATINGSYSTEM > KERNEL_VERSION + * - non-empty OPERATINGSYSTEM > KERNEL_VERSION * or * - (on AIX and non empty HARDWARE > OSVERSION ) * Other cases are failure (missing required info) diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala new file mode 100644 index 00000000000..58f48e8fec6 --- /dev/null +++ b/webapp/sources/ldap-inventory/inventory-fusion/src/main/scala/com/normation/inventory/provisioning/fusion/PreInventoryParserCheckInventoryAge.scala @@ -0,0 +1,131 @@ +/* + ************************************************************************************* + * Copyright 2024 Normation SAS + ************************************************************************************* + * + * This file is part of Rudder. + * + * Rudder is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In accordance with the terms of section 7 (7. Additional Terms.) of + * the GNU General Public License version 3, the copyright holders add + * the following Additional permissions: + * Notwithstanding to the terms of section 5 (5. Conveying Modified Source + * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General + * Public License version 3, when you create a Related Module, this + * Related Module is not considered as a part of the work and may be + * distributed under the license agreement of your choice. + * A "Related Module" means a set of sources files including their + * documentation that, without modification of the Source Code, enables + * supplementary functions or services in addition to those offered by + * the Software. + * + * Rudder is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rudder. If not, see . + + * + ************************************************************************************* + */ + +package com.normation.inventory.provisioning.fusion + +import com.normation.errors.* +import com.normation.inventory.provisioning.fusion.OptText.optText +import com.normation.inventory.services.provisioning.* +import java.time.* +import java.time.format.DateTimeFormatter +import scala.xml.NodeSeq +import zio.* + +class PreInventoryParserCheckInventoryAge(maxBeforeNow: Duration, maxAfterNow: Duration) extends PreInventoryParser { + override val name = "post_process_inventory:check_inventory_age" + + /** + * We don't want to accept inventories; + * - older than maxBeforeNow + * - more in the future than maxAfterNow + * + * Age is deducted from the attribute: + * - ACCESSLOG > LOGDATE, which is a datetime in local server time, without timezone. + * - OPERATINGSYSTEM > TIMEZONE > TIMEZONE that may not be there, for more fun. + * + * If we are not able to parse the timezone, then assume UTC because it's at least a known error. + * If logdate is missing or not parsable, assume the inventory is out of range (it will likely + * be refused further one). + */ + override def apply(inventory: NodeSeq): IOResult[NodeSeq] = { + val now = OffsetDateTime.now() + val date = PreInventoryParserCheckInventoryAge.extracDateValue(inventory) + + (for { + d <- PreInventoryParserCheckInventoryAge.parseInventoryDate(date) + _ <- PreInventoryParserCheckInventoryAge.checkDate(d, now, maxBeforeNow, maxAfterNow) + } yield inventory).toIO + } + +} + +object PreInventoryParserCheckInventoryAge { + + val dateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") + val offsetDateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssZ") + + def extracDateValue(inventory: NodeSeq): String = { + val logdate = optText(inventory \\ "ACCESSLOG" \ "LOGDATE").getOrElse("") + val timezone = optText(inventory \\ "OPERATINGSYSTEM" \ "TIMEZONE" \ "OFFSET").getOrElse("") + + logdate + timezone + } + + // try to parse the inventory date in format YYYY-MM-dd HH:mm:ssZ + // ex UTC: 2023-11-16 10:46:35+0000 + // ex Europe/Paris: 2023-11-16 11:46:12+0100 + def parseOffsetDateTime(date: String): PureResult[OffsetDateTime] = { + PureResult.attempt(s"Error when parsing date '${date}' as a time-zoned date with offset")( + OffsetDateTime.parse(date, offsetDateTimeFormat) + ) + } + + // try to parse the inventory date in format: YYYY-MM-dd HH:mm:ss + // Since in that case, we don't have timezone information, we need to chose arbitrary, + // and we chose UTC so that it's easier to identify problem when they arise. + def parseLocalDateTime(date: String): PureResult[OffsetDateTime] = { + PureResult + .attempt(s"Error when parsing date '${date}' as a local date time")( + LocalDateTime.parse(date, dateTimeFormat) + ) + .map(OffsetDateTime.of(_, ZoneOffset.UTC)) + } + + def parseInventoryDate(date: String): PureResult[OffsetDateTime] = { + parseOffsetDateTime(date) match { + case Left(_) => parseLocalDateTime(date) + case Right(d) => Right(d) + } + } + + // check that date is in window between [now-maxBeforeNow, now+maxAfterNow] + def checkDate(date: OffsetDateTime, now: OffsetDateTime, maxBeforeNow: Duration, maxAfterNow: Duration): PureResult[Unit] = { + val pastLimit = now.minus(maxBeforeNow) + val futureLimit = now.plus(maxAfterNow) + + val beforePastLimit = Inconsistency(s"Inventory is too old, refusing (inventory date is before '${pastLimit.toString}')") + val afterFutureLimit = Inconsistency( + s"Inventory is too far in the future, refusing (inventory date is after '${futureLimit.toString}')" + ) + + (date.isBefore(pastLimit), date.isAfter(futureLimit)) match { + case (false, false) => Right(()) + case (true, _) => Left(beforePastLimit) + case (_, true) => Left(afterFutureLimit) + } + } +} diff --git a/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala b/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala new file mode 100644 index 00000000000..56e382d37ee --- /dev/null +++ b/webapp/sources/ldap-inventory/inventory-fusion/src/test/scala/com/normation/inventory/provisioning/fusion/TestPreParsingDate.scala @@ -0,0 +1,97 @@ +/* + ************************************************************************************* + * Copyright 2024 Normation SAS + ************************************************************************************* + * + * This file is part of Rudder. + * + * Rudder is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In accordance with the terms of section 7 (7. Additional Terms.) of + * the GNU General Public License version 3, the copyright holders add + * the following Additional permissions: + * Notwithstanding to the terms of section 5 (5. Conveying Modified Source + * Versions) and 6 (6. Conveying Non-Source Forms.) of the GNU General + * Public License version 3, when you create a Related Module, this + * Related Module is not considered as a part of the work and may be + * distributed under the license agreement of your choice. + * A "Related Module" means a set of sources files including their + * documentation that, without modification of the Source Code, enables + * supplementary functions or services in addition to those offered by + * the Software. + * + * Rudder is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rudder. If not, see . + * + ************************************************************************************* + */ + +package com.normation.inventory.provisioning.fusion + +import better.files.Resource +import com.normation.inventory.provisioning.fusion.PreInventoryParserCheckInventoryAge.* +import com.normation.zio.* +import java.time.OffsetDateTime +import java.time.ZoneOffset +import org.junit.runner.* +import org.specs2.mutable.* +import org.specs2.runner.* +import scala.xml.XML +import zio.* + +@RunWith(classOf[JUnitRunner]) +class TestPreParsingDate extends Specification { + + val now = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC) + val centos8 = OffsetDateTime.of(2023, 11, 16, 11, 26, 18, 0, ZoneOffset.UTC) + val maxBefore = 1.day + val maxAfter = 2.hours + val okDate = centos8 + val before = now.minus(maxBefore).minusSeconds(1) + val after = now.plus(maxAfter).plusSeconds(1) + + "Pre inventory check age" should { + "be able to extract a date without timezone info in inventory" in { + val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs")) + val v = extracDateValue(xml) + val date = parseInventoryDate(v) + (v === "2023-11-16 11:26:18") and + (date must beRight[OffsetDateTime](beEqualTo(centos8))) + } + "be able to extract a date timezone info in inventory" in { + val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/sles15sp4.ocs")) + val v = extracDateValue(xml) + val date = parseInventoryDate(v) + (v === "2023-11-16 11:46:12+0100") and + (date must beRight[OffsetDateTime](beEqualTo(OffsetDateTime.of(2023, 11, 16, 11, 46, 12, 0, ZoneOffset.ofHours(1))))) + } + "accept date in range" in { + checkDate(okDate, now, maxBefore, maxAfter) === Right(()) + } + "reject date before" in { + checkDate(before, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft( + matching(s".*Inventory is too old.*") + ) + } + "reject date after" in { + checkDate(after, now, maxBefore, maxAfter).left.map(_.fullMsg) must beLeft( + matching(s".*Inventory is too far in the future.*") + ) + } + "reject centos8 inventory from 2023 (testing full process)" in { + val xml = XML.load(Resource.getAsStream("fusion-inventories/8.0/centos8.ocs")) + val checker = new PreInventoryParserCheckInventoryAge(maxBefore, maxAfter) + ZioRuntime.unsafeRun(checker(xml).either).left.map(_.fullMsg) must beLeft( + matching(s".*Inventory is too old.*") + ) + } + } +} diff --git a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample index a5bd6c7eacb..8f08e9b9652 100644 --- a/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample +++ b/webapp/sources/rudder/rudder-web/src/main/resources/configuration.properties.sample @@ -243,7 +243,16 @@ inventories.watcher.period.garbage.old=5 minutes # trying to add them. This can be important if Rudder was down for # a long time and inventories accumulated, no need to keep the older. # -inventories.watcher.max.age.before.deletion=3 days +inventories.watcher.max.age.before.deletion=2 days + +# +# To prevent data corruption from server sending inventories with unexpected date, +# either in the past (for example because an old inventory was blocked in some pipe +# somewhere), or in the future (for example because server's clock is incorrectly set), +# Rudder refuse inventories out of following range: +# +inventories.reject.maxAgeBeforeNow=2 days +inventories.reject.maxAgeAfterNow=12 hours # # You may want to limit the number of inventory files parsed and save in parallel. @@ -262,6 +271,7 @@ inventory.parse.parallelization=2 # In that case, node's processes will appear to be empty in node details. # Boolean, default to "false" (ie: processes are parsed and displayed) inventory.parse.ignore.processes=false + # # In some case, cause performance problems in rudder web-app # and they are used so you can't ignore them. In that case, Rudder may need to @@ -269,15 +279,6 @@ inventory.parse.ignore.processes=false # above which one dedicated request will be done just for processes. inventory.threshold.processes.isolatedWrite=1000 -# -# You can keep exhaustive information about LDAP base modification -# happening in relation to inventory processing. It is mostly used -# for debug. -# You can enable that log by setting the following logger in -# /opt/rudder/etc/logback.xml file to "trace" level (default "off"): -# -# -ldif.tracelog.rootdir=/var/rudder/inventories/debug # # Automatic inventories cleaning diff --git a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala index f4523d8285b..9fd76cff043 100644 --- a/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala +++ b/webapp/sources/rudder/rudder-web/src/main/scala/bootstrap/liftweb/RudderConfig.scala @@ -859,24 +859,53 @@ object RudderParsedProperties { ) } - val LDIF_TRACELOG_ROOT_DIR: String = { + // don't parse some elements in inventories: processes + val INVENTORIES_IGNORE_PROCESSES: Boolean = { try { - config.getString("ldif.tracelog.rootdir") + config.getBoolean("inventory.parse.ignore.processes") } catch { - case ex: ConfigException => "/var/rudder/inventories/debug" + case ex: ConfigException => false } } - // don't parse some elements in inventories: processes - val INVENTORIES_IGNORE_PROCESSES: Boolean = { + val INVENTORIES_MAX_BEFORE_NOW: Duration = { try { - config.getBoolean("inventory.parse.ignore.processes") + Duration.fromScala( + scala.concurrent.duration.Duration.apply( + config.getString( + "inventories.reject.maxAgeBeforeNow" + ) + ) + ) } catch { - case ex: ConfigException => false + case ex: Exception => + ApplicationLogger.info( + s"Error when reading key: 'inventories.reject.maxAgeBeforeNow', defaulting to 2 days: ${ex.getMessage}" + ) + 2.days } } + + val INVENTORIES_MAX_AFTER_NOW: Duration = { + try { + Duration.fromScala( + scala.concurrent.duration.Duration.apply( + config.getString( + "inventories.reject.maxAgeAfterNow" + ) + ) + ) + } catch { + case ex: Exception => + ApplicationLogger.info( + s"Error when reading key: 'inventories.reject.maxAgeAfterNow', defaulting to 12h: ${ex.getMessage}" + ) + 12.hours + } + } + // the limit above which processes need an individual LDAP write request - val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int = { + val INVENTORIES_THRESHOLD_PROCESSES_ISOLATED_WRITE: Int = { try { config.getInt("inventory.threshold.processes.isolatedWrite") } catch { @@ -1878,7 +1907,8 @@ object RudderConfigInit { new DefaultInventoryParser( fusionReportParser, Seq( - new PreInventoryParserCheckConsistency + new PreInventoryParserCheckConsistency, + new PreInventoryParserCheckInventoryAge(INVENTORIES_MAX_BEFORE_NOW, INVENTORIES_MAX_AFTER_NOW) ) ) }