From 171b0c242bdffd2197c4825460f3bacbd3446e9d Mon Sep 17 00:00:00 2001 From: Robin Chalas Date: Tue, 13 Jul 2021 11:28:32 +0200 Subject: [PATCH 1/3] [WIP] Domain log archives --- .../Entity/Archive/FunctionalLogArchive.php | 114 +++++++++++++++ .../log/Entity/Archive/MessageLogArchive.php | 129 +++++++++++++++++ .../log/Entity/Archive/SecurityLogArchive.php | 132 ++++++++++++++++++ 3 files changed, 375 insertions(+) create mode 100644 src/main/log/Entity/Archive/FunctionalLogArchive.php create mode 100644 src/main/log/Entity/Archive/MessageLogArchive.php create mode 100644 src/main/log/Entity/Archive/SecurityLogArchive.php diff --git a/src/main/log/Entity/Archive/FunctionalLogArchive.php b/src/main/log/Entity/Archive/FunctionalLogArchive.php new file mode 100644 index 00000000000..5c7c2591094 --- /dev/null +++ b/src/main/log/Entity/Archive/FunctionalLogArchive.php @@ -0,0 +1,114 @@ +getUser(); + $workspace = $functionalLog->getWorkspace(); + $resource = $functionalLog->getResource(); + + $archive = new self(); + $archive->setUserId($user->getId()); + $archive->setUserUuid($user->getUuid()); + $archive->setUserUsername($user->getUsername()); + $archive->setResourceUuid($resource ? $resource->getUuid() : null); + $archive->setWorkspaceUuid($workspace ? $workspace->getUuid() : null); + + $archive->setDate($functionalLog->getDate()); + $archive->setDetails($functionalLog->getDetails()); + $archive->setEvent($functionalLog->getEvent()); + + return $archive; + } + + public function getUserId(): ?int + { + return $this->userId; + } + + public function setUserId(?int $userId): void + { + $this->userId = $userId; + } + + public function getUserUuid(): ?string + { + return $this->userUuid; + } + + public function setUserUuid(?string $userUuid): void + { + $this->userUuid = $userUuid; + } + + public function getUserUsername(): ?string + { + return $this->userUsername; + } + + public function setUserUsername(?string $userUsername): void + { + $this->userUsername = $userUsername; + } + + public function getWorkspaceUuid(): ?string + { + return $this->workspaceUuid; + } + + public function setWorkspaceUuid(?string $workspaceUuid): void + { + $this->workspaceUuid = $workspaceUuid; + } + + public function getResourceUuid(): ?string + { + return $this->resourceUuid; + } + + public function setResourceUuid(?string $resourceUuid): void + { + $this->resourceUuid = $resourceUuid; + } +} diff --git a/src/main/log/Entity/Archive/MessageLogArchive.php b/src/main/log/Entity/Archive/MessageLogArchive.php new file mode 100644 index 00000000000..2e8fe1adc8f --- /dev/null +++ b/src/main/log/Entity/Archive/MessageLogArchive.php @@ -0,0 +1,129 @@ +getSender(); + $receiver = $messageLog->getReceiver(); + + $archive = new self(); + $archive->setDate($messageLog->getDate()); + $archive->setDetails($messageLog->getDetails()); + $archive->setEvent($messageLog->getEvent()); + $archive->setSenderId($sender->getId()); + $archive->setSenderUuid($sender->getUuid()); + $archive->setSenderUsername($sender->getUsername()); + $archive->setReceiverId($receiver->getId()); + $archive->setReceiverUuid($receiver->getUuid()); + $archive->setReceiverUsername($receiver->getUsername()); + + return $archive; + } + + public function getSenderId(): ?int + { + return $this->senderId; + } + + public function setSenderId(?int $senderId): void + { + $this->senderId = $senderId; + } + + public function getSenderUuid(): ?string + { + return $this->senderUuid; + } + + public function setSenderUuid(?string $senderUuid): void + { + $this->senderUuid = $senderUuid; + } + + public function getSenderUsername(): ?string + { + return $this->senderUsername; + } + + public function setSenderUsername(?string $senderUsername): void + { + $this->senderUsername = $senderUsername; + } + + public function getReceiverId(): ?int + { + return $this->receiverId; + } + + public function setReceiverId(?int $receiverId): void + { + $this->receiverId = $receiverId; + } + + public function getReceiverUuid(): ?string + { + return $this->receiverUuid; + } + + public function setReceiverUuid(?string $receiverUuid): void + { + $this->receiverUuid = $receiverUuid; + } + + public function getReceiverUsername(): ?string + { + return $this->receiverUsername; + } + + public function setReceiverUsername(?string $receiverUsername): void + { + $this->receiverUsername = $receiverUsername; + } +} diff --git a/src/main/log/Entity/Archive/SecurityLogArchive.php b/src/main/log/Entity/Archive/SecurityLogArchive.php new file mode 100644 index 00000000000..3b62ed20b59 --- /dev/null +++ b/src/main/log/Entity/Archive/SecurityLogArchive.php @@ -0,0 +1,132 @@ +getDoer(); + + $archive = new self(); + $archive->setEvent($securityLog->getEvent()); + $archive->setDetails($securityLog->getDetails()); + $archive->setDate($securityLog->getDate()); + + if ($doer) { + $archive->setDoerId($doer->getId()); + $archive->setDoerUuid($doer->getUuid()); + $archive->setDoerUsername($doer->getUsername()); + } + + $archive->setDoerIp($securityLog->getDoerIp()); + $archive->setDoerCountry($securityLog->getCountry()); + $archive->setDoerCity($securityLog->getCity()); + + return $archive; + } + + public function getDoerId(): ?int + { + return $this->doerId; + } + + public function setDoerId(?int $doerId): void + { + $this->doerId = $doerId; + } + + public function getDoerUuid(): ?string + { + return $this->doerUuid; + } + + public function setDoerUuid(?string $doerUuid): void + { + $this->doerUuid = $doerUuid; + } + + public function getDoerUsername(): string + { + return $this->doerUsername; + } + + public function setDoerUsername(?string $doerUsername): void + { + $this->doerUsername = $doerUsername; + } + + public function getDoerIp(): ?string + { + return $this->doerIp; + } + + public function setDoerIp(?string $doerIp): void + { + $this->doerIp = $doerIp; + } + + public function getDoerCountry(): ?string + { + return $this->doerCountry; + } + + public function setDoerCountry(?string $doerCountry): void + { + $this->doerCountry = $doerCountry; + } + + public function getDoerCity(): ?string + { + return $this->doerCity; + } + + public function setDoerCity(?string $doerCity): void + { + $this->doerCity = $doerCity; + } +} From 2b42e6ba65e6a6a1c0a895b1d7c1027af69061cc Mon Sep 17 00:00:00 2001 From: Robin Chalas Date: Thu, 21 Apr 2022 14:22:31 +0200 Subject: [PATCH 2/3] Rotate logs --- src/main/log/Archive/LogRotator.php | 28 ++++++++++++++++++++ src/main/log/Archive/LogRotatorInterface.php | 10 +++++++ 2 files changed, 38 insertions(+) create mode 100644 src/main/log/Archive/LogRotator.php create mode 100644 src/main/log/Archive/LogRotatorInterface.php diff --git a/src/main/log/Archive/LogRotator.php b/src/main/log/Archive/LogRotator.php new file mode 100644 index 00000000000..0fff38700f1 --- /dev/null +++ b/src/main/log/Archive/LogRotator.php @@ -0,0 +1,28 @@ + Date: Thu, 21 Apr 2022 17:50:09 +0200 Subject: [PATCH 3/3] wip --- src/main/log/Archive/LogRotator.php | 75 +++++++++++++++++-- src/main/log/Archive/LogRotatorInterface.php | 4 +- .../Entity/Archive/FunctionalLogArchive.php | 2 + .../log/Entity/Archive/MessageLogArchive.php | 2 + .../log/Entity/Archive/SecurityLogArchive.php | 2 + src/main/log/Entity/FunctionalLog.php | 4 +- src/main/log/Entity/MessageLog.php | 2 +- src/main/log/Entity/SecurityLog.php | 2 +- .../Repository/FunctionalLogRepository.php | 17 +++++ .../log/Repository/LogRepositoryInterface.php | 10 +++ .../log/Repository/LogRepositoryTrait.php | 23 ++++++ .../log/Repository/MessageLogRepository.php | 17 +++++ .../log/Repository/SecurityLogRepository.php | 17 +++++ 13 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 src/main/log/Repository/FunctionalLogRepository.php create mode 100644 src/main/log/Repository/LogRepositoryInterface.php create mode 100644 src/main/log/Repository/LogRepositoryTrait.php create mode 100644 src/main/log/Repository/MessageLogRepository.php create mode 100644 src/main/log/Repository/SecurityLogRepository.php diff --git a/src/main/log/Archive/LogRotator.php b/src/main/log/Archive/LogRotator.php index 0fff38700f1..7fd87b689b6 100644 --- a/src/main/log/Archive/LogRotator.php +++ b/src/main/log/Archive/LogRotator.php @@ -3,26 +3,87 @@ namespace Claroline\LogBundle\Archive; use Claroline\AppBundle\Persistence\ObjectManager; +use Claroline\LogBundle\Entity\AbstractLog; +use Claroline\LogBundle\Entity\Archive\SecurityLogArchive; +use Claroline\LogBundle\Repository\LogRepositoryInterface; +use Doctrine\Common\Collections\Collection; +use Doctrine\DBAL\Connection; class LogRotator implements LogRotatorInterface { - public function __construct(ObjectManager $om) + private $connection; + private $om; + + public function __construct(Connection $connection, ObjectManager $om) { - + $this->connection = $connection; + $this->om = $om; } - public function rotateSecurityLogs(\DateTime $from, \DateTime $to): void + /** + * @param \DateTimeInterface $to The date until when log entries should be archived + * @param class-string $logType The FQCN of the log entity whose entries should be rotated + */ + public function rotateLogs(\DateTimeInterface $to, string $logType): void { + $logRepository = $this->om->getRepository($logType); - } + if (!is_subclass_of($logRepository, LogRepositoryInterface::class)) { + throw new \LogicException(sprintf('The log entity must have a repository implementing "%s", "%s" given.', LogRepositoryInterface::class, get_debug_type($logType))); + } - public function rotateFunctionalLogs(\DateTime $from, \DateTime $to): void - { + // Find all log entries created before $to + + /** @var AbstractLog $log */ + foreach ($qb->getQuery()->getResult() as $log) { + + } + + /** @var AbstractLog $oldestLog */ + $oldestLog = $logs->last(); + + $archiveTableName = $logType::ARCHIVE_TABLE_PREFIX."{$to->format('Ymd')}-{$oldestLog->getDate()->format('Ymd')}"; + + $findLastArchivedLogQuery = "SELECT date FROM {$lastArchiveTable} ORDER BY date DESC LIMIT 1"; + $date = ''; // fixme perform query + $newArchiveTableName = SecurityLogArchive::ARCHIVE_TABLE_PREFIX.$now->getTimestamp(); + $createArchiveTableQuery = <<connection->getSchemaManager()->listTableNames(), + fn($tableName) => str_contains($tableName, $archiveTablePrefix) + ); + + $now = (new \DateTime)->getTimestamp(); + $mostRecentDate = 0; + $lastArchiveTable = null; + + foreach ($archiveTables as $archiveTable) { + $date = substr($archiveTable, strpos($archiveTable, \strlen($archiveTablePrefix) -1); + $archiveTableDate = strtotime($date); + + if ($archiveTableDate > $mostRecentDate && $archiveTableDate < $now) { + $mostRecentDate = $archiveTableDate; + $lastArchiveTable = $archiveTable; + } + } + return $lastArchiveTable ?? null; } } \ No newline at end of file diff --git a/src/main/log/Archive/LogRotatorInterface.php b/src/main/log/Archive/LogRotatorInterface.php index f76858d0108..9755fdd780a 100644 --- a/src/main/log/Archive/LogRotatorInterface.php +++ b/src/main/log/Archive/LogRotatorInterface.php @@ -4,7 +4,5 @@ interface LogRotatorInterface { - public function rotateSecurityLogs(\DateTime $from, \DateTime $to): void; - public function rotateFunctionalLogs(\DateTime $from, \DateTime $to): void; - public function rotateMessageLogs(\DateTime $from, \DateTime to): void; + public function rotateLogs(\DateTimeInterface $interval, string $type): void; } \ No newline at end of file diff --git a/src/main/log/Entity/Archive/FunctionalLogArchive.php b/src/main/log/Entity/Archive/FunctionalLogArchive.php index 5c7c2591094..a17dfd596aa 100644 --- a/src/main/log/Entity/Archive/FunctionalLogArchive.php +++ b/src/main/log/Entity/Archive/FunctionalLogArchive.php @@ -12,6 +12,8 @@ */ class FunctionalLogArchive extends AbstractLog { + public const ARCHIVE_TABLE_PREFIX = 'claro_log_functional_archive_'; + /** * @var string|null * @ORM\Column(type="int", nullable=true) diff --git a/src/main/log/Entity/Archive/MessageLogArchive.php b/src/main/log/Entity/Archive/MessageLogArchive.php index 2e8fe1adc8f..571e1e2a340 100644 --- a/src/main/log/Entity/Archive/MessageLogArchive.php +++ b/src/main/log/Entity/Archive/MessageLogArchive.php @@ -12,6 +12,8 @@ */ class MessageLogArchive extends AbstractLog { + public const ARCHIVE_TABLE_PREFIX = 'claro_log_message_archive_'; + /** * @var int|null * @ORM\Column(type="integer", nullable=true) diff --git a/src/main/log/Entity/Archive/SecurityLogArchive.php b/src/main/log/Entity/Archive/SecurityLogArchive.php index 3b62ed20b59..6eb58112699 100644 --- a/src/main/log/Entity/Archive/SecurityLogArchive.php +++ b/src/main/log/Entity/Archive/SecurityLogArchive.php @@ -12,6 +12,8 @@ */ class SecurityLogArchive extends AbstractLog { + public const ARCHIVE_TABLE_PREFIX = 'claro_log_security_archive_'; + /** * @var int|null * @ORM\Column(type="integer", nullable=true) diff --git a/src/main/log/Entity/FunctionalLog.php b/src/main/log/Entity/FunctionalLog.php index e423c20726b..812a51b396f 100644 --- a/src/main/log/Entity/FunctionalLog.php +++ b/src/main/log/Entity/FunctionalLog.php @@ -8,11 +8,13 @@ use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Entity + * @ORM\Entity(repositoryClass=Claroline\LogBundle\Repository\FunctionnalLogRepository) * @ORM\Table(name="claro_log_functionnal") */ class FunctionalLog extends AbstractLog { + public const ARCHIVE_TABLE_PREFIX = 'claro_log_functional_archive'; + /** * @var User * @ORM\ManyToOne(targetEntity="Claroline\CoreBundle\Entity\User") diff --git a/src/main/log/Entity/MessageLog.php b/src/main/log/Entity/MessageLog.php index 4f145d4630c..f74abb4689e 100644 --- a/src/main/log/Entity/MessageLog.php +++ b/src/main/log/Entity/MessageLog.php @@ -6,7 +6,7 @@ use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Entity + * @ORM\Entity(repositoryClass=Claroline\LogBundle\Repository\MessageLogRepository) * @ORM\Table(name="claro_log_message") */ class MessageLog extends AbstractLog diff --git a/src/main/log/Entity/SecurityLog.php b/src/main/log/Entity/SecurityLog.php index 9cd1117dce3..19ba7932bb7 100644 --- a/src/main/log/Entity/SecurityLog.php +++ b/src/main/log/Entity/SecurityLog.php @@ -6,7 +6,7 @@ use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Entity + * @ORM\Entity(repositoryClass=Claroline\LogBundle\Repository\SecurityLogRepository) * @ORM\Table(name="claro_log_security") */ class SecurityLog extends AbstractLog diff --git a/src/main/log/Repository/FunctionalLogRepository.php b/src/main/log/Repository/FunctionalLogRepository.php new file mode 100644 index 00000000000..5780b972fef --- /dev/null +++ b/src/main/log/Repository/FunctionalLogRepository.php @@ -0,0 +1,17 @@ +createQueryBuilder('log'); + $qb + ->where('log.date < :from') + ->orderBy('date') + ->setParameter(':from', $date); + + /** @var Collection|AbstractLog[] $logs */ + $logs = $qb->getQuery()->getResult(); + + return $logs; + } +} \ No newline at end of file diff --git a/src/main/log/Repository/MessageLogRepository.php b/src/main/log/Repository/MessageLogRepository.php new file mode 100644 index 00000000000..26be676a9e2 --- /dev/null +++ b/src/main/log/Repository/MessageLogRepository.php @@ -0,0 +1,17 @@ +