diff --git a/CHANGELOG.md b/CHANGELOG.md index 1141635..1b2de1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ See [keep a changelog] for information about writing changes to this log. ## [Unreleased] +## [3.0.3] - 2025-07-29 + +* [PR-37](https://github.com/itk-dev/sysstatus/pull/37) + 5050: Optimize db structure and indexes, Add progressbar to import jobs + ## [3.0.2] - 2025-06-25 * [PR-35](https://github.com/itk-dev/sysstatus/pull/35) diff --git a/migrations/Version20250729142154.php b/migrations/Version20250729142154.php new file mode 100644 index 0000000..efc818d --- /dev/null +++ b/migrations/Version20250729142154.php @@ -0,0 +1,65 @@ +addSql(<<<'SQL' + ALTER TABLE report CHANGE sys_status sys_status VARCHAR(255) DEFAULT NULL, CHANGE sys_owner_sub sys_owner_sub VARCHAR(255) DEFAULT NULL + SQL); + $this->addSql(<<<'SQL' + CREATE INDEX archived_status_owner_idx ON report (archived_at, sys_status, sys_owner_sub) + SQL); + $this->addSql(<<<'SQL' + CREATE INDEX sys_internal_id_idx ON report (sys_internal_id) + SQL); + $this->addSql(<<<'SQL' + ALTER TABLE system CHANGE sys_owner_sub sys_owner_sub VARCHAR(255) DEFAULT NULL + SQL); + $this->addSql(<<<'SQL' + CREATE INDEX archived_status_owner_idx ON system (archived_at, sys_status, sys_owner_sub) + SQL); + $this->addSql(<<<'SQL' + CREATE INDEX sys_internal_id_idx ON system (sys_internal_id) + SQL); + } + + public function down(Schema $schema): void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql(<<<'SQL' + DROP INDEX archived_status_owner_idx ON system + SQL); + $this->addSql(<<<'SQL' + DROP INDEX sys_internal_id_idx ON system + SQL); + $this->addSql(<<<'SQL' + ALTER TABLE system CHANGE sys_owner_sub sys_owner_sub LONGTEXT DEFAULT NULL + SQL); + $this->addSql(<<<'SQL' + DROP INDEX archived_status_owner_idx ON report + SQL); + $this->addSql(<<<'SQL' + DROP INDEX sys_internal_id_idx ON report + SQL); + $this->addSql(<<<'SQL' + ALTER TABLE report CHANGE sys_status sys_status LONGTEXT DEFAULT NULL, CHANGE sys_owner_sub sys_owner_sub LONGTEXT DEFAULT NULL + SQL); + } +} diff --git a/src/Command/AbstractImportCommand.php b/src/Command/AbstractImportCommand.php index a88feb2..825ce37 100644 --- a/src/Command/AbstractImportCommand.php +++ b/src/Command/AbstractImportCommand.php @@ -6,6 +6,7 @@ use App\Service\BaseImporter; use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\OutputInterface; abstract class AbstractImportCommand extends Command @@ -22,13 +23,15 @@ public function __construct( * * @throws \Exception */ - protected function import(string $type, string $src, OutputInterface $output): void + protected function import(string $type, string $src, OutputInterface $output, bool $progress = false): void { $success = true; $errorMessage = null; + $progressBar = $progress ? new ProgressBar($output) : null; + try { - $this->importer->import($src); + $this->importer->import($src, $progressBar); } catch (\Exception $e) { $success = false; $errorMessage = $e->getMessage(); diff --git a/src/Command/ReportImportCommand.php b/src/Command/ReportImportCommand.php index 41e1944..3a1b24e 100644 --- a/src/Command/ReportImportCommand.php +++ b/src/Command/ReportImportCommand.php @@ -7,6 +7,7 @@ use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class ReportImportCommand extends AbstractImportCommand @@ -22,6 +23,7 @@ protected function configure(): void ->setName('itstyr:import:report') ->setDescription('Import reports from feed.') ->addArgument('src', InputArgument::REQUIRED, 'The src of the feed.') + ->addOption('progress', 'p', InputOption::VALUE_NONE, 'The src of the feed.') ; } @@ -30,7 +32,7 @@ protected function configure(): void */ protected function execute(InputInterface $input, OutputInterface $output): int { - $this->import(Report::class, $input->getArgument('src'), $output); + $this->import(Report::class, $input->getArgument('src'), $output, $input->getOption('progress')); return 0; } diff --git a/src/Entity/Report.php b/src/Entity/Report.php index 5523f0d..9fd310a 100644 --- a/src/Entity/Report.php +++ b/src/Entity/Report.php @@ -14,6 +14,8 @@ use Gedmo\Timestampable\Traits\TimestampableEntity; #[ORM\Entity(repositoryClass: ReportRepository::class)] +#[ORM\Index(columns: ['archived_at', 'sys_status', 'sys_owner_sub'], name: 'archived_status_owner_idx')] +#[ORM\Index(columns: ['sys_internal_id'], name: 'sys_internal_id_idx')] #[Loggable] class Report implements \Stringable { @@ -108,7 +110,7 @@ class Report implements \Stringable #[ORM\Column(type: Types::DATETIME_MUTABLE, nullable: true)] private ?\DateTimeInterface $sysDateUse = null; - #[ORM\Column(type: Types::TEXT, nullable: true)] + #[ORM\Column(type: Types::STRING, nullable: true)] private ?string $sysStatus = null; #[ORM\Column(type: Types::TEXT, nullable: true)] @@ -132,7 +134,7 @@ class Report implements \Stringable #[ORM\Column(type: Types::TEXT, nullable: true)] private ?string $sysVersion = null; - #[ORM\Column(type: Types::TEXT, nullable: true)] + #[ORM\Column(type: Types::STRING, nullable: true)] private ?string $sysOwnerSub = null; #[ORM\Column(type: Types::TEXT, nullable: true)] diff --git a/src/Entity/System.php b/src/Entity/System.php index 1ae13d3..b36ee00 100644 --- a/src/Entity/System.php +++ b/src/Entity/System.php @@ -14,6 +14,8 @@ use Gedmo\Timestampable\Traits\TimestampableEntity; #[ORM\Entity(repositoryClass: SystemRepository::class)] +#[ORM\Index(columns: ['archived_at', 'sys_status', 'sys_owner_sub'], name: 'archived_status_owner_idx')] +#[ORM\Index(columns: ['sys_internal_id'], name: 'sys_internal_id_idx')] #[Loggable] class System implements \Stringable { @@ -123,7 +125,7 @@ class System implements \Stringable #[ORM\Column(type: Types::TEXT, nullable: true)] private ?string $sysVersion = null; - #[ORM\Column(type: Types::TEXT, nullable: true)] + #[ORM\Column(type: Types::STRING, nullable: true)] private ?string $sysOwnerSub = null; #[ORM\Column(length: 255, nullable: true)] @@ -132,7 +134,7 @@ class System implements \Stringable #[ORM\Column(nullable: true)] private ?int $sysInternalId = null; - #[ORM\Column(length: 255, nullable: true)] + #[ORM\Column(type: Types::STRING, nullable: true)] private ?string $sysStatus = null; #[ORM\Column(name: 'edoc_url', length: 255, nullable: true)] diff --git a/src/Service/ImportInterface.php b/src/Service/ImportInterface.php index 0e1ed91..d1f5b8d 100644 --- a/src/Service/ImportInterface.php +++ b/src/Service/ImportInterface.php @@ -2,6 +2,8 @@ namespace App\Service; +use Symfony\Component\Console\Helper\ProgressBar; + interface ImportInterface { /** @@ -10,5 +12,5 @@ interface ImportInterface * @param string $src * Path to the source */ - public function import(string $src): void; + public function import(string $src, ?ProgressBar $progressBar = null): void; } diff --git a/src/Service/ReportImporter.php b/src/Service/ReportImporter.php index e73dfee..9d86763 100644 --- a/src/Service/ReportImporter.php +++ b/src/Service/ReportImporter.php @@ -3,10 +3,11 @@ namespace App\Service; use App\Entity\Report; +use Symfony\Component\Console\Helper\ProgressBar; class ReportImporter extends BaseImporter { - public function import(string $src): void + public function import(string $src, ?ProgressBar $progressBar = null): void { $systemURL = getenv('SYSTEM_URL'); @@ -18,6 +19,8 @@ public function import(string $src): void return; } + $progressBar?->setMaxSteps(\count($entries)); + // List of ids from Anmeldelsesportalen. $sysInternalIds = []; @@ -99,8 +102,12 @@ public function import(string $src): void $report->setSysOwnerSub($subGroupName); } } + + $progressBar?->advance(); } + $progressBar?->setMessage('Starting archiving ...'); + // Archive reports that no longer exist in anmeldelsesportalen. $this->reportRepository->createQueryBuilder('e') ->update() @@ -112,6 +119,10 @@ public function import(string $src): void ->execute() ; + $progressBar?->setMessage('Flushing ...'); + $this->entityManager->flush(); + + $progressBar?->finish(); } } diff --git a/src/Service/SystemImporter.php b/src/Service/SystemImporter.php index 9b66c4f..c264a8b 100644 --- a/src/Service/SystemImporter.php +++ b/src/Service/SystemImporter.php @@ -8,6 +8,7 @@ use App\Repository\SelfServiceAvailableFromItemRepository; use App\Repository\SystemRepository; use Doctrine\ORM\EntityManagerInterface; +use Symfony\Component\Console\Helper\ProgressBar; class SystemImporter extends BaseImporter { @@ -21,7 +22,7 @@ public function __construct( parent::__construct($reportRepository, $systemRepository, $groupRepository, $entityManager); } - public function import(string $src): void + public function import(string $src, ?ProgressBar $progressBar = null): void { $systemURL = getenv('SYSTEM_URL'); @@ -33,6 +34,8 @@ public function import(string $src): void return; } + $progressBar?->setMaxSteps(\count($entries)); + // List of ids from Systemoversigten. $sysInternalIds = []; @@ -137,10 +140,14 @@ public function import(string $src): void $system->setSysOwnerSub($subGroupName); } } + + $progressBar?->advance(); } // Archive systems that no longer exist in Systemoversigten. + $progressBar?->setMessage('Starting archiving ...'); + $this->systemRepository->createQueryBuilder('e') ->update() ->set('e.archivedAt', ':now') @@ -152,6 +159,10 @@ public function import(string $src): void ->execute() ; + $progressBar?->setMessage('Flushing ...'); + $this->entityManager->flush(); + + $progressBar?->finish(); } }