Hotfix/3.0.3

CHANGELOG.md

@@ -8,6 +8,11 @@ See [keep a changelog] for information about writing changes to this log.
 
 ## [Unreleased]
 
+## [3.0.3] - 2025-07-29
+
+* [PR-37](https://github.com/itk-dev/sysstatus/pull/37)
+  5050: Optimize db structure and indexes, Add progressbar to import jobs
+
 ## [3.0.2] - 2025-06-25
 
 * [PR-35](https://github.com/itk-dev/sysstatus/pull/35)

migrations/Version20250729142154.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20250729142154 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema): void
+    {
+        // this up() migration is auto-generated, please modify it to your needs
+        $this->addSql(<<<'SQL'
+            ALTER TABLE report CHANGE sys_status sys_status VARCHAR(255) DEFAULT NULL, CHANGE sys_owner_sub sys_owner_sub VARCHAR(255) DEFAULT NULL
+        SQL);
+        $this->addSql(<<<'SQL'
+            CREATE INDEX archived_status_owner_idx ON report (archived_at, sys_status, sys_owner_sub)
+        SQL);
+        $this->addSql(<<<'SQL'
+            CREATE INDEX sys_internal_id_idx ON report (sys_internal_id)
+        SQL);
+        $this->addSql(<<<'SQL'
+            ALTER TABLE system CHANGE sys_owner_sub sys_owner_sub VARCHAR(255) DEFAULT NULL
+        SQL);
+        $this->addSql(<<<'SQL'
+            CREATE INDEX archived_status_owner_idx ON system (archived_at, sys_status, sys_owner_sub)
+        SQL);
+        $this->addSql(<<<'SQL'
+            CREATE INDEX sys_internal_id_idx ON system (sys_internal_id)
+        SQL);
+    }
+
+    public function down(Schema $schema): void
+    {
+        // this down() migration is auto-generated, please modify it to your needs
+        $this->addSql(<<<'SQL'
+            DROP INDEX archived_status_owner_idx ON system
+        SQL);
+        $this->addSql(<<<'SQL'
+            DROP INDEX sys_internal_id_idx ON system
+        SQL);
+        $this->addSql(<<<'SQL'
+            ALTER TABLE system CHANGE sys_owner_sub sys_owner_sub LONGTEXT DEFAULT NULL
+        SQL);
+        $this->addSql(<<<'SQL'
+            DROP INDEX archived_status_owner_idx ON report
+        SQL);
+        $this->addSql(<<<'SQL'
+            DROP INDEX sys_internal_id_idx ON report
+        SQL);
+        $this->addSql(<<<'SQL'
+            ALTER TABLE report CHANGE sys_status sys_status LONGTEXT DEFAULT NULL, CHANGE sys_owner_sub sys_owner_sub LONGTEXT DEFAULT NULL
+        SQL);
+    }
+}

src/Command/AbstractImportCommand.php

@@ -6,6 +6,7 @@
 use App\Service\BaseImporter;
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Output\OutputInterface;
 
 abstract class AbstractImportCommand extends Command
@@ -22,13 +23,15 @@ public function __construct(
      *
      * @throws \Exception
      */
-    protected function import(string $type, string $src, OutputInterface $output): void
+    protected function import(string $type, string $src, OutputInterface $output, bool $progress = false): void
     {
         $success = true;
         $errorMessage = null;
 
+        $progressBar = $progress ? new ProgressBar($output) : null;
+
         try {
-            $this->importer->import($src);
+            $this->importer->import($src, $progressBar);
         } catch (\Exception $e) {
             $success = false;
             $errorMessage = $e->getMessage();

src/Command/ReportImportCommand.php

@@ -7,6 +7,7 @@
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class ReportImportCommand extends AbstractImportCommand
@@ -22,6 +23,7 @@ protected function configure(): void
             ->setName('itstyr:import:report')
             ->setDescription('Import reports from feed.')
             ->addArgument('src', InputArgument::REQUIRED, 'The src of the feed.')
+            ->addOption('progress', 'p', InputOption::VALUE_NONE, 'The src of the feed.')
         ;
     }
 
@@ -30,7 +32,7 @@ protected function configure(): void
      */
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        $this->import(Report::class, $input->getArgument('src'), $output);
+        $this->import(Report::class, $input->getArgument('src'), $output, $input->getOption('progress'));
 
         return 0;
     }

src/Entity/Report.php

@@ -14,6 +14,8 @@
 use Gedmo\Timestampable\Traits\TimestampableEntity;
 
 #[ORM\Entity(repositoryClass: ReportRepository::class)]
+#[ORM\Index(columns: ['archived_at', 'sys_status', 'sys_owner_sub'], name: 'archived_status_owner_idx')]
+#[ORM\Index(columns: ['sys_internal_id'], name: 'sys_internal_id_idx')]
 #[Loggable]
 class Report implements \Stringable
 {
@@ -108,7 +110,7 @@ class Report implements \Stringable
     #[ORM\Column(type: Types::DATETIME_MUTABLE, nullable: true)]
     private ?\DateTimeInterface $sysDateUse = null;
 
-    #[ORM\Column(type: Types::TEXT, nullable: true)]
+    #[ORM\Column(type: Types::STRING, nullable: true)]
     private ?string $sysStatus = null;
 
     #[ORM\Column(type: Types::TEXT, nullable: true)]
@@ -132,7 +134,7 @@ class Report implements \Stringable
     #[ORM\Column(type: Types::TEXT, nullable: true)]
     private ?string $sysVersion = null;
 
-    #[ORM\Column(type: Types::TEXT, nullable: true)]
+    #[ORM\Column(type: Types::STRING, nullable: true)]
     private ?string $sysOwnerSub = null;
 
     #[ORM\Column(type: Types::TEXT, nullable: true)]

src/Entity/System.php

@@ -14,6 +14,8 @@
 use Gedmo\Timestampable\Traits\TimestampableEntity;
 
 #[ORM\Entity(repositoryClass: SystemRepository::class)]
+#[ORM\Index(columns: ['archived_at', 'sys_status', 'sys_owner_sub'], name: 'archived_status_owner_idx')]
+#[ORM\Index(columns: ['sys_internal_id'], name: 'sys_internal_id_idx')]
 #[Loggable]
 class System implements \Stringable
 {
@@ -123,7 +125,7 @@ class System implements \Stringable
     #[ORM\Column(type: Types::TEXT, nullable: true)]
     private ?string $sysVersion = null;
 
-    #[ORM\Column(type: Types::TEXT, nullable: true)]
+    #[ORM\Column(type: Types::STRING, nullable: true)]
     private ?string $sysOwnerSub = null;
 
     #[ORM\Column(length: 255, nullable: true)]
@@ -132,7 +134,7 @@ class System implements \Stringable
     #[ORM\Column(nullable: true)]
     private ?int $sysInternalId = null;
 
-    #[ORM\Column(length: 255, nullable: true)]
+    #[ORM\Column(type: Types::STRING, nullable: true)]
     private ?string $sysStatus = null;
 
     #[ORM\Column(name: 'edoc_url', length: 255, nullable: true)]

src/Service/ImportInterface.php

@@ -2,6 +2,8 @@
 
 namespace App\Service;
 
+use Symfony\Component\Console\Helper\ProgressBar;
+
 interface ImportInterface
 {
     /**
@@ -10,5 +12,5 @@ interface ImportInterface
      * @param string $src
      *   Path to the source
      */
-    public function import(string $src): void;
+    public function import(string $src, ?ProgressBar $progressBar = null): void;
 }

src/Service/ReportImporter.php

@@ -3,10 +3,11 @@
 namespace App\Service;
 
 use App\Entity\Report;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 class ReportImporter extends BaseImporter
 {
-    public function import(string $src): void
+    public function import(string $src, ?ProgressBar $progressBar = null): void
     {
         $systemURL = getenv('SYSTEM_URL');
 
@@ -18,6 +19,8 @@ public function import(string $src): void
             return;
         }
 
+        $progressBar?->setMaxSteps(\count($entries));
+
         // List of ids from Anmeldelsesportalen.
         $sysInternalIds = [];
 
@@ -99,8 +102,12 @@ public function import(string $src): void
                     $report->setSysOwnerSub($subGroupName);
                 }
             }
+
+            $progressBar?->advance();
         }
 
+        $progressBar?->setMessage('Starting archiving ...');
+
         // Archive reports that no longer exist in anmeldelsesportalen.
         $this->reportRepository->createQueryBuilder('e')
             ->update()
@@ -112,6 +119,10 @@ public function import(string $src): void
             ->execute()
         ;
 
+        $progressBar?->setMessage('Flushing ...');
+
         $this->entityManager->flush();
+
+        $progressBar?->finish();
     }
 }

src/Service/SystemImporter.php

@@ -8,6 +8,7 @@
 use App\Repository\SelfServiceAvailableFromItemRepository;
 use App\Repository\SystemRepository;
 use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 class SystemImporter extends BaseImporter
 {
@@ -21,7 +22,7 @@ public function __construct(
         parent::__construct($reportRepository, $systemRepository, $groupRepository, $entityManager);
     }
 
-    public function import(string $src): void
+    public function import(string $src, ?ProgressBar $progressBar = null): void
     {
         $systemURL = getenv('SYSTEM_URL');
 
@@ -33,6 +34,8 @@ public function import(string $src): void
             return;
         }
 
+        $progressBar?->setMaxSteps(\count($entries));
+
         // List of ids from Systemoversigten.
         $sysInternalIds = [];
 
@@ -137,10 +140,14 @@ public function import(string $src): void
                     $system->setSysOwnerSub($subGroupName);
                 }
             }
+
+            $progressBar?->advance();
         }
 
         // Archive systems that no longer exist in Systemoversigten.
 
+        $progressBar?->setMessage('Starting archiving ...');
+
         $this->systemRepository->createQueryBuilder('e')
             ->update()
             ->set('e.archivedAt', ':now')
@@ -152,6 +159,10 @@ public function import(string $src): void
             ->execute()
         ;
 
+        $progressBar?->setMessage('Flushing ...');
+
         $this->entityManager->flush();
+
+        $progressBar?->finish();
     }
 }