diff --git a/README.md b/README.md
index 07bc602..8cbc5c8 100644
--- a/README.md
+++ b/README.md
@@ -42,13 +42,21 @@ requested before it is pre-generated it will still be shown.
## How does the app work
-1. Listen to events that a file has been written or modified and store it in the database
-2. On cron run request previews for the files that have been written or modified
+1. Listen to events that a file has been written or modified and store it in the database.
+2. Generates previews for the files that have been written or modified in a background job.
+3. Optional: Dedicated occ command to generate previews using a custom schedule (for example, in a
+ separate system cron job).
-If a preview already exists at step 2 then requesting it is really cheap. If not
+If a preview already exists at step 2 (or 3) then requesting it is really cheap. If not
it will be generated. Depending on the sizes of the files and the hardware you
are running on the time this takes can vary.
+By default, the background job to generate previews for modified files is limited to a maximum
+execution time of five minutes. Additionally, it requires using the cron background job mode.
+Webcron and AJAX modes are not supported. The background job is limited to prevent stalling the PHP
+process. The limits are configurable via app configs (see below) or admins can configure a dedicated
+system cron job which runs the `occ preview:pre-generate` command.
+
## Commands
#### `preview:generate-all [--workers=WORKERS] [--path=PATH ...] [user_id ...]`
@@ -101,6 +109,19 @@ the aspect ratio.
Will retain the aspect ratio and use the specified height. The width will be scaled according to
the aspect ratio.
+#### `occ config:app:set --value=false --type=bool previewgenerator job_disabled`
+Set to true to disable the background job that generates previews by default without having to
+configure a manual system cron job. It is recommended to disable the default background job in case
+a custom system cron entry with `occ preview:pre-generate` is configured (set this config to true).
+
+#### `occ config:app:set --value=600 --type=int previewgenerator job_max_execution_time`
+Limits the maximum execution time in seconds of the preview background job. (A value of zero means
+unlimited.)
+
+#### `occ config:app:set --value=0 --type=int previewgenerator job_max_previews`
+Limits the count of previews to be generated in each execution of the preview background job. (A
+value of zero means unlimited.) Configure one, both or no limit (not recommended!). In case both
+limits are configured, the more restrictive one takes precedence.
## FAQ
diff --git a/appinfo/info.xml b/appinfo/info.xml
index 9c3f9ce..422272d 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -15,7 +15,7 @@ The first time you install this app, before using a cron job, you properly want
**Important**: To enable pre-generation of previews you must add **php /var/www/nextcloud/occ preview:pre-generate** to a system cron job that runs at times of your choosing.]]>
- 5.12.0-dev.2
+ 5.12.0-dev.3
agpl
Richard Steinmetz
PreviewGenerator
@@ -30,7 +30,9 @@ The first time you install this app, before using a cron job, you properly want
-
+
+ OCA\PreviewGenerator\BackgroundJob\PreviewJob
+
OCA\PreviewGenerator\Command\Generate
OCA\PreviewGenerator\Command\PreGenerate
diff --git a/lib/BackgroundJob/PreviewJob.php b/lib/BackgroundJob/PreviewJob.php
new file mode 100644
index 0000000..5e437bf
--- /dev/null
+++ b/lib/BackgroundJob/PreviewJob.php
@@ -0,0 +1,54 @@
+limiter = new MultiLimiter([
+ new CountLimiter($this->configService->getMaxBackgroundJobPreviews()),
+ new ExecutionTimeLimiter(
+ $time,
+ $this->configService->getMaxBackgroundJobExecutionTime(),
+ ),
+ ]);
+
+ $this->setInterval(5 * 60);
+ $this->setTimeSensitivity(self::TIME_SENSITIVE);
+ }
+
+ protected function run($argument) {
+ if ($this->configService->isBackgroundJobDisabled()
+ || !$this->configService->usesCronDaemon()) {
+ return;
+ }
+
+ $this->preGenerateService->setLimiter($this->limiter);
+ $this->preGenerateService->preGenerate($this->outputInterface);
+ }
+}
diff --git a/lib/Command/Generate.php b/lib/Command/Generate.php
index 5fe7f98..6e647a8 100644
--- a/lib/Command/Generate.php
+++ b/lib/Command/Generate.php
@@ -97,7 +97,7 @@ protected function configure(): void {
protected function execute(InputInterface $input, OutputInterface $output): int {
if ($this->encryptionManager->isEnabled()) {
- $output->writeln('Encryption is enabled. Aborted.');
+ $output->writeln('Encryption is enabled. Aborted.');
return 1;
}
diff --git a/lib/Command/PreGenerate.php b/lib/Command/PreGenerate.php
index df7974c..397653b 100644
--- a/lib/Command/PreGenerate.php
+++ b/lib/Command/PreGenerate.php
@@ -9,73 +9,17 @@
namespace OCA\PreviewGenerator\Command;
-use OCA\PreviewGenerator\Service\NoMediaService;
-use OCA\PreviewGenerator\SizeHelper;
-use OCP\AppFramework\Db\TTransactional;
-use OCP\AppFramework\Utility\ITimeFactory;
-use OCP\Encryption\IManager;
-use OCP\Files\File;
-use OCP\Files\GenericFileException;
-use OCP\Files\IRootFolder;
-use OCP\Files\NotFoundException;
-use OCP\IConfig;
-use OCP\IDBConnection;
-use OCP\IPreview;
-use OCP\IUserManager;
+use OCA\PreviewGenerator\Exceptions\EncryptionEnabledException;
+use OCA\PreviewGenerator\Service\PreGenerateService;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class PreGenerate extends Command {
- use TTransactional;
-
- /* @return array{width: int, height: int, crop: bool} */
- protected array $specifications;
-
- protected string $appName;
- protected IUserManager $userManager;
- protected IRootFolder $rootFolder;
- protected IPreview $previewGenerator;
- protected IConfig $config;
- protected IDBConnection $connection;
- protected OutputInterface $output;
- protected IManager $encryptionManager;
- protected ITimeFactory $time;
- protected NoMediaService $noMediaService;
- protected SizeHelper $sizeHelper;
-
- /**
- * @param string $appName
- * @param IRootFolder $rootFolder
- * @param IUserManager $userManager
- * @param IPreview $previewGenerator
- * @param IConfig $config
- * @param IDBConnection $connection
- * @param IManager $encryptionManager
- * @param ITimeFactory $time
- */
- public function __construct(string $appName,
- IRootFolder $rootFolder,
- IUserManager $userManager,
- IPreview $previewGenerator,
- IConfig $config,
- IDBConnection $connection,
- IManager $encryptionManager,
- ITimeFactory $time,
- NoMediaService $noMediaService,
- SizeHelper $sizeHelper) {
+ public function __construct(
+ private readonly PreGenerateService $preGenerateService,
+ ) {
parent::__construct();
-
- $this->appName = $appName;
- $this->userManager = $userManager;
- $this->rootFolder = $rootFolder;
- $this->previewGenerator = $previewGenerator;
- $this->config = $config;
- $this->connection = $connection;
- $this->encryptionManager = $encryptionManager;
- $this->time = $time;
- $this->noMediaService = $noMediaService;
- $this->sizeHelper = $sizeHelper;
}
protected function configure(): void {
@@ -85,119 +29,10 @@ protected function configure(): void {
}
protected function execute(InputInterface $input, OutputInterface $output): int {
- if ($this->encryptionManager->isEnabled()) {
- $output->writeln('Encryption is enabled. Aborted.');
- return 1;
- }
-
- // Set timestamp output
- $formatter = new TimestampFormatter($this->config, $output->getFormatter());
- $output->setFormatter($formatter);
- $this->output = $output;
-
- $this->specifications = $this->sizeHelper->generateSpecifications();
- if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERY_VERBOSE) {
- $output->writeln('Specifications: ' . json_encode($this->specifications));
- }
- $this->startProcessing();
-
- return 0;
- }
-
- private function startProcessing(): void {
- while (true) {
- /*
- * Get and delete the row so that if preview generation fails for some reason the next
- * run can just continue. Wrap in transaction to make sure that one row is handled by
- * one process only.
- */
- $row = $this->atomic(function () {
- $qb = $this->connection->getQueryBuilder();
- $qb->select('*')
- ->from('preview_generation')
- ->orderBy('id')
- ->setMaxResults(1);
- $result = $qb->executeQuery();
- $row = $result->fetch();
- $result->closeCursor();
-
- if (!$row) {
- return null;
- }
-
- $qb = $this->connection->getQueryBuilder();
- $qb->delete('preview_generation')
- ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
- $qb->executeStatement();
-
- return $row;
- }, $this->connection);
-
-
- if (!$row) {
- break;
- }
-
- $this->processRow($row);
- }
- }
-
- private function processRow($row): void {
- //Get user
- $user = $this->userManager->get($row['uid']);
-
- if ($user === null) {
- return;
- }
-
- \OC_Util::tearDownFS();
- \OC_Util::setupFS($row['uid']);
-
try {
- $userFolder = $this->rootFolder->getUserFolder($user->getUID());
- $userRoot = $userFolder->getParent();
- } catch (NotFoundException $e) {
- return;
- }
-
- //Get node
- $nodes = $userRoot->getById($row['file_id']);
-
- if ($nodes === []) {
- return;
- }
-
- $node = $nodes[0];
- if ($node instanceof File) {
- $this->processFile($node);
- }
- }
-
- private function processFile(File $file): void {
- $absPath = ltrim($file->getPath(), '/');
- $pathComponents = explode('/', $absPath);
- if (isset($pathComponents[1]) && $pathComponents[1] === 'files_trashbin') {
- return;
- }
-
- if ($this->noMediaService->hasNoMediaFile($file)) {
- return;
- }
-
- if ($this->previewGenerator->isMimeSupported($file->getMimeType())) {
- if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) {
- $this->output->writeln('Generating previews for ' . $file->getPath());
- }
-
- try {
- $this->previewGenerator->generatePreviews($file, $this->specifications);
- } catch (NotFoundException $e) {
- // Maybe log that previews could not be generated?
- } catch (\InvalidArgumentException|GenericFileException $e) {
- $class = $e::class;
- $error = $e->getMessage();
- $this->output->writeln("{$class}: {$error}");
- }
+ $this->preGenerateService->preGenerate($output);
+ } catch (EncryptionEnabledException $e) {
+ $output->writeln('Encryption is enabled. Aborted.');
}
}
}
diff --git a/lib/Exceptions/EncryptionEnabledException.php b/lib/Exceptions/EncryptionEnabledException.php
new file mode 100644
index 0000000..7ca11e0
--- /dev/null
+++ b/lib/Exceptions/EncryptionEnabledException.php
@@ -0,0 +1,21 @@
+config->getSystemValue('preview_max_x', 4096);
+ }
+
+ public function getPreviewMaxY(): int {
+ return (int)$this->config->getSystemValue('preview_max_y', 4096);
+ }
+
+ public function isBackgroundJobDisabled(): bool {
+ return $this->appConfig->getAppValueBool('job_disabled');
+ }
+
+ public function getMaxBackgroundJobExecutionTime(): int {
+ return $this->appConfig->getAppValueInt('job_max_execution_time', 5 * 60);
+ }
+
+ public function getMaxBackgroundJobPreviews(): int {
+ return $this->appConfig->getAppValueInt('job_max_previews', 0);
+ }
+
+ public function usesCronDaemon(): bool {
+ return $this->config->getAppValue('core', 'backgroundjobs_mode') === 'cron';
+ }
+}
diff --git a/lib/Service/PreGenerateService.php b/lib/Service/PreGenerateService.php
new file mode 100644
index 0000000..2b5789d
--- /dev/null
+++ b/lib/Service/PreGenerateService.php
@@ -0,0 +1,175 @@
+limiter = $limiter;
+ }
+
+ /**
+ * @throws EncryptionEnabledException If encryption is enabled.
+ */
+ public function preGenerate(OutputInterface $output): void {
+ if ($this->encryptionManager->isEnabled()) {
+ throw new EncryptionEnabledException();
+ }
+
+ // Set timestamp output
+ $formatter = new TimestampFormatter($this->config, $output->getFormatter());
+ $output->setFormatter($formatter);
+ $this->output = $output;
+
+ if ($this->limiter) {
+ $output->writeln('Using limiter: ' . get_class($this->limiter));
+ }
+
+ $this->specifications = $this->sizeHelper->generateSpecifications();
+ if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERY_VERBOSE) {
+ $output->writeln('Specifications: ' . json_encode($this->specifications));
+ }
+ $this->startProcessing();
+ }
+
+ private function startProcessing(): void {
+ while ($this->limiter?->next() ?? true) {
+ /*
+ * Get and delete the row so that if preview generation fails for some reason the next
+ * run can just continue. Wrap in transaction to make sure that one row is handled by
+ * one process only.
+ */
+ $row = $this->atomic(function () {
+ $qb = $this->connection->getQueryBuilder();
+ $qb->select('*')
+ ->from('preview_generation')
+ ->orderBy('id')
+ ->setMaxResults(1);
+ $result = $qb->executeQuery();
+ $row = $result->fetch();
+ $result->closeCursor();
+
+ if (!$row) {
+ return null;
+ }
+
+ $qb = $this->connection->getQueryBuilder();
+ $qb->delete('preview_generation')
+ ->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
+ $qb->executeStatement();
+
+ return $row;
+ }, $this->connection);
+
+
+ if (!$row) {
+ break;
+ }
+
+ $this->processRow($row);
+ }
+ }
+
+ private function processRow($row): void {
+ //Get user
+ $user = $this->userManager->get($row['uid']);
+
+ if ($user === null) {
+ return;
+ }
+
+ \OC_Util::tearDownFS();
+ \OC_Util::setupFS($row['uid']);
+
+ try {
+ $userFolder = $this->rootFolder->getUserFolder($user->getUID());
+ $userRoot = $userFolder->getParent();
+ } catch (NotFoundException $e) {
+ return;
+ }
+
+ //Get node
+ $nodes = $userRoot->getById($row['file_id']);
+
+ if ($nodes === []) {
+ return;
+ }
+
+ $node = $nodes[0];
+ if ($node instanceof File) {
+ $this->processFile($node);
+ }
+ }
+
+ private function processFile(File $file): void {
+ $absPath = ltrim($file->getPath(), '/');
+ $pathComponents = explode('/', $absPath);
+ if (isset($pathComponents[1]) && $pathComponents[1] === 'files_trashbin') {
+ return;
+ }
+
+ if ($this->noMediaService->hasNoMediaFile($file)) {
+ return;
+ }
+
+ if ($this->previewGenerator->isMimeSupported($file->getMimeType())) {
+ if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) {
+ $this->output->writeln('Generating previews for ' . $file->getPath());
+ }
+
+ try {
+ $this->previewGenerator->generatePreviews($file, $this->specifications);
+ } catch (NotFoundException $e) {
+ // Maybe log that previews could not be generated?
+ } catch (\InvalidArgumentException|GenericFileException $e) {
+ $class = $e::class;
+ $error = $e->getMessage();
+ $this->output->writeln("{$class}: {$error}");
+ }
+ }
+ }
+}
diff --git a/lib/SizeHelper.php b/lib/SizeHelper.php
index ab6e2b2..44c8764 100644
--- a/lib/SizeHelper.php
+++ b/lib/SizeHelper.php
@@ -10,12 +10,14 @@
namespace OCA\PreviewGenerator;
use OCA\PreviewGenerator\AppInfo\Application;
+use OCA\PreviewGenerator\Service\ConfigService;
use OCP\IConfig;
use OCP\IPreview;
class SizeHelper {
public function __construct(
private IConfig $config,
+ private ConfigService $configService,
) {
}
@@ -35,8 +37,8 @@ public function generateSpecifications(): array {
'width' => [],
];
- $maxW = (int)$this->config->getSystemValue('preview_max_x', 4096);
- $maxH = (int)$this->config->getSystemValue('preview_max_y', 4096);
+ $maxW = $this->configService->getPreviewMaxX();
+ $maxH = $this->configService->getPreviewMaxY();
$s = 64;
while ($s <= $maxW || $s <= $maxH) {
diff --git a/lib/Support/OutputInterfaceLoggerAdapter.php b/lib/Support/OutputInterfaceLoggerAdapter.php
new file mode 100644
index 0000000..58180d6
--- /dev/null
+++ b/lib/Support/OutputInterfaceLoggerAdapter.php
@@ -0,0 +1,99 @@
+logLevel = $logLevel;
+ }
+
+ public function writeln(string|iterable $messages, int $options = 0) {
+ if (is_iterable($messages)) {
+ $message = implode(' ', [...$messages]);
+ } else {
+ $message = $messages;
+ }
+
+ $this->logger->log($this->logLevel, $message, [
+ 'source' => self::class,
+ ]);
+ }
+
+ public function write(
+ iterable|string $messages,
+ bool $newline = false,
+ int $options = 0
+ ) {
+ $this->writeln($messages, $options);
+ }
+
+ public function setVerbosity(int $level) {
+ $this->logLevel = match ($level) {
+ self::VERBOSITY_DEBUG => LogLevel::DEBUG,
+ self::VERBOSITY_VERY_VERBOSE => LogLevel::INFO,
+ self::VERBOSITY_VERBOSE => LogLevel::NOTICE,
+ self::VERBOSITY_NORMAL => LogLevel::WARNING,
+ _ => LogLevel::ERROR,
+ };
+ }
+
+ public function getVerbosity(): int {
+ return match ($this->logLevel) {
+ LogLevel::DEBUG => self::VERBOSITY_DEBUG,
+ LogLevel::INFO => self::VERBOSITY_VERY_VERBOSE,
+ LogLevel::NOTICE => self::VERBOSITY_VERBOSE,
+ LogLevel::WARNING => self::VERBOSITY_NORMAL,
+ _ => self::VERBOSITY_QUIET,
+ };
+ }
+
+ public function isQuiet(): bool {
+ return $this->getVerbosity() === self::VERBOSITY_QUIET;
+ }
+
+ public function isVerbose(): bool {
+ return $this->getVerbosity() === self::VERBOSITY_VERBOSE;
+ }
+
+ public function isVeryVerbose(): bool {
+ return $this->getVerbosity() === self::VERBOSITY_VERY_VERBOSE;
+ }
+
+ public function isDebug(): bool {
+ return $this->getVerbosity() === self::VERBOSITY_DEBUG;
+ }
+
+ public function setDecorated(bool $decorated) {
+ }
+
+ public function isDecorated(): bool {
+ throw new RuntimeException('Not implemented');
+ }
+
+ public function setFormatter(OutputFormatterInterface $formatter) {
+ }
+
+ public function getFormatter(): OutputFormatterInterface {
+ throw new RuntimeException('Not implemented');
+ }
+}
diff --git a/lib/Support/PreviewLimiter/CountLimiter.php b/lib/Support/PreviewLimiter/CountLimiter.php
new file mode 100644
index 0000000..b8c30f2
--- /dev/null
+++ b/lib/Support/PreviewLimiter/CountLimiter.php
@@ -0,0 +1,28 @@
+previews >= $this->maxPreviews) {
+ return false;
+ }
+
+ $this->previews++;
+ return true;
+ }
+}
diff --git a/lib/Support/PreviewLimiter/ExecutionTimeLimiter.php b/lib/Support/PreviewLimiter/ExecutionTimeLimiter.php
new file mode 100644
index 0000000..a582b9e
--- /dev/null
+++ b/lib/Support/PreviewLimiter/ExecutionTimeLimiter.php
@@ -0,0 +1,27 @@
+deadline = $time->getTime() + $this->maxExecutionTimeSeconds;
+ }
+
+ public function next(): bool {
+ return $this->time->getTime() < $this->deadline;
+ }
+}
diff --git a/lib/Support/PreviewLimiter/MultiLimiter.php b/lib/Support/PreviewLimiter/MultiLimiter.php
new file mode 100644
index 0000000..189694a
--- /dev/null
+++ b/lib/Support/PreviewLimiter/MultiLimiter.php
@@ -0,0 +1,30 @@
+limiters as $limiter) {
+ if (!$limiter->next()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
diff --git a/lib/Support/PreviewLimiter/PreviewLimiter.php b/lib/Support/PreviewLimiter/PreviewLimiter.php
new file mode 100644
index 0000000..03cb26e
--- /dev/null
+++ b/lib/Support/PreviewLimiter/PreviewLimiter.php
@@ -0,0 +1,14 @@
+