diff --git a/bin/watch b/bin/watch index 9934f83e..dda345b6 100755 --- a/bin/watch +++ b/bin/watch @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import datetime +import os import subprocess - -from inotify.adapters import InotifyTree +import time +from pathlib import Path +from time import sleep def build_docs(): @@ -15,54 +16,57 @@ def refresh_assets(): ) -def watch_for_changes(debug=False): - i = InotifyTree("docs") +def watch_for_changes(): + """Watch for changes using watchdog.""" + from watchdog.events import FileSystemEvent, FileSystemEventHandler + from watchdog.observers import Observer + + class MyHandler(FileSystemEventHandler): + def __init__(self): + super().__init__() + self.last_build = time.time() - IGNORED_EVENTS = ["IN_CLOSE_WRITE", "IN_CLOSE_NOWRITE", "IN_ACCESS", "IN_OPEN"] - IGNORED_PATHS = ["docs/_build"] + def on_modified(self, event: FileSystemEvent): + cwd = os.getcwd() + file_path = Path(event.src_path) - last_build = None + IGNORED_PATHS = ["docs/_build"] - for event in i.event_gen(yield_nones=False): - (_, type_names, path, filename) = event + if event.is_directory: + return - ignore = False - for ignored_event_type in IGNORED_EVENTS: - if ignored_event_type in type_names: - ignore = True - break + if file_path.name.isdigit(): + return - for ignored_path in IGNORED_PATHS: - if path.startswith(ignored_path): - ignore = True - break + if ( + file_path.name.endswith(".swx") + or file_path.name.endswith(".swp") + or file_path.name.endswith("~") + ): + return - if ( - filename.endswith(".swx") - or filename.endswith(".swp") - or filename.endswith("~") - ): - ignore = True + for ignored_path in IGNORED_PATHS: + ignored_path = os.path.join(cwd, ignored_path) + if str(file_path).startswith(ignored_path): + return - if ignore: - continue + if self.last_build and time.time() - self.last_build < 0.5: + return - if last_build: - # If build has been triggered in the last 50 ms, skip - delta = (datetime.datetime.now() - last_build).microseconds / 1000 - if delta < 50: - continue + build_docs() + refresh_assets() + self.last_build = time.time() - build_docs() - refresh_assets() - last_build = datetime.datetime.now() + event_handler = MyHandler() + observer = Observer() + observer.schedule(event_handler, path="docs", recursive=True) + observer.start() - if debug: - print( - "PATH=[{}] FILENAME=[{}] EVENT_TYPES={}".format( - path, filename, type_names - ) - ) + try: + while True: + sleep(1) + except KeyboardInterrupt: + observer.stop() if __name__ == "__main__": diff --git a/docs/best-practices/database/how-to-run-rabbitmq-on-hypernode.md b/docs/best-practices/database/how-to-run-rabbitmq-on-hypernode.md index 3e145071..014cf022 100644 --- a/docs/best-practices/database/how-to-run-rabbitmq-on-hypernode.md +++ b/docs/best-practices/database/how-to-run-rabbitmq-on-hypernode.md @@ -55,7 +55,7 @@ You can run this command to restart RabbitMQ: hypernode-servicectl restart rabbitmq-server ``` -## Accessing RabbitMQ +## Accessing the RabbitMQ admin interface - RabbitMQ only binds on localhost - The default admin account is username `guest` and password `guest`. You can change and add users via the admin interface. @@ -67,22 +67,29 @@ Use your browser to go to `localhost:55672` and logon using guest/guest. Another way to access the admin interface is via [hypernode-vpn](https://changelog.hypernode.com/changelog/release-6064-rabbitmq-can-be-accessed-via-the-hypernode-vpn/) -## RabbitMQ and Magento - -You need to make some changes in Magento in order to use RabbitMQ. For example in `/data/web/magento2/app/etc/env.php`: - -```php -'queue' => - array ( - 'amqp' => - array ( - 'host' => 'rabbitmqmaster', - 'port' => '5672', - 'user' => 'guest', - 'password' => 'guest', - 'virtualhost' => '/', - ), - ), +## Creating RabbitMQ users + +You can also create your own RabbitMQ users. This can be done by in the RabbitMQ admin interface, as follows: + +1. Go to the RabbitMQ admin interface +1. Click on the `Admin` tab +1. Click on `Add a user` +1. Fill in the username and password +1. Click on `Add user` +1. Click on the user you just created +1. Click on `Set permission` + +## RabbitMQ and Magento 2 + +To configure RabbitMQ in Magento 2, you can run the following command: + +```bash +bin/magento setup:config:set \ + --amqp-host="rabbitmqmaster" \ + --amqp-port="5672" \ + --amqp-user="guest" \ + --amqp-password="guest" \ + --amqp-virtualhost="/" ``` Note: Hypernode provisions a non-default user called `hypernode-admin` but you are free to create new users. diff --git a/docs/ecommerce-applications/magento-2/how-to-configure-remote-storage-for-magento-2-x.md b/docs/ecommerce-applications/magento-2/how-to-configure-remote-storage-for-magento-2-x.md new file mode 100644 index 00000000..fc1bfa0e --- /dev/null +++ b/docs/ecommerce-applications/magento-2/how-to-configure-remote-storage-for-magento-2-x.md @@ -0,0 +1,115 @@ +--- +myst: + html_meta: + description: Configure remote storage for Magento 2.x. Learn how to configure + Magento 2 to start storing files in your bucket using a single command. + title: How to Configure Remote Storage for Magento 2.x | Hypernode +--- + +# How to Configure Remote Storage for Magento 2.x + +Magento 2.x supports remote storage for media files, import/export files, and other files. +This feature allows you to store files in a remote storage location, such as an Amazon S3 bucket, instead of storing them on the server itself. + +This can be useful for many reasons, such as: + +- Offloading storage from your server, reducing the load on your server, and improving performance. +- Allows you to make use of [horizontal scaling](../../hypernode-platform/autoscaling/how-does-horizontal-autoscaling-work), as you can easily add more servers without having to worry about syncing files between them. +- Allows for effortless storage capacity scaling, as you can easily increase the storage capacity of your remote storage location. +- Serving assets from a CDN, which can improve the performance of your website. + +## Configuring the application + +Configuring Magento 2 to start storing files in your bucket is done using a single command. + +**AWS S3** + +```bash +bin/magento setup:config:set \ + --remote-storage-driver="aws-s3" \ + --remote-storage-bucket="my_bucket_name" \ + --remote-storage-region="my-aws-region" \ + --remote-storage-key="abcd1234" \ + --remote-storage-secret="abcd1234" +``` + +**Other S3 compatible providers** + +If you're using a different provider than AWS S3, you need to specify the `--remote-storage-endpoint` option. + +```bash +bin/magento setup:config:set \ + --remote-storage-driver="aws-s3" \ + --remote-storage-bucket="my_bucket_name" \ + --remote-storage-region="provider-region" \ + --remote-storage-key="abcd1234" \ + --remote-storage-secret="abcd1234" \ + --remote-storage-endpoint="https://my-s3-compatible.endpoint.com" +``` + +## Syncing the files + +Instead of running (which is Magento's official way to do this): + +```bash +bin/magento remote-storage:sync +``` + +One can run the following instead to really speed up the process: + +```bash +aws s3 sync pub/media/ s3://my_bucket_name/media/ +aws s3 sync var/import_export s3://my_bucket_name/import_export +``` + +This is much faster than Magento's built-in sync, because `aws s3 sync` uploads files concurrently. + +## The storage flag file in the bucket + +Magento's S3 implementation creates a test file called `storage.flag`, which is basically created to test if the connection works. So this is not a magic file to mark anything ([source](https://github.com/magento/magento2/blob/6f4805f82bb7511f72935daa493d48ebda3d9039/app/code/Magento/AwsS3/Driver/AwsS3.php#L104)). + +## Serving assets from your S3 bucket + +To start serving media assets from your S3 bucket, you need to make some adjustments to your nginx configuration. + +```nginx +location /media { + # ... + location ~* \.(ico|jpg|jpeg|png|gif|svg|js|css|swf|eot|ttf|otf|woff|woff2)$ { + resolver 8.8.8.8; + set $bucket ""; + proxy_pass https://s3.amazonaws.com/$bucket$uri; + proxy_pass_request_body off; + proxy_pass_request_headers off; + proxy_intercept_errors on; + proxy_hide_header "x-amz-id-2"; + proxy_hide_header "x-amz-request-id"; + proxy_hide_header "x-amz-storage-class"; + proxy_hide_header "Set-Cookie"; + proxy_ignore_headers "Set-Cookie"; + } + # ... +} +``` + +Also make sure your S3 bucket policies are configured correctly, so that only `/media` is publicly readable. For example: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowPublicReadOnlyForMedia", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::/media/*" + } + ] +} +``` + +## Magento remote storage documentation + +- [Configure Remote Storage](https://experienceleague.adobe.com/en/docs/commerce-operations/configuration-guide/storage/remote-storage/remote-storage) +- [Configure AWS S3 bucket for remote storage](https://experienceleague.adobe.com/en/docs/commerce-operations/configuration-guide/storage/remote-storage/remote-storage-aws-s3) diff --git a/docs/hypernode-platform/autoscaling/how-does-horizontal-autoscaling-work.md b/docs/hypernode-platform/autoscaling/how-does-horizontal-autoscaling-work.md index 880e95c3..e97bf4a5 100644 --- a/docs/hypernode-platform/autoscaling/how-does-horizontal-autoscaling-work.md +++ b/docs/hypernode-platform/autoscaling/how-does-horizontal-autoscaling-work.md @@ -63,7 +63,9 @@ We will divide them between Hypernode-specific and Application-specific requirem #### Operating system -- The operating system of the Hypernode should be Debian Bookworm. If you would like to upgrade the os of your Hypernode, feel free to contact our support team for help. https://www.hypernode.com/en/contact/ +Horizontal autoscaling is only available on Hypernodes with the Debian Bookworm operating system. + +If you would like to upgrade the operating system of your Hypernode, feel free to [contact our support team](https://www.hypernode.com/en/contact/) for help. #### Make sure the Hypernode is a production plan @@ -71,36 +73,31 @@ For now, we don't support horizontal autoscaling for development plans. #### Enable and configure Varnish -To make use of Horizontal autoscaling, Varnish should be enabled and configured on the Hypernode. -You can check if Varnish is enabled on your Hypernode by running +Varnish is a requirement, because it's essential to deliver cached pages from the load balancer instead of the application servers when scaled up. +The benefits are: -```console -hypernode-systemctl settings varnish_enabled -``` +- Pages in the cache are served faster, because it is one less network hop +- Saves network bandwidth +- Saves CPU usage for both load balancer and application servers +- Page cache is not affected by scaling up or down -Example output if Varnish is enabled: +You can check if Varnish is enabled on your Hypernode by running ```console +app@abcdef-example-magweb-cmbl:~$ hypernode-systemctl settings varnish_enabled varnish_enabled is set to value True ``` If Varnish is not enabled, you can [enable Varnish](../varnish/how-to-enable-varnish-on-hypernode.md) by following the documentation -If varnish is enabled on your Hypernode, your Magento store should also be configured to make use of varnish. -You can verify if the Varnish host is configured correctly by running the following command from the Magento root: +If Varnish is enabled on your Hypernode, your Magento store should also be configured to make use of varnish. -```console -php bin/magento config:show system/full_page_cache/varnish/backend_host -``` - -The output should show `varnish` as backend. If it is configured as something else (like `localhost` or `127.0.0.1`), you can update it by running which sets the backend host to `varnish` instead. +Verify that your VCL has the health probe disabled. While we do support the health probe, we do not recommend to use it in a basic setup. +The health probe can be removed by removing the `.probe` from your `backend default` section in the VCL file. +If you do, however, want to use the health probe, make sure that you set the backend host to `127.0.0.1` instead of `localhost`. -```console -php bin/magento config:set system/full_page_cache/varnish/backend_host varnish -``` +Additionally, make sure the IP range `10.0.0.0/24` is allowed under the `acl purge` section in the Varnish VCL file. The `acl purge` section should look like the following: -Additionally make sure the IP range `10.0.0.0/24` is set to the `acl_purge` section in the Varnish VCL file. The `acl_purge` section should look something similar:loaded Varnish VCL. - -```console +```vcl acl purge { "localhost"; "10.0.0.0/24"; @@ -109,15 +106,12 @@ acl purge { #### Make sure to use MySQL 5.7 or higher -The configured MySQL version should be 5.7 or above. You can check the enabled MySQL version by running the following command. - -```console -hypernode-systemctl settings mysql_version -``` +MySQL 5.7 or higher is a requirement, because those versions support auto negotiation of TLS. TLS is an essential part for horizontal autoscaling, because the application servers will connect to the MySQL instance over the internet instead of wireguard. -Example output if MySQL version is 8.0: +You can check the enabled MySQL version by running the following command: ```console +app@abcdef-example-magweb-cmbl:~$ hypernode-systemctl settings mysql_version mysql_version is set to value 8.0 ``` @@ -131,44 +125,54 @@ You should see something similar to `'host' => 'mysqlmaster',`. If this is not t Make sure supervisor is disabled and that there are no supervisor services configured. +For now, we don't support having supervisor services, because we can't guarantee everything works when scaling up. If you do think your use case should be supported, please let us know! + #### Disable Podman services Make sure podman is disabled and that there are no podman services running. +For now, we don't support having podman containers/services, because we can't guarantee everything works when scaling up. If you do think your use case should be supported, please let us know! + #### Configure hostnames correctly The database, cache, session and queue of the application must be configured with correct hostnames instead of `localhost` or `127.0.0.1`. This way the services will be available across multiple hypernodes when the app is autoscaled. Make sure the env variables (db, cache, session, queue) are not using localhost. -### Application Specific Requirements - Magento2 +### Application Specific Requirements - Magento 2 #### Supported CMS Horizontal autoscaling is available for Magento 2.4.7 and higher. To make use of Horizontal autoscaling, there are a couple of other requirements the application should meet. -#### Enable and configure Redis Persistent +#### Make sure Redis cache is configured -Redis persistent is another requirement before you can make use of Horizontal autoscaling. -The persistent instance will be used to store the sessions so we can access the same sessions from the Horizontal autoscale Hypernodes. +Having a centralized cache database is essential for a consistent experience for your application. -You can check if Redis Persistent is enabled on your Hypernode by running +Please make sure Redis cache is configured properly in the Magento 2 configuration file at `/app/etc/env.php`. -```console -hypernode-systemctl settings redis_persistent_instance -``` +For the Redis hostname, use `redismaster` instead of `localhost` or `127.0.0.1`. + +More information about [Redis on Hypernode](../../ecommerce-applications/magento-2/how-to-configure-redis-for-magento-2.md). + +#### Enable and configure Redis sessions + +Redis sessions on the persistent instance is also required to make use of Horizontal autoscaling. -Example output if Redis Persistent is enabled: +The persistent instance will be used to store the sessions so we can access the same sessions from the Horizontal autoscale Hypernodes. + +You can check if Redis Persistent is enabled on your Hypernode by running ```console +app@abcdef-example-magweb-cmbl:~$ hypernode-systemctl settings redis_persistent_instance redis_persistent_instance is set to value True ``` If Redis Persistent instance is not enabled, you can enable the second Redis instance for sessions you run the command: ```console -hypernode-systemctl settings redis_persistent_instance --value True +app@abcdef-example-magweb-cmbl:~$ hypernode-systemctl settings redis_persistent_instance True ``` Make sure Redis session is configured as [described](../../ecommerce-applications/magento-2/how-to-configure-redis-for-magento-2.md#configure-magento-2-to-use-redis-as-the-session-store) in our docs @@ -176,24 +180,45 @@ Please notice the Redis host in the setup documentation. The Redis host should b #### Make sure Elasticsearch/Opensearch is configured properly -Please make sure Elasticsearch or Opensearch host is set to `elasticsearchmaster` in the Magento2 configuration file at `/app/etc/env.php` -More information about [Elasticsearch on Hypernode](../../hypernode-platform/tools/how-to-use-elasticsearch-on-hypernode.md) +Please make sure Elasticsearch or Opensearch host is set to `elasticsearchmaster` your Magento 2 configuration. + +To set the hostname, you can run the following command for OpenSearch: + +```console +app@abcdef-example-magweb-cmbl:~$ bin/magento config:set catalog/search/opensearch_server_hostname elasticsearchmaster +``` + +For ElasticSearch: + +```console +app@abcdef-example-magweb-cmbl:~$ bin/magento config:set catalog/search/elasticsearch7_server_hostname elasticsearchmaster +``` + +More information about [Elasticsearch on Hypernode](../../hypernode-platform/tools/how-to-use-elasticsearch-on-hypernode.md). #### Make sure RabbitMQ configured properly -Please make sure RabbitMQ host is set to `rabbitmqmaster` in the Magento2 configuration file at `/app/etc/env.php` -More information about [RabbitMQ o Hypernode](../../best-practices/database/how-to-run-rabbitmq-on-hypernode.md) +Please make sure RabbitMQ host is set to `rabbitmqmaster` in the Magento 2 configuration file at `/app/etc/env.php` +More information about [RabbitMQ on Hypernode](../../best-practices/database/how-to-run-rabbitmq-on-hypernode.md) -There is a rabbitmq user provisioned by Hypernode called hypernode-admin as a non-default user. But you can also configure RabbitMQ with a new different user of your own. -But please make sure to configure RabbitMQ without the default guest user. +There is a rabbitmq user provisioned by Hypernode called `hypernode-admin` as a non-default user, but you can also configure RabbitMQ with a new different user of your own. +Please make sure to configure RabbitMQ without the default guest user. -#### Make sure Database storage is disabled & Remote storage is enabled and configured. +For example, you can run the following command to change your RabbitMQ config in Magento 2: + +```bash +bin/magento setup:config:set \ + --amqp-host="rabbitmqmaster" \ + --amqp-user="my_rabbitmq_user" \ + --amqp-password="my_rabbitmq_password" +``` -Please make sure to enable remote storage for your application and configure it correctly as only AWS-s3 remote storage drivers are supported. +#### Make sure database storage is disabled & remote storage is enabled and configured -Make sure that the `remote_storage` key is present in the Magento2 configuration file at `/app/etc/env.php` with the correct config. +Remote storage is a requirement for Horizontal autoscaling. This is because the media files should be stored on a remote storage location instead of the local filesystem. +This way the media files are available across multiple Hypernodes when the app is scaled up, and after scaling down the media files are still available. -More information about [S3 Remote Storage with Magento2](https://experienceleague.adobe.com/en/docs/commerce-operations/configuration-guide/storage/remote-storage/remote-storage-aws-s3) +Follow our documentation on [remote storage for Magento 2](../../ecommerce-applications/magento-2/how-to-configure-remote-storage-for-magento-2-x) to configure remote storage. ## Enabling Horizontal Autoscaling diff --git a/requirements/development.txt b/requirements/development.txt index 5ee18e77..d47687d4 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -8,4 +8,4 @@ pytest-xdist==2.5.0 mypy==1.0.0 flake8==3.9.2 tox==3.25.0 -inotify==0.2.10 +watchdog