WIP Proposal: Unified authoring app

.annotation_safe_list.yml

@@ -19,43 +19,43 @@ auth.User:
   ".. pii_retirement": "consumer_api"
 contenttypes.ContentType:
   ".. no_pii:": "This model has no PII"
-oel_collections.Collection:
+oel_authoring.Collection:
   ".. no_pii:": "This model has no PII"
-oel_collections.CollectionPublishableEntity:
+oel_authoring.CollectionPublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_components.Component:
+oel_authoring.Component:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentType:
+oel_authoring.ComponentType:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersion:
+oel_authoring.ComponentVersion:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersionContent:
+oel_authoring.ComponentVersionContent:
   ".. no_pii:": "This model has no PII"
-oel_contents.Content:
+oel_authoring.Content:
   ".. no_pii:": "This model has no PII"
-oel_contents.MediaType:
+oel_authoring.MediaType:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Container:
+oel_authoring.Container:
   ".. no_pii:": "This model has no PII"
-oel_publishing.ContainerVersion:
+oel_authoring.ContainerVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Draft:
+oel_authoring.Draft:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityList:
+oel_authoring.EntityList:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityListRow:
+oel_authoring.EntityListRow:
   ".. no_pii:": "This model has no PII"
-oel_publishing.LearningPackage:
+oel_authoring.LearningPackage:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLog:
+oel_authoring.PublishLog:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLogRecord:
+oel_authoring.PublishLogRecord:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntity:
+oel_authoring.PublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntityVersion:
+oel_authoring.PublishableEntityVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Published:
+oel_authoring.Published:
   ".. no_pii:": "This model has no PII"
 oel_tagging.ObjectTag:
   ".. no_pii:": "This model has no PII"
@@ -65,17 +65,17 @@ oel_tagging.TagImportTask:
   ".. no_pii:": "This model has no PII"
 oel_tagging.Taxonomy:
   ".. no_pii:": "This model has no PII"
-oel_sections.Section:
+oel_authoring.Section:
   ".. no_pii:": "This model has no PII"
-oel_sections.SectionVersion:
+oel_authoring.SectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_subsections.Subsection:
+oel_authoring.Subsection:
   ".. no_pii:": "This model has no PII"
-oel_subsections.SubsectionVersion:
+oel_authoring.SubsectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_units.Unit:
+oel_authoring.Unit:
   ".. no_pii:": "This model has no PII"
-oel_units.UnitVersion:
+oel_authoring.UnitVersion:
   ".. no_pii:": "This model has no PII"
 social_django.Association:
   ".. no_pii:": "This model has no PII"

.gitignore

@@ -75,7 +75,12 @@ venv/
 !.vscode/settings.json.example
 
 # Media files (for uploads)
-media/
+/media/
 
 # Media files generated during test runs
-test_media/
+/test_media/
+
+# uv stuff
+.lock
+CACHEDIR.TAG
+pyvenv.cfg

.importlinter

@@ -36,24 +36,24 @@ layers=
     openedx_learning.api.authoring
 
     # The "backup_restore" app handle the new export and import mechanism.
-    openedx_learning.apps.authoring.backup_restore
+    openedx_learning.apps.authoring.applets.backup_restore
 
     # The "components" app is responsible for storing versioned Components,
     # which is Open edX Studio terminology maps to things like individual
     # Problems, Videos, and blocks of HTML text. This is also the type we would
     # associate with a single "leaf" XBlock–one that is not a container type and
     # has no child elements.
-    openedx_learning.apps.authoring.components
+    openedx_learning.apps.authoring.applets.components
 
     # The "contents" app stores the simplest pieces of binary and text data,
     # without versioning information. These belong to a single Learning Package.
-    openedx_learning.apps.authoring.contents
+    openedx_learning.apps.authoring.applets.contents
 
     # The "collections" app stores arbitrary groupings of PublishableEntities.
     # Its only dependency should be the publishing app.
-    openedx_learning.apps.authoring.collections
+    openedx_learning.apps.authoring.applets.collections
 
     # The lowest layer is "publishing", which holds the basic primitives needed
     # to create Learning Packages and manage the draft and publish states for
     # various types of content.
-    openedx_learning.apps.authoring.publishing
+    openedx_learning.apps.authoring.applets.publishing

docs/decisions/0016-python-public-api-conventions.rst

@@ -20,7 +20,7 @@ Learning Core Django apps will be grouped into packages.
   Apps in ``openedx_learning`` will be grouped into broadly related packages under ``openedx_learning.apps``. The first of these groups will be "authoring" (``openedx_learning.apps.authoring``). Future packages may include "learner", "personalization", "activity", "grading", etc.
 
 Learning Core Django apps will continue to have their own ``api`` modules.
-  So for example, ``openedx_learning.apps.authoring.components.api`` will continue to exist.
+  So for example, ``openedx_learning.apps.authoring.applets.components.api`` will continue to exist.
 
 Learning Core will have a top level package for its public API.
   All public APIs intended for use by consumers of Learning Core will be represented as modules in the ``openedx_learning.api`` package that corresponds to the app groupings (e.g. ``openedx_learning.api.authoring``).
@@ -35,14 +35,14 @@ App ``api`` modules will define their public functions using ``__all__``.
   This relies on the individual apps to properly set ``__all__`` to the list of functions that they are willing to publicly support.
 
 App ``api`` modules within a package of apps still import from each other.
-  So for example, ``openedx_learning.apps.authoring.components.api`` will continue to import APIs that it needs from ``..publishing.api``, instead of using the public API at ``openedx_learning.api.authoring``. These imports should not use wildcards.
+  So for example, ``openedx_learning.apps.authoring.applets.components.api`` will continue to import APIs that it needs from ``..publishing.api``, instead of using the public API at ``openedx_learning.api.authoring``. These imports should not use wildcards.
 
   Functions and constants that are not listed as part of a module's ``__all__`` may still be imported by other app APIs in the same package grouping. This should allow a package more flexibility to create provisional APIs that we may not want to support publicly.
 
   If a function or attribute is intended to be completely private to an app's ``api`` module (i.e. not used even by other apps in its package), it should be prefixed with an underscore.
 
 App ``api`` modules should not import directly from apps outside their package.
-  For example, ``openedx_learning.apps.personalization.api`` should import authoring API functions from ``openedx_learning.api.authoring``, **not** directly from something like ``openedx_learning.apps.authoring.components.api``. This will help to limit the impact of refactoring app package internal changes, as well as exposing shortcomings in the existing public APIs.
+  For example, ``openedx_learning.apps.personalization.api`` should import authoring API functions from ``openedx_learning.api.authoring``, **not** directly from something like ``openedx_learning.apps.authoring.applets.components.api``. This will help to limit the impact of refactoring app package internal changes, as well as exposing shortcomings in the existing public APIs.
 
 Public API modules may implement their own functions.
   In addition to aggregating app ``api`` modules via wildcard imports, public API modules like ``openedx_learning.api.authoring`` may implement their own functionality. This will be useful for convenience functions that invoke multiple app APIs, and for backwards compatibility shims. When possible, the bulk of the logic for these should continue to live in app-defined APIs, with the public API module acting more as a glue layer.

docs/decisions/0020-authoring-as-one-app.rst

@@ -0,0 +1,49 @@
+20. Authoring as an Umbrella App of Smaller Applets
+===================================================
+
+Context
+-------
+
+Up to this point, Learning Core has used many small apps with a narrow focus (e.g. ``components``, ``collections``, etc.) in order to make each individual app simpler to reason about. This has been useful overall, but it has made refactoring more cumbersome. For instance:
+
+#. Moving models between apps is tricky, requiring the use of Django's ``SeparateDatabaseAndState`` functionality to fake a deletion in one app and a creation in another without actually altering the database. It also requires doctoring the migration files for models in other repos that might have foreign key relations to the model being moved, so that they're pointing to the new ``app_label``.  This will be an issue when we try to extract container-related models and logic out of publishing and into a new ``containers`` app.
+#. Renaming an app is also cumbersome, because the process requires creating a new app and transitioning the models over. This came up when trying to rename the ``contents`` app to ``media``.
+
+There have also been minor inconveniences, like having a long list of ``INSTALLED_APPS`` to maintain in edx-platform over time, or not having these tables easily grouped together in the Django admin interface.
+
+Decisions
+---------
+
+1. Single Authoring App
+~~~~~~~~~~~~~~~~~~~~~~~
+
+All existing authoring apps will be merged into one Django app (``openedx_learning.app.authoring``). Some consequences of this decision:
+
+- The tables will be renamed to have the ``oel_authoring`` label prefix.
+- All management commands will be moved to the ``authoring`` app.
+
+2. Logical Separation via Applets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We will continue to keep internal API boundaries between individual applets, and use the ``api.py`` modules. This is both to insulate applets from implementation changes in other applets, as well as to provide a set of APIs that third-party plugins can utilize. As before, we will use Import Linter to enforce dependency ordering.
+
+3. Restructuring Specifics
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In one pull request, we are going to:
+
+#. Create bare shells of the existing ``authoring`` apps (``backup_restore``, ``collections``, ``components``, ``contents``, ``publishing``, ``sections``, ``subsections``, ``units``), and move them to the ``openedx_learning.apps.authoring.backcompat`` package. These shells will have an ``apps.py`` file and the ``migrations`` package for each existing app. This will allow for a smooth schema migration to transition the models from these individual apps to ``authoring``.
+#. Move the actual models files and API logic for our existing authoring apps to the ``openedx_learning.apps.authoring.applets`` package.
+#. Convert the top level ``openedx_learning.apps.authoring`` package to be a Django app. The top level ``admin.py``, ``api.py``, and ``models.py`` modules will do wildcard imports from the corresponding modules across all applet packages.
+
+In terms of model migrations, all existing apps will have a final migration that uses ``SeparateDatabaseAndState`` to remove all model state, but make no actual database changes. After that, the initial ``authoring`` app migration will list all these "deletion" migrations as dependencies, and then also use ``SeparateDatabaseAndState`` to create the model state without doing any actual database operations. The next ``authoring`` app migration will rename all existing
+database tables to use the ``oel_authoring`` prefix, for uniformity.
+
+There are a few edx-platform apps that already have foreign keys and migrations that reference these models. It will be necessary to alter those historical migrations to pretend that these models have always come from the ``authoring`` app (with the label ``oel_authoring``).
+
+In a future release (no earlier than Willow), we would remove the old apps entirely, and alter the intial ``authoring`` app migration so that it looks like a simple schema creation without state separation. We would also remove all references to the original set of small apps. This shouldn't affect existing installs because the ``authoring`` migration would have already run on those sites. This may require edx-platform apps to alter their migration dependencies to repoint to the initial ``authoring`` migration.
+
+4. The Bigger Picture
+~~~~~~~~~~~~~~~~~~~~~
+
+This practice means that the ``authoring`` Django app corresponds to a Subdomain in Domain Driven Design terminology, with each applet being a Bounded Context. We call these "Applets" instead of "Bounded Contexts" because we don't want it to get confused for Django's notion of Contexts and Context Processors (or Python's notion of Context Managers).

olx_importer/management/commands/load_components.py

@@ -28,9 +28,9 @@
 from django.db import transaction
 
 # Model references to remove
-from openedx_learning.apps.authoring.components import api as components_api
-from openedx_learning.apps.authoring.contents import api as contents_api
-from openedx_learning.apps.authoring.publishing import api as publishing_api
+from openedx_learning.apps.authoring.applets.components import api as components_api
+from openedx_learning.apps.authoring.applets.contents import api as contents_api
+from openedx_learning.apps.authoring.applets.publishing import api as publishing_api
 
 SUPPORTED_TYPES = ["problem", "video", "html"]
 logger = logging.getLogger(__name__)

openedx_learning/__init__.py

@@ -2,4 +2,4 @@
 Open edX Learning ("Learning Core").
 """
 
-__version__ = "0.30.2"
+__version__ = "0.31.0"

openedx_learning/api/authoring.py

@@ -9,14 +9,7 @@
 """
 # These wildcard imports are okay because these api modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.backup_restore.api import *
-from ..apps.authoring.collections.api import *
-from ..apps.authoring.components.api import *
-from ..apps.authoring.contents.api import *
-from ..apps.authoring.publishing.api import *
-from ..apps.authoring.sections.api import *
-from ..apps.authoring.subsections.api import *
-from ..apps.authoring.units.api import *
+from ..apps.authoring.api import *
 
 # This was renamed after the authoring API refactoring pushed this and other
 # app APIs into the openedx_learning.api.authoring module. Here I'm aliasing to

openedx_learning/api/authoring_models.py

@@ -7,10 +7,10 @@
 """
 # These wildcard imports are okay because these modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.collections.models import *
-from ..apps.authoring.components.models import *
-from ..apps.authoring.contents.models import *
-from ..apps.authoring.publishing.models import *
-from ..apps.authoring.sections.models import *
-from ..apps.authoring.subsections.models import *
-from ..apps.authoring.units.models import *
+from ..apps.authoring.applets.collections.models import *
+from ..apps.authoring.applets.components.models import *
+from ..apps.authoring.applets.contents.models import *
+from ..apps.authoring.applets.publishing.models import *
+from ..apps.authoring.applets.sections.models import *
+from ..apps.authoring.applets.subsections.models import *
+from ..apps.authoring.applets.units.models import *

openedx_learning/api/django.py

@@ -0,0 +1,23 @@
+"""
+Module for parts of the Learning Core API that exist to make it easier to use in
+Django projects.
+"""
+
+def learning_core_apps_to_install():
+    """
+    Return all app names for appending to INSTALLED_APPS.
+
+    This function exists to better insulate edx-platform and potential plugins
+    over time, as we eventually plan to remove the backcompat apps.
+    """
+    return [
+        "openedx_learning.apps.authoring",
+        "openedx_learning.apps.authoring.backcompat.backup_restore",
+        "openedx_learning.apps.authoring.backcompat.collections",
+        "openedx_learning.apps.authoring.backcompat.components",
+        "openedx_learning.apps.authoring.backcompat.contents",
+        "openedx_learning.apps.authoring.backcompat.publishing",
+        "openedx_learning.apps.authoring.backcompat.sections",
+        "openedx_learning.apps.authoring.backcompat.subsections",
+        "openedx_learning.apps.authoring.backcompat.units",
+    ]

openedx_learning/apps/authoring/admin.py

@@ -0,0 +1,13 @@
+"""
+This module aggregates all applet Django Admin modules.
+"""
+# pylint: disable=wildcard-import
+
+from .applets.backup_restore.admin import *
+from .applets.collections.admin import *
+from .applets.components.admin import *
+from .applets.contents.admin import *
+from .applets.publishing.admin import *
+from .applets.sections.admin import *
+from .applets.subsections.admin import *
+from .applets.units.admin import *

openedx_learning/apps/authoring/api.py

@@ -0,0 +1,16 @@
+"""
+This module aggregates all applet API modules.
+
+Question: Should this replace openedx_learning.api.authoring?
+"""
+
+# pylint: disable=wildcard-import
+
+from .applets.backup_restore.api import *
+from .applets.collections.api import *
+from .applets.components.api import *
+from .applets.contents.api import *
+from .applets.publishing.api import *
+from .applets.sections.api import *
+from .applets.subsections.api import *
+from .applets.units.api import *

openedx_learning/apps/authoring/backup_restore/api.py → openedx_learning/apps/authoring/applets/backup_restore/api.py

@@ -5,8 +5,8 @@
 
 from django.contrib.auth.models import User as UserType  # pylint: disable=imported-auth-user
 
-from openedx_learning.apps.authoring.backup_restore.zipper import LearningPackageUnzipper, LearningPackageZipper
-from openedx_learning.apps.authoring.publishing.api import get_learning_package_by_key
+from .zipper import LearningPackageUnzipper, LearningPackageZipper
+from ..publishing.api import get_learning_package_by_key
 
 
 def create_zip_file(lp_key: str, path: str, user: UserType | None = None, origin_server: str | None = None) -> None:

openedx_learning/apps/authoring/backup_restore/serializers.py → openedx_learning/apps/authoring/applets/backup_restore/serializers.py

@@ -5,7 +5,7 @@
 
 from rest_framework import serializers
 
-from openedx_learning.apps.authoring.components import api as components_api
+from ..components import api as components_api
 
 
 class LearningPackageSerializer(serializers.Serializer):  # pylint: disable=abstract-method

openedx_learning/apps/authoring/backup_restore/toml.py → openedx_learning/apps/authoring/applets/backup_restore/toml.py

@@ -8,10 +8,10 @@
 import tomlkit
 from django.contrib.auth.models import User as UserType  # pylint: disable=imported-auth-user
 
-from openedx_learning.apps.authoring.collections.models import Collection
-from openedx_learning.apps.authoring.publishing import api as publishing_api
-from openedx_learning.apps.authoring.publishing.models import PublishableEntity, PublishableEntityVersion
-from openedx_learning.apps.authoring.publishing.models.learning_package import LearningPackage
+from ..collections.models import Collection
+from ..publishing import api as publishing_api
+from ..publishing.models import PublishableEntity, PublishableEntityVersion
+from ..publishing.models.learning_package import LearningPackage
 
 
 def toml_learning_package(

openedx_learning/apps/authoring/backup_restore/zipper.py → openedx_learning/apps/authoring/applets/backup_restore/zipper.py

@@ -28,7 +28,7 @@
     PublishableEntity,
     PublishableEntityVersion,
 )
-from openedx_learning.apps.authoring.backup_restore.serializers import (
+from .serializers import (
     CollectionSerializer,
     ComponentSerializer,
     ComponentVersionSerializer,
@@ -37,21 +37,21 @@
     LearningPackageMetadataSerializer,
     LearningPackageSerializer,
 )
-from openedx_learning.apps.authoring.backup_restore.toml import (
+from .toml import (
     parse_collection_toml,
     parse_learning_package_toml,
     parse_publishable_entity_toml,
     toml_collection,
     toml_learning_package,
     toml_publishable_entity,
 )
-from openedx_learning.apps.authoring.collections import api as collections_api
-from openedx_learning.apps.authoring.components import api as components_api
-from openedx_learning.apps.authoring.contents import api as contents_api
-from openedx_learning.apps.authoring.publishing import api as publishing_api
-from openedx_learning.apps.authoring.sections import api as sections_api
-from openedx_learning.apps.authoring.subsections import api as subsections_api
-from openedx_learning.apps.authoring.units import api as units_api
+from ..collections import api as collections_api
+from ..components import api as components_api
+from ..contents import api as contents_api
+from ..publishing import api as publishing_api
+from ..sections import api as sections_api
+from ..subsections import api as subsections_api
+from ..units import api as units_api
 
 TOML_PACKAGE_NAME = "package.toml"
 DEFAULT_USERNAME = "command"